#include <sys/cdefs.h>
#include "opt_inet.h"
#include "opt_inet6.h"
#include "opt_ipsec.h"
#include "opt_ratelimit.h"
#include "opt_kern_tls.h"
#if defined(INET) || defined(INET6)
#include <sys/param.h>
#include <sys/arb.h>
#include <sys/module.h>
#include <sys/kernel.h>
#ifdef TCP_HHOOK
#include <sys/hhook.h>
#endif
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/mbuf.h>
#include <sys/proc.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#ifdef STATS
#include <sys/qmath.h>
#include <sys/tree.h>
#include <sys/stats.h>
#else
#include <sys/tree.h>
#endif
#include <sys/refcount.h>
#include <sys/queue.h>
#include <sys/tim_filter.h>
#include <sys/smp.h>
#include <sys/kthread.h>
#include <sys/kern_prefetch.h>
#include <sys/protosw.h>
#ifdef TCP_ACCOUNTING
#include <sys/sched.h>
#include <machine/cpu.h>
#endif
#include <vm/uma.h>
#include <net/route.h>
#include <net/route/nhop.h>
#include <net/vnet.h>
#define TCPSTATES
#include <netinet/in.h>
#include <netinet/in_kdtrace.h>
#include <netinet/in_pcb.h>
#include <netinet/ip.h>
#include <netinet/ip_var.h>
#include <netinet/ip6.h>
#include <netinet6/in6_pcb.h>
#include <netinet6/ip6_var.h>
#include <netinet/tcp.h>
#include <netinet/tcp_fsm.h>
#include <netinet/tcp_seq.h>
#include <netinet/tcp_timer.h>
#include <netinet/tcp_var.h>
#include <netinet/tcp_log_buf.h>
#include <netinet/tcp_syncache.h>
#include <netinet/tcp_hpts.h>
#include <netinet/tcp_ratelimit.h>
#include <netinet/tcp_accounting.h>
#include <netinet/tcpip.h>
#include <netinet/cc/cc.h>
#include <netinet/cc/cc_newreno.h>
#include <netinet/tcp_fastopen.h>
#include <netinet/tcp_lro.h>
#ifdef NETFLIX_SHARED_CWND
#include <netinet/tcp_shared_cwnd.h>
#endif
#ifdef TCP_OFFLOAD
#include <netinet/tcp_offload.h>
#endif
#ifdef INET6
#include <netinet6/tcp6_var.h>
#endif
#include <netinet/tcp_ecn.h>
#include <netipsec/ipsec_support.h>
#if defined(IPSEC) || defined(IPSEC_SUPPORT)
#include <netipsec/ipsec.h>
#include <netipsec/ipsec6.h>
#endif
#include <netinet/udp.h>
#include <netinet/udp_var.h>
#include <machine/in_cksum.h>
#ifdef MAC
#include <security/mac/mac_framework.h>
#endif
#include "sack_filter.h"
#include "tcp_rack.h"
#include "tailq_hash.h"
#include "rack_bbr_common.h"
MALLOC_DECLARE(M_TCPPCM);
void
rack_update_pcm_ack(struct tcp_rack *rack, int was_cumack, uint32_t start, uint32_t end)
{
struct rack_pcm_stats *e;
int i, completed = 0;
uint64_t ack_arrival;
int segsiz;
if (rack->pcm_in_progress == 0)
return;
if (SEQ_LEQ(end, rack->r_ctl.pcm_i.sseq)) {
return;
}
segsiz = ctf_fixed_maxseg(rack->rc_tp);
if (SEQ_GEQ(end, (rack->r_ctl.pcm_i.eseq - segsiz))) {
completed = 1;
rack->pcm_in_progress = 0;
rack->r_ctl.last_pcm_round = rack->r_ctl.current_round;
rack->r_ctl.pcm_idle_rounds = 0;
}
if (SEQ_GEQ(start, rack->r_ctl.pcm_i.eseq)) {
goto skip_ack_accounting;
}
ack_arrival = tcp_tv_to_lusec(&rack->r_ctl.act_rcv_time);
if (SEQ_GT(end, rack->r_ctl.pcm_i.eseq)) {
end = rack->r_ctl.pcm_i.eseq;
}
if ((rack->r_ctl.pcm_i.cnt + 1) > rack->r_ctl.pcm_i.cnt_alloc) {
size_t sz;
if (rack->r_ctl.pcm_i.cnt_alloc == 0) {
rack->r_ctl.pcm_i.cnt_alloc = RACK_DEFAULT_PCM_ARRAY;
sz = (sizeof(struct rack_pcm_stats) * rack->r_ctl.pcm_i.cnt_alloc);
rack->r_ctl.pcm_s = malloc(sz, M_TCPPCM, M_NOWAIT);
if (rack->r_ctl.pcm_s == NULL) {
rack->r_ctl.pcm_i.cnt_alloc = 0;
rack->pcm_in_progress = 0;
return;
}
} else {
struct rack_pcm_stats *n;
uint16_t new_cnt;
new_cnt = rack->r_ctl.pcm_i.cnt_alloc * 2;
sz = (sizeof(struct rack_pcm_stats) * new_cnt);
n = malloc(sz,M_TCPPCM, M_NOWAIT);
if (n == NULL) {
rack->pcm_in_progress = 0;
rack->r_ctl.pcm_i.cnt = 0;
return;
}
sz = (sizeof(struct rack_pcm_stats) * rack->r_ctl.pcm_i.cnt_alloc);
memcpy(n, rack->r_ctl.pcm_s, sz);
free(rack->r_ctl.pcm_s, M_TCPPCM);
rack->r_ctl.pcm_s = n;
rack->r_ctl.pcm_i.cnt_alloc = new_cnt;
}
}
e = &rack->r_ctl.pcm_s[rack->r_ctl.pcm_i.cnt];
rack->r_ctl.pcm_i.cnt++;
e->sseq = start;
e->eseq = end;
e->ack_time = ack_arrival;
skip_ack_accounting:
if (completed == 0)
return;
if (tcp_bblogging_on(rack->rc_tp)) {
union tcp_log_stackspecific log;
struct timeval tv;
uint64_t prev_time = 0;
uint64_t tot_byt = 0;
uint32_t tot_lt_12us = 0;
uint32_t tot_gt_2mss = 0;
(void)tcp_get_usecs(&tv);
for (i=0; i<rack->r_ctl.pcm_i.cnt; i++) {
e = &rack->r_ctl.pcm_s[i];
memset(&log, 0, sizeof(log));
log.u_bbr.timeStamp = tcp_tv_to_usec(&tv);
log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
log.u_bbr.flex8 = 1;
log.u_bbr.flex1 = e->sseq;
log.u_bbr.flex2 = e->eseq;
tot_byt += (e->eseq - e->sseq);
if ((i > 0) &&
(e->ack_time > prev_time)) {
log.u_bbr.flex3 = (uint32_t)(e->ack_time - prev_time);
} else {
log.u_bbr.flex3 = 0;
}
if (e->ack_time > rack->r_ctl.pcm_i.send_time) {
log.u_bbr.flex4 = (uint32_t)(e->ack_time - rack->r_ctl.pcm_i.send_time);
} else {
log.u_bbr.flex4 = 0;
}
if ((e->eseq - e->sseq) > (segsiz * 2)) {
tot_gt_2mss++;
}
if ((i > 0) &&
(log.u_bbr.flex3 < 12)) {
tot_lt_12us++;
}
prev_time = e->ack_time;
log.u_bbr.cur_del_rate = rack->r_ctl.pcm_i.send_time;
if ((i > 0) &&
(log.u_bbr.flex3 > 0)) {
log.u_bbr.delRate = (e->eseq - e->sseq);
log.u_bbr.delRate *= HPTS_USEC_IN_SEC;
log.u_bbr.delRate /= (uint64_t)log.u_bbr.flex3;
}
log.u_bbr.rttProp = e->ack_time;
(void)tcp_log_event(rack->rc_tp, NULL, NULL, NULL, TCP_PCM_MEASURE, ERRNO_UNK,
0, &log, false, NULL, NULL, 0, &tv);
}
if (prev_time > rack->r_ctl.pcm_i.send_time) {
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.timeStamp = tcp_tv_to_usec(&tv);
log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
log.u_bbr.flex8 = 2;
log.u_bbr.flex1 = rack->r_ctl.pcm_i.sseq;
log.u_bbr.flex2 = rack->r_ctl.pcm_i.eseq;
log.u_bbr.flex3 = tot_byt;
log.u_bbr.flex4 = tot_lt_12us;
log.u_bbr.flex5 = tot_gt_2mss;
log.u_bbr.flex7 = rack->r_ctl.pcm_i.cnt;
log.u_bbr.cwnd_gain = rack->r_ctl.pcm_i.cnt_alloc;
log.u_bbr.cur_del_rate = rack->r_ctl.pcm_i.send_time;
log.u_bbr.rttProp = prev_time;
log.u_bbr.delRate = tot_byt;
log.u_bbr.delRate *= HPTS_USEC_IN_SEC;
log.u_bbr.delRate /= (prev_time - rack->r_ctl.pcm_i.send_time);
(void)tcp_log_event(rack->rc_tp, NULL, NULL, NULL, TCP_PCM_MEASURE, ERRNO_UNK,
0, &log, false, NULL, NULL, 0, &tv);
}
}
rack->r_ctl.pcm_i.cnt = 0;
}
#endif