#include "opt_inet.h"
#include "opt_inet6.h"
#include "opt_tcpdebug.h"
#include "opt_tcp_input.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/sysctl.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/proc.h>
#include <sys/protosw.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/syslog.h>
#include <sys/in_cksum.h>
#include <sys/socketvar2.h>
#include <machine/cpu.h>
#include <machine/stdarg.h>
#include <net/if.h>
#include <net/route.h>
#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#include <netinet/ip_icmp.h>
#include <netinet/in_var.h>
#include <netinet/icmp_var.h>
#include <netinet/in_pcb.h>
#include <netinet/ip_var.h>
#include <netinet/ip6.h>
#include <netinet/icmp6.h>
#include <netinet6/nd6.h>
#include <netinet6/ip6_var.h>
#include <netinet6/in6_pcb.h>
#include <netinet/tcp.h>
#include <netinet/tcp_fsm.h>
#include <netinet/tcp_seq.h>
#include <netinet/tcp_timer.h>
#include <netinet/tcp_timer2.h>
#include <netinet/tcp_var.h>
#include <netinet6/tcp6_var.h>
#include <netinet/tcpip.h>
#ifdef TCPDEBUG
#include <netinet/tcp_debug.h>
u_char tcp_saveipgen[40];
struct tcphdr tcp_savetcp;
#endif
#define TCP_SACK_MAXBURST 4
MALLOC_DEFINE(M_TSEGQ, "tseg_qent", "TCP segment queue entry");
static int log_in_vain = 0;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, log_in_vain, CTLFLAG_RW,
&log_in_vain, 0, "Log all incoming TCP connections");
static int blackhole = 0;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, blackhole, CTLFLAG_RW,
&blackhole, 0, "Do not send RST when dropping refused connections");
int tcp_delack_enabled = 1;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, delayed_ack, CTLFLAG_RW,
&tcp_delack_enabled, 0,
"Delay ACK to try to piggyback it onto a data packet");
#ifdef TCP_DROP_SYNFIN
static int drop_synfin = 0;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, drop_synfin, CTLFLAG_RW,
&drop_synfin, 0, "Drop TCP packets with SYN+FIN set");
#endif
static int tcp_do_limitedtransmit = 1;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, limitedtransmit, CTLFLAG_RW,
&tcp_do_limitedtransmit, 0, "Enable RFC 3042 (Limited Transmit)");
static int tcp_do_early_retransmit = 1;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, earlyretransmit, CTLFLAG_RW,
&tcp_do_early_retransmit, 0, "Early retransmit");
int tcp_aggregate_acks = 1;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, aggregate_acks, CTLFLAG_RW,
&tcp_aggregate_acks, 0, "Aggregate built-up acks into one ack");
static int tcp_do_eifel_detect = 1;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, eifel, CTLFLAG_RW,
&tcp_do_eifel_detect, 0, "Eifel detection algorithm (RFC 3522)");
static int tcp_do_abc = 1;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, abc, CTLFLAG_RW,
&tcp_do_abc, 0,
"TCP Appropriate Byte Counting (RFC 3465)");
static u_int tcp_paws_tolerance = 25;
SYSCTL_UINT(_net_inet_tcp, OID_AUTO, paws_tolerance, CTLFLAG_RW,
&tcp_paws_tolerance, 0, "RFC1323 PAWS tolerance");
int tcp_do_sack = 1;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, sack, CTLFLAG_RW,
&tcp_do_sack, 0, "Enable SACK Algorithms");
int tcp_do_smartsack = 1;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, smartsack, CTLFLAG_RW,
&tcp_do_smartsack, 0, "Enable Smart SACK Algorithms");
int tcp_do_rescuesack = 1;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, rescuesack, CTLFLAG_RW,
&tcp_do_rescuesack, 0, "Rescue retransmission for SACK");
int tcp_aggressive_rescuesack = 0;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, rescuesack_agg, CTLFLAG_RW,
&tcp_aggressive_rescuesack, 0, "Aggressive rescue retransmission for SACK");
static int tcp_force_sackrxt = 1;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, force_sackrxt, CTLFLAG_RW,
&tcp_force_sackrxt, 0, "Allowed forced SACK retransmit burst");
int tcp_do_rfc6675 = 1;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, rfc6675, CTLFLAG_RW,
&tcp_do_rfc6675, 0, "Enable RFC6675");
int tcp_rfc6675_rxt = 0;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, rfc6675_rxt, CTLFLAG_RW,
&tcp_rfc6675_rxt, 0, "Enable RFC6675 retransmit");
SYSCTL_NODE(_net_inet_tcp, OID_AUTO, reass, CTLFLAG_RW, 0,
"TCP Segment Reassembly Queue");
int tcp_reass_maxseg = 0;
SYSCTL_INT(_net_inet_tcp_reass, OID_AUTO, maxsegments, CTLFLAG_RD,
&tcp_reass_maxseg, 0,
"Global maximum number of TCP Segments in Reassembly Queue");
int tcp_reass_qsize = 0;
SYSCTL_INT(_net_inet_tcp_reass, OID_AUTO, cursegments, CTLFLAG_RD,
&tcp_reass_qsize, 0,
"Global number of TCP Segments currently in Reassembly Queue");
static int tcp_reass_overflows = 0;
SYSCTL_INT(_net_inet_tcp_reass, OID_AUTO, overflows, CTLFLAG_RD,
&tcp_reass_overflows, 0,
"Global number of TCP Segment Reassembly Queue Overflows");
int tcp_do_autorcvbuf = 1;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, recvbuf_auto, CTLFLAG_RW,
&tcp_do_autorcvbuf, 0, "Enable automatic receive buffer sizing");
int tcp_autorcvbuf_inc = 16*1024;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, recvbuf_inc, CTLFLAG_RW,
&tcp_autorcvbuf_inc, 0,
"Incrementor step size of automatic receive buffer");
int tcp_autorcvbuf_max = 2*1024*1024;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, recvbuf_max, CTLFLAG_RW,
&tcp_autorcvbuf_max, 0, "Max size of automatic receive buffer");
int tcp_sosend_agglim = 2;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, sosend_agglim, CTLFLAG_RW,
&tcp_sosend_agglim, 0, "TCP sosend mbuf aggregation limit");
int tcp_sosend_async = 1;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, sosend_async, CTLFLAG_RW,
&tcp_sosend_async, 0, "TCP asynchronized pru_send");
int tcp_sosend_jcluster = 1;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, sosend_jcluster, CTLFLAG_RW,
&tcp_sosend_jcluster, 0, "TCP output uses jcluster");
static int tcp_ignore_redun_dsack = 1;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, ignore_redun_dsack, CTLFLAG_RW,
&tcp_ignore_redun_dsack, 0, "Ignore redundant DSACK");
static int tcp_reuseport_ext = 1;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, reuseport_ext, CTLFLAG_RW,
&tcp_reuseport_ext, 0, "SO_REUSEPORT extension");
static void tcp_dooptions(struct tcpopt *, u_char *, int, boolean_t,
tcp_seq);
static void tcp_pulloutofband(struct socket *,
struct tcphdr *, struct mbuf *, int);
static int tcp_reass(struct tcpcb *, struct tcphdr *, int *,
struct mbuf *);
static void tcp_xmit_timer(struct tcpcb *, int, tcp_seq);
static void tcp_newreno_partial_ack(struct tcpcb *, struct tcphdr *, int);
static void tcp_sack_rexmt(struct tcpcb *, boolean_t);
static boolean_t tcp_sack_limitedxmit(struct tcpcb *);
static int tcp_rmx_msl(const struct tcpcb *);
static void tcp_established(struct tcpcb *);
static boolean_t tcp_recv_dupack(struct tcpcb *, tcp_seq, u_int);
#ifdef INET6
#define ND6_HINT(tp) \
do { \
if ((tp) && (tp)->t_inpcb && \
INP_ISIPV6((tp)->t_inpcb) && \
(tp)->t_inpcb->in6p_route.ro_rt) \
nd6_nud_hint((tp)->t_inpcb->in6p_route.ro_rt, NULL, 0); \
} while (0)
#else
#define ND6_HINT(tp)
#endif
#define DELAY_ACK(tp) \
(tcp_delack_enabled && !tcp_callout_pending(tp, tp->tt_delack) && \
!(tp->t_flags & TF_RXWIN0SENT))
#define acceptable_window_update(tp, th, tiwin) \
(SEQ_LT(tp->snd_wl1, th->th_seq) || \
(tp->snd_wl1 == th->th_seq && \
(SEQ_LT(tp->snd_wl2, th->th_ack) || \
(tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))
#define iceildiv(n, d) (((n)+(d)-1) / (d))
#define need_early_retransmit(tp, ownd) \
(tcp_do_early_retransmit && \
(tcp_do_eifel_detect && (tp->t_flags & TF_RCVD_TSTMP)) && \
ownd < ((tp->t_rxtthresh + 1) * tp->t_maxseg) && \
tp->t_dupacks + 1 >= iceildiv(ownd, tp->t_maxseg) && \
(!TCP_DO_SACK(tp) || ownd <= tp->t_maxseg || \
tcp_sack_has_sacked(&tp->scb, ownd - tp->t_maxseg)))
static __inline boolean_t
tcp_paws_canreasslast(const struct tcpcb *tp, const struct tcphdr *th, int tlen)
{
const struct tseg_qent *last, *prev;
last = TAILQ_LAST(&tp->t_segq, tsegqe_head);
if (last == NULL)
return FALSE;
if (last->tqe_th->th_seq + last->tqe_len == th->th_seq) {
if (last->tqe_th->th_flags & TH_FIN) {
return FALSE;
}
return TRUE;
}
if (th->th_seq + tlen != last->tqe_th->th_seq)
return FALSE;
prev = TAILQ_PREV(last, tsegqe_head, tqe_q);
if (prev == NULL) {
return TRUE;
}
if (SEQ_GEQ(th->th_seq, prev->tqe_th->th_seq + prev->tqe_len))
return TRUE;
return FALSE;
}
static __inline void
tcp_ncr_update_rxtthresh(struct tcpcb *tp)
{
int old_rxtthresh = tp->t_rxtthresh;
uint32_t ownd = tp->snd_max - tp->snd_una;
tp->t_rxtthresh = min(tcp_ncr_rxtthresh_max,
max(tcprexmtthresh, ((ownd / tp->t_maxseg) >> 1)));
if (tp->t_rxtthresh != old_rxtthresh) {
tcp_sack_update_lostseq(&tp->scb, tp->snd_una,
tp->t_maxseg, tp->t_rxtthresh);
}
}
static int
tcp_reass(struct tcpcb *tp, struct tcphdr *th, int *tlenp, struct mbuf *m)
{
struct tseg_qent *q;
struct tseg_qent *p = NULL;
struct tseg_qent *te;
struct socket *so = tp->t_inpcb->inp_socket;
int flags;
if (th == NULL)
goto present;
if (th->th_seq != tp->rcv_nxt &&
tcp_reass_qsize + 1 >= tcp_reass_maxseg) {
tcp_reass_overflows++;
tcpstat.tcps_rcvmemdrop++;
m_freem(m);
tp->reportblk.rblk_start = tp->reportblk.rblk_end;
return (0);
}
te = kmalloc(sizeof(struct tseg_qent), M_TSEGQ, M_INTWAIT | M_NULLOK);
if (te == NULL) {
tcpstat.tcps_rcvmemdrop++;
m_freem(m);
tp->reportblk.rblk_start = tp->reportblk.rblk_end;
return (0);
}
atomic_add_int(&tcp_reass_qsize, 1);
if (th->th_flags & TH_FIN)
tp->t_flags |= TF_QUEDFIN;
TAILQ_FOREACH(q, &tp->t_segq, tqe_q) {
if (SEQ_GT(q->tqe_th->th_seq, th->th_seq))
break;
p = q;
}
if (p != NULL) {
tcp_seq_diff_t i;
i = p->tqe_th->th_seq + p->tqe_len - th->th_seq;
if (i > 0) {
tp->sack_flags |=
(TSACK_F_DUPSEG | TSACK_F_ENCLOSESEG);
tp->encloseblk.rblk_start = p->tqe_th->th_seq;
if (i >= *tlenp) {
if (th->th_flags & TH_FIN)
p->tqe_th->th_flags |= TH_FIN;
tp->encloseblk.rblk_end = TCP_SACK_BLKEND(
p->tqe_th->th_seq + p->tqe_len,
p->tqe_th->th_flags);
tcpstat.tcps_rcvduppack++;
tcpstat.tcps_rcvdupbyte += *tlenp;
m_freem(m);
kfree(te, M_TSEGQ);
atomic_add_int(&tcp_reass_qsize, -1);
goto present;
}
m_adj(m, i);
*tlenp -= i;
th->th_seq += i;
tp->encloseblk.rblk_end = TCP_SACK_BLKEND(
th->th_seq + *tlenp, th->th_flags);
tp->reportblk.rblk_end = th->th_seq;
}
}
tcpstat.tcps_rcvoopack++;
tcpstat.tcps_rcvoobyte += *tlenp;
while (q) {
tcp_seq_diff_t i = (th->th_seq + *tlenp) - q->tqe_th->th_seq;
tcp_seq qend = q->tqe_th->th_seq + q->tqe_len;
tcp_seq qend_sack = TCP_SACK_BLKEND(qend, q->tqe_th->th_flags);
struct tseg_qent *nq;
if (i <= 0)
break;
if (!(tp->sack_flags & TSACK_F_DUPSEG)) {
tp->sack_flags |= (TSACK_F_DUPSEG | TSACK_F_ENCLOSESEG);
tp->encloseblk = tp->reportblk;
tp->reportblk.rblk_start = q->tqe_th->th_seq;
}
if ((tp->sack_flags & TSACK_F_ENCLOSESEG) &&
SEQ_GT(qend_sack, tp->encloseblk.rblk_end)) {
tp->encloseblk.rblk_end = qend_sack;
}
if (i < q->tqe_len) {
q->tqe_th->th_seq += i;
q->tqe_len -= i;
m_adj(q->tqe_m, i);
break;
}
if (q->tqe_th->th_flags & TH_FIN)
th->th_flags |= TH_FIN;
nq = TAILQ_NEXT(q, tqe_q);
TAILQ_REMOVE(&tp->t_segq, q, tqe_q);
m_freem(q->tqe_m);
kfree(q, M_TSEGQ);
atomic_add_int(&tcp_reass_qsize, -1);
q = nq;
}
te->tqe_m = m;
te->tqe_th = th;
te->tqe_len = *tlenp;
if (q != NULL && (th->th_seq + *tlenp == q->tqe_th->th_seq)) {
tcp_seq tend_sack;
te->tqe_len += q->tqe_len;
if (q->tqe_th->th_flags & TH_FIN)
te->tqe_th->th_flags |= TH_FIN;
tend_sack = TCP_SACK_BLKEND(te->tqe_th->th_seq + te->tqe_len,
te->tqe_th->th_flags);
m_cat(te->tqe_m, q->tqe_m);
tp->encloseblk.rblk_end = tend_sack;
if (!(tp->sack_flags & TSACK_F_DUPSEG))
tp->reportblk.rblk_end = tend_sack;
TAILQ_REMOVE(&tp->t_segq, q, tqe_q);
kfree(q, M_TSEGQ);
atomic_add_int(&tcp_reass_qsize, -1);
}
if (p == NULL) {
TAILQ_INSERT_HEAD(&tp->t_segq, te, tqe_q);
} else {
if (p->tqe_th->th_seq + p->tqe_len == th->th_seq) {
if (te->tqe_th->th_flags & TH_FIN)
p->tqe_th->th_flags |= TH_FIN;
p->tqe_len += te->tqe_len;
m_cat(p->tqe_m, te->tqe_m);
tp->encloseblk.rblk_start = p->tqe_th->th_seq;
if (!(tp->sack_flags & TSACK_F_DUPSEG))
tp->reportblk.rblk_start = p->tqe_th->th_seq;
kfree(te, M_TSEGQ);
atomic_add_int(&tcp_reass_qsize, -1);
} else {
TAILQ_INSERT_AFTER(&tp->t_segq, p, te, tqe_q);
}
}
present:
if (!TCPS_HAVEESTABLISHED(tp->t_state))
return (0);
q = TAILQ_FIRST(&tp->t_segq);
if (q == NULL || q->tqe_th->th_seq != tp->rcv_nxt)
return (0);
tp->rcv_nxt += q->tqe_len;
if (!(tp->sack_flags & TSACK_F_DUPSEG)) {
tp->reportblk.rblk_start = tp->reportblk.rblk_end;
}
tp->sack_flags &= ~TSACK_F_ENCLOSESEG;
flags = q->tqe_th->th_flags & TH_FIN;
TAILQ_REMOVE(&tp->t_segq, q, tqe_q);
KASSERT(TAILQ_EMPTY(&tp->t_segq) ||
TAILQ_FIRST(&tp->t_segq)->tqe_th->th_seq != tp->rcv_nxt,
("segment not coalesced"));
if (so->so_state & SS_CANTRCVMORE) {
m_freem(q->tqe_m);
} else {
lwkt_gettoken(&so->so_rcv.ssb_token);
ssb_appendstream(&so->so_rcv, q->tqe_m);
lwkt_reltoken(&so->so_rcv.ssb_token);
}
kfree(q, M_TSEGQ);
atomic_add_int(&tcp_reass_qsize, -1);
ND6_HINT(tp);
sorwakeup(so);
return (flags);
}
#ifdef INET6
int
tcp6_input(struct mbuf **mp, int *offp, int proto)
{
struct mbuf *m = *mp;
struct in6_ifaddr *ia6;
IP6_EXTHDR_CHECK(m, *offp, sizeof(struct tcphdr), IPPROTO_DONE);
ia6 = ip6_getdstifaddr(m);
if (ia6 && (ia6->ia6_flags & IN6_IFF_ANYCAST)) {
icmp6_error(m, ICMP6_DST_UNREACH, ICMP6_DST_UNREACH_ADDR,
offsetof(struct ip6_hdr, ip6_dst));
return (IPPROTO_DONE);
}
tcp_input(mp, offp, proto);
return (IPPROTO_DONE);
}
#endif
int
tcp_input(struct mbuf **mp, int *offp, int proto)
{
int off0;
struct tcphdr *th;
struct ip *ip = NULL;
struct ipovly *ipov;
struct inpcb *inp = NULL;
u_char *optp = NULL;
int optlen = 0;
int tlen, off;
int len = 0;
int drop_hdrlen;
struct tcpcb *tp = NULL;
int thflags;
struct socket *so = NULL;
int todrop, acked;
boolean_t ourfinisacked, needoutput = FALSE, delayed_dupack = FALSE;
tcp_seq th_dupack = 0;
u_int to_flags = 0;
u_long tiwin;
int recvwin;
struct tcpopt to;
struct sockaddr_in *next_hop = NULL;
int rstreason;
int cpu;
struct ip6_hdr *ip6 = NULL;
struct mbuf *m;
#ifdef INET6
boolean_t isipv6;
#else
const boolean_t isipv6 = FALSE;
#endif
#ifdef TCPDEBUG
short ostate = 0;
#endif
off0 = *offp;
m = *mp;
*mp = NULL;
tcpstat.tcps_rcvtotal++;
if (m->m_pkthdr.fw_flags & IPFORWARD_MBUF_TAGGED) {
struct m_tag *mtag;
mtag = m_tag_find(m, PACKET_TAG_IPFORWARD, NULL);
KKASSERT(mtag != NULL);
next_hop = m_tag_data(mtag);
}
#ifdef INET6
isipv6 = (mtod(m, struct ip *)->ip_v == 6) ? TRUE : FALSE;
#endif
if (isipv6) {
ip6 = mtod(m, struct ip6_hdr *);
tlen = (sizeof *ip6) + ntohs(ip6->ip6_plen) - off0;
if (in6_cksum(m, IPPROTO_TCP, off0, tlen)) {
tcpstat.tcps_rcvbadsum++;
goto drop;
}
th = (struct tcphdr *)((caddr_t)ip6 + off0);
if (IN6_IS_ADDR_UNSPECIFIED(&ip6->ip6_src)) {
goto drop;
}
} else {
if (off0 > sizeof(struct ip)) {
ip_stripoptions(m);
off0 = sizeof(struct ip);
}
KASSERT(m->m_len >= sizeof(struct tcpiphdr),
("TCP header not in one mbuf: m->m_len %d", m->m_len));
ip = mtod(m, struct ip *);
ipov = (struct ipovly *)ip;
th = (struct tcphdr *)((caddr_t)ip + off0);
tlen = ntohs(ip->ip_len) - off0;
if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) {
if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR)
th->th_sum = m->m_pkthdr.csum_data;
else
th->th_sum = in_pseudo(ip->ip_src.s_addr,
ip->ip_dst.s_addr,
htonl(m->m_pkthdr.csum_data +
ntohs(ip->ip_len) +
IPPROTO_TCP));
th->th_sum ^= 0xffff;
} else {
len = sizeof(struct ip) + tlen;
bzero(ipov->ih_x1, sizeof ipov->ih_x1);
ipov->ih_len = (u_short)tlen;
ipov->ih_len = htons(ipov->ih_len);
th->th_sum = in_cksum(m, len);
}
if (th->th_sum) {
tcpstat.tcps_rcvbadsum++;
goto drop;
}
#ifdef INET6
ip->ip_v = IPVERSION;
#endif
}
off = th->th_off << 2;
KASSERT(off >= sizeof(struct tcphdr) && off <= tlen,
("bad TCP data offset %d (tlen %d)", off, tlen));
tlen -= off;
if (off > sizeof(struct tcphdr)) {
if (isipv6) {
IP6_EXTHDR_CHECK(m, off0, off, IPPROTO_DONE);
ip6 = mtod(m, struct ip6_hdr *);
th = (struct tcphdr *)((caddr_t)ip6 + off0);
} else {
KASSERT(m->m_len >= sizeof(struct ip) + off,
("TCP header and options not in one mbuf: "
"m_len %d, off %d", m->m_len, off));
}
optlen = off - sizeof(struct tcphdr);
optp = (u_char *)(th + 1);
}
thflags = th->th_flags;
#ifdef TCP_DROP_SYNFIN
if (drop_synfin && (thflags & (TH_SYN | TH_FIN)) == (TH_SYN | TH_FIN))
goto drop;
#endif
th->th_seq = ntohl(th->th_seq);
th->th_ack = ntohl(th->th_ack);
th->th_win = ntohs(th->th_win);
th->th_urp = ntohs(th->th_urp);
drop_hdrlen = off0 + off;
findpcb:
if (next_hop != NULL && !isipv6) {
cpu = mycpu->gd_cpuid;
inp = in_pcblookup_hash(&tcbinfo[cpu],
ip->ip_src, th->th_sport,
ip->ip_dst, th->th_dport,
0, m->m_pkthdr.rcvif);
if (!inp) {
in_port_t dport = next_hop->sin_port ?
htons(next_hop->sin_port) :
th->th_dport;
cpu = tcp_addrcpu(ip->ip_src.s_addr, th->th_sport,
next_hop->sin_addr.s_addr, dport);
inp = in_pcblookup_hash(&tcbinfo[cpu],
ip->ip_src, th->th_sport,
next_hop->sin_addr, dport,
1, m->m_pkthdr.rcvif);
}
} else {
if (isipv6) {
inp = in6_pcblookup_hash(&tcbinfo[0],
&ip6->ip6_src, th->th_sport,
&ip6->ip6_dst, th->th_dport,
1, m->m_pkthdr.rcvif);
} else {
cpu = mycpu->gd_cpuid;
inp = in_pcblookup_pkthash(&tcbinfo[cpu],
ip->ip_src, th->th_sport,
ip->ip_dst, th->th_dport,
1, m->m_pkthdr.rcvif,
tcp_reuseport_ext ? m : NULL);
}
}
if (inp == NULL) {
if (log_in_vain) {
#ifdef INET6
char dbuf[INET6_ADDRSTRLEN+2], sbuf[INET6_ADDRSTRLEN+2];
#else
char dbuf[INET_ADDRSTRLEN], sbuf[INET_ADDRSTRLEN];
#endif
if (isipv6) {
strcpy(dbuf, "[");
strcat(dbuf, ip6_sprintf(&ip6->ip6_dst));
strcat(dbuf, "]");
strcpy(sbuf, "[");
strcat(sbuf, ip6_sprintf(&ip6->ip6_src));
strcat(sbuf, "]");
} else {
kinet_ntoa(ip->ip_dst, dbuf);
kinet_ntoa(ip->ip_src, sbuf);
}
switch (log_in_vain) {
case 1:
if (!(thflags & TH_SYN))
break;
case 2:
log(LOG_INFO,
"Connection attempt to TCP %s:%d "
"from %s:%d flags:0x%02x\n",
dbuf, ntohs(th->th_dport), sbuf,
ntohs(th->th_sport), thflags);
break;
default:
break;
}
}
if (blackhole) {
switch (blackhole) {
case 1:
if (thflags & TH_SYN)
goto drop;
break;
case 2:
goto drop;
default:
goto drop;
}
}
rstreason = BANDLIM_RST_CLOSEDPORT;
goto dropwithreset;
}
#ifdef INET6
if ((isipv6 ? ip6->ip6_hlim : ip->ip_ttl) < inp->inp_ip_minttl)
goto drop;
#endif
tp = intotcpcb(inp);
KASSERT(tp != NULL, ("tcp_input: tp is NULL"));
if (tp->t_state <= TCPS_CLOSED)
goto drop;
so = inp->inp_socket;
#ifdef TCPDEBUG
if (so->so_options & SO_DEBUG) {
ostate = tp->t_state;
if (isipv6)
bcopy(ip6, tcp_saveipgen, sizeof(*ip6));
else
bcopy(ip, tcp_saveipgen, sizeof(*ip));
tcp_savetcp = *th;
}
#endif
bzero(&to, sizeof to);
if (so->so_options & SO_ACCEPTCONN) {
struct in_conninfo inc;
#ifdef INET6
inc.inc_isipv6 = (isipv6 == TRUE);
#endif
if (isipv6) {
inc.inc6_faddr = ip6->ip6_src;
inc.inc6_laddr = ip6->ip6_dst;
inc.inc6_route.ro_rt = NULL;
} else {
inc.inc_faddr = ip->ip_src;
inc.inc_laddr = ip->ip_dst;
inc.inc_route.ro_rt = NULL;
}
inc.inc_fport = th->th_sport;
inc.inc_lport = th->th_dport;
if ((thflags & (TH_RST | TH_ACK | TH_SYN)) != TH_SYN) {
if ((thflags & (TH_RST | TH_ACK | TH_SYN)) == TH_ACK) {
if (!syncache_expand(&inc, th, &so, m)) {
tcpstat.tcps_badsyn++;
rstreason = BANDLIM_RST_OPENPORT;
goto dropwithreset;
}
if (so == NULL)
return(IPPROTO_DONE);
KKASSERT(so->so_port == &curthread->td_msgport);
inp = so->so_pcb;
tp = intotcpcb(inp);
tp->snd_up = tp->snd_una;
tp->snd_max = tp->snd_nxt = tp->iss + 1;
tp->last_ack_sent = tp->rcv_nxt;
goto after_listen;
}
if (thflags & TH_RST) {
syncache_chkrst(&inc, th);
goto drop;
}
if (thflags & TH_ACK) {
syncache_badack(&inc);
tcpstat.tcps_badsyn++;
rstreason = BANDLIM_RST_OPENPORT;
goto dropwithreset;
}
goto drop;
}
#ifdef INET6
if (isipv6 && !ip6_use_deprecated) {
struct in6_ifaddr *ia6;
if ((ia6 = ip6_getdstifaddr(m)) &&
(ia6->ia6_flags & IN6_IFF_DEPRECATED)) {
tp = NULL;
rstreason = BANDLIM_RST_OPENPORT;
goto dropwithreset;
}
}
#endif
if (th->th_dport == th->th_sport) {
if (isipv6) {
if (IN6_ARE_ADDR_EQUAL(&ip6->ip6_dst,
&ip6->ip6_src))
goto drop;
} else {
if (ip->ip_dst.s_addr == ip->ip_src.s_addr)
goto drop;
}
}
if (m->m_flags & (M_BCAST | M_MCAST))
goto drop;
if (isipv6) {
if (IN6_IS_ADDR_MULTICAST(&ip6->ip6_dst) ||
IN6_IS_ADDR_MULTICAST(&ip6->ip6_src))
goto drop;
} else {
if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr)) ||
IN_MULTICAST(ntohl(ip->ip_src.s_addr)) ||
ip->ip_src.s_addr == htonl(INADDR_BROADCAST) ||
in_broadcast(ip->ip_dst, m->m_pkthdr.rcvif))
goto drop;
}
if (so->so_qlen <= so->so_qlimit) {
tcp_dooptions(&to, optp, optlen, TRUE, th->th_ack);
if (!syncache_add(&inc, &to, th, so, m))
goto drop;
return(IPPROTO_DONE);
}
goto drop;
}
after_listen:
KASSERT(tp->t_state != TCPS_LISTEN, ("tcp_input: TCPS_LISTEN state"));
KKASSERT(so->so_port == &curthread->td_msgport);
if (!(thflags & TH_SYN))
tiwin = th->th_win << tp->snd_scale;
else
tiwin = th->th_win;
if ((int)(ticks - tp->t_rcvtime) > hz)
tcp_timer_keep_activity(tp, thflags);
tcp_dooptions(&to, optp, optlen, (thflags & TH_SYN) != 0, th->th_ack);
if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) {
if ((to.to_flags & TOF_SCALE) && (tp->t_flags & TF_REQ_SCALE)) {
tp->t_flags |= TF_RCVD_SCALE;
tp->snd_scale = to.to_requested_s_scale;
}
tp->snd_wnd = th->th_win;
if (to.to_flags & TOF_TS) {
tp->t_flags |= TF_RCVD_TSTMP;
tp->ts_recent = to.to_tsval;
tp->ts_recent_age = ticks;
}
if (!(to.to_flags & TOF_MSS))
to.to_mss = 0;
tcp_rmx_init(tp, to.to_mss);
if (tcp_do_sack && (to.to_flags & TOF_SACK_PERMITTED))
tp->t_flags |= TF_SACK_PERMITTED;
}
if (tp->t_state == TCPS_ESTABLISHED &&
(thflags & (TH_SYN|TH_FIN|TH_RST|TH_URG|TH_ACK)) == TH_ACK &&
!(tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN)) &&
(!(to.to_flags & TOF_TS) ||
TSTMP_GEQ(to.to_tsval, tp->ts_recent)) &&
th->th_seq == tp->rcv_nxt &&
tp->snd_nxt == tp->snd_max) {
if ((to.to_flags & TOF_TS) &&
SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {
tp->ts_recent_age = ticks;
tp->ts_recent = to.to_tsval;
}
if (tlen == 0) {
if (SEQ_GT(th->th_ack, tp->snd_una) &&
SEQ_LEQ(th->th_ack, tp->snd_max) &&
tp->snd_cwnd >= tp->snd_wnd &&
!IN_FASTRECOVERY(tp)) {
++tcpstat.tcps_predack;
if (tcp_do_eifel_detect &&
(to.to_flags & TOF_TS) && to.to_tsecr &&
(tp->rxt_flags & TRXT_F_FIRSTACCACK)) {
if (to.to_tsecr < tp->t_rexmtTS) {
tcp_revert_congestion_state(tp);
++tcpstat.tcps_eifeldetected;
if (tp->t_rxtshift != 1 ||
ticks >= tp->t_badrxtwin)
++tcpstat.tcps_rttcantdetect;
}
} else if (tp->t_rxtshift == 1 &&
ticks < tp->t_badrxtwin) {
tcp_revert_congestion_state(tp);
++tcpstat.tcps_rttdetected;
}
tp->rxt_flags &= ~(TRXT_F_FIRSTACCACK |
TRXT_F_FASTREXMT | TRXT_F_EARLYREXMT);
if ((to.to_flags & TOF_TS) && to.to_tsecr) {
tcp_xmit_timer(tp,
ticks - to.to_tsecr + 1,
th->th_ack);
} else if (tp->t_rtttime &&
SEQ_GT(th->th_ack, tp->t_rtseq)) {
tcp_xmit_timer(tp,
ticks - tp->t_rtttime + 1,
th->th_ack);
}
tcp_xmit_bandwidth_limit(tp, th->th_ack);
acked = th->th_ack - tp->snd_una;
tcpstat.tcps_rcvackpack++;
tcpstat.tcps_rcvackbyte += acked;
sbdrop(&so->so_snd.sb, acked);
tp->snd_recover = th->th_ack - 1;
tp->snd_una = th->th_ack;
tp->t_dupacks = 0;
if (tiwin != tp->snd_wnd &&
acceptable_window_update(tp, th, tiwin)) {
if (tp->snd_wl2 == th->th_ack &&
tiwin > tp->snd_wnd)
tcpstat.tcps_rcvwinupd++;
tp->snd_wnd = tiwin;
tp->snd_wl1 = th->th_seq;
tp->snd_wl2 = th->th_ack;
if (tp->snd_wnd > tp->max_sndwnd)
tp->max_sndwnd = tp->snd_wnd;
}
m_freem(m);
ND6_HINT(tp);
if (tp->snd_una == tp->snd_max) {
tcp_callout_stop(tp, tp->tt_rexmt);
} else if (!tcp_callout_active(tp,
tp->tt_persist)) {
tcp_callout_reset(tp, tp->tt_rexmt,
tp->t_rxtcur, tcp_timer_rexmt);
}
sowwakeup(so);
if (so->so_snd.ssb_cc > 0 &&
!tcp_output_pending(tp))
tcp_output_fair(tp);
return(IPPROTO_DONE);
}
} else if (tiwin == tp->snd_wnd &&
th->th_ack == tp->snd_una &&
TAILQ_EMPTY(&tp->t_segq) &&
tlen <= ssb_space(&so->so_rcv)) {
u_long newsize = 0;
++tcpstat.tcps_preddat;
tp->rcv_nxt += tlen;
tcpstat.tcps_rcvpack++;
tcpstat.tcps_rcvbyte += tlen;
ND6_HINT(tp);
if (tcp_do_autorcvbuf &&
to.to_tsecr &&
(so->so_rcv.ssb_flags & SSB_AUTOSIZE)) {
if (to.to_tsecr > tp->rfbuf_ts &&
to.to_tsecr - tp->rfbuf_ts < hz) {
if (tp->rfbuf_cnt >
(so->so_rcv.ssb_hiwat / 8 * 7) &&
so->so_rcv.ssb_hiwat <
tcp_autorcvbuf_max) {
newsize =
ulmin(so->so_rcv.ssb_hiwat +
tcp_autorcvbuf_inc,
tcp_autorcvbuf_max);
}
tp->rfbuf_ts = 0;
tp->rfbuf_cnt = 0;
} else
tp->rfbuf_cnt += tlen;
}
if (so->so_state & SS_CANTRCVMORE) {
m_freem(m);
} else {
lwkt_gettoken(&so->so_rcv.ssb_token);
if (newsize) {
tp->t_flags |= TF_RXRESIZED;
if (!ssb_reserve(&so->so_rcv, newsize,
so, NULL)) {
atomic_clear_int(&so->so_rcv.ssb_flags, SSB_AUTOSIZE);
}
if (newsize >=
(TCP_MAXWIN << tp->rcv_scale)) {
atomic_clear_int(&so->so_rcv.ssb_flags, SSB_AUTOSIZE);
}
}
m_adj(m, drop_hdrlen);
ssb_appendstream(&so->so_rcv, m);
lwkt_reltoken(&so->so_rcv.ssb_token);
}
sorwakeup(so);
if (DELAY_ACK(tp)) {
tcp_callout_reset(tp, tp->tt_delack,
tcp_delacktime, tcp_timer_delack);
} else if (tcp_aggregate_acks) {
tp->t_flags |= TF_ACKNOW;
if (!(tp->t_flags & TF_ONOUTPUTQ)) {
tp->t_flags |= TF_ONOUTPUTQ;
tp->tt_cpu = mycpu->gd_cpuid;
TAILQ_INSERT_TAIL(
&tcpcbackq[tp->tt_cpu].head,
tp, t_outputq);
}
} else {
tp->t_flags |= TF_ACKNOW;
tcp_output(tp);
}
return(IPPROTO_DONE);
}
}
recvwin = ssb_space(&so->so_rcv);
if (recvwin < 0)
recvwin = 0;
tp->rcv_wnd = imax(recvwin, (int)(tp->rcv_adv - tp->rcv_nxt));
tp->rfbuf_ts = 0;
tp->rfbuf_cnt = 0;
switch (tp->t_state) {
case TCPS_SYN_RECEIVED:
if ((thflags & TH_ACK) &&
(SEQ_LEQ(th->th_ack, tp->snd_una) ||
SEQ_GT(th->th_ack, tp->snd_max))) {
rstreason = BANDLIM_RST_OPENPORT;
goto dropwithreset;
}
break;
case TCPS_SYN_SENT:
if ((thflags & TH_ACK) &&
(SEQ_LEQ(th->th_ack, tp->iss) ||
SEQ_GT(th->th_ack, tp->snd_max))) {
rstreason = BANDLIM_UNLIMITED;
goto dropwithreset;
}
if (thflags & TH_RST) {
if (thflags & TH_ACK)
tp = tcp_drop(tp, ECONNREFUSED);
goto drop;
}
if (!(thflags & TH_SYN))
goto drop;
tp->irs = th->th_seq;
tcp_rcvseqinit(tp);
if (thflags & TH_ACK) {
tcpstat.tcps_connects++;
soisconnected(so);
if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
(TF_RCVD_SCALE | TF_REQ_SCALE))
tp->rcv_scale = tp->request_r_scale;
tp->rcv_adv += tp->rcv_wnd;
tp->snd_una++;
tcp_callout_stop(tp, tp->tt_rexmt);
if (DELAY_ACK(tp) && tlen != 0) {
tcp_callout_reset(tp, tp->tt_delack,
tcp_delacktime, tcp_timer_delack);
} else {
tp->t_flags |= TF_ACKNOW;
}
tp->t_starttime = ticks;
if (tp->t_flags & TF_NEEDFIN) {
TCP_STATE_CHANGE(tp, TCPS_FIN_WAIT_1);
tp->t_flags &= ~TF_NEEDFIN;
thflags &= ~TH_SYN;
} else {
tcp_established(tp);
}
} else {
tp->t_flags |= TF_ACKNOW;
tcp_callout_stop(tp, tp->tt_rexmt);
TCP_STATE_CHANGE(tp, TCPS_SYN_RECEIVED);
}
th->th_seq++;
if (tlen > tp->rcv_wnd) {
todrop = tlen - tp->rcv_wnd;
m_adj(m, -todrop);
tlen = tp->rcv_wnd;
thflags &= ~TH_FIN;
tcpstat.tcps_rcvpackafterwin++;
tcpstat.tcps_rcvbyteafterwin += todrop;
}
tp->snd_wl1 = th->th_seq - 1;
tp->rcv_up = th->th_seq;
if (thflags & TH_ACK)
goto process_ACK;
goto step6;
case TCPS_LAST_ACK:
case TCPS_CLOSING:
case TCPS_TIME_WAIT:
break;
}
if (thflags & TH_RST) {
if (SEQ_GEQ(th->th_seq, tp->last_ack_sent) &&
SEQ_LEQ(th->th_seq, tp->last_ack_sent + tp->rcv_wnd)) {
switch (tp->t_state) {
case TCPS_SYN_RECEIVED:
so->so_error = ECONNREFUSED;
goto close;
case TCPS_ESTABLISHED:
case TCPS_FIN_WAIT_1:
case TCPS_FIN_WAIT_2:
case TCPS_CLOSE_WAIT:
so->so_error = ECONNRESET;
close:
TCP_STATE_CHANGE(tp, TCPS_CLOSED);
tcpstat.tcps_drops++;
tp = tcp_close(tp);
break;
case TCPS_CLOSING:
case TCPS_LAST_ACK:
tp = tcp_close(tp);
break;
case TCPS_TIME_WAIT:
break;
}
}
goto drop;
}
if ((thflags & TH_SYN) &&
tp->t_state == TCPS_TIME_WAIT &&
th->th_seq != tp->irs) {
tp = tcp_close(tp);
goto findpcb;
}
if ((to.to_flags & TOF_TS) && tp->ts_recent != 0 &&
TSTMP_LT(to.to_tsval, tp->ts_recent)) {
if ((int)(ticks - tp->ts_recent_age) > TCP_PAWS_IDLE) {
tp->ts_recent = 0;
} else if (tcp_paws_tolerance && tlen != 0 &&
tp->t_state == TCPS_ESTABLISHED &&
(thflags & (TH_SYN|TH_FIN|TH_RST|TH_URG|TH_ACK)) == TH_ACK&&
!(tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN)) &&
th->th_ack == tp->snd_una &&
tiwin == tp->snd_wnd &&
TSTMP_GEQ(to.to_tsval + tcp_paws_tolerance, tp->ts_recent)&&
(th->th_seq == tp->rcv_nxt ||
(SEQ_GT(th->th_seq, tp->rcv_nxt) &&
tcp_paws_canreasslast(tp, th, tlen)))) {
tcpstat.tcps_pawsaccept++;
tcpstat.tcps_pawsdrop++;
} else {
tcpstat.tcps_rcvduppack++;
tcpstat.tcps_rcvdupbyte += tlen;
tcpstat.tcps_pawsdrop++;
if (tlen)
goto dropafterack;
goto drop;
}
}
if (tp->t_state == TCPS_SYN_RECEIVED && SEQ_LT(th->th_seq, tp->irs)) {
rstreason = BANDLIM_RST_OPENPORT;
goto dropwithreset;
}
todrop = tp->rcv_nxt - th->th_seq;
if (todrop > 0) {
if (TCP_DO_SACK(tp)) {
tp->reportblk.rblk_start = th->th_seq;
tp->reportblk.rblk_end = TCP_SACK_BLKEND(
th->th_seq + tlen, thflags);
if (SEQ_GT(tp->reportblk.rblk_end, tp->rcv_nxt))
tp->reportblk.rblk_end = tp->rcv_nxt;
tp->sack_flags |= (TSACK_F_DUPSEG | TSACK_F_SACKLEFT);
tp->t_flags |= TF_ACKNOW;
}
if (thflags & TH_SYN) {
thflags &= ~TH_SYN;
th->th_seq++;
if (th->th_urp > 1)
th->th_urp--;
else
thflags &= ~TH_URG;
todrop--;
}
if (todrop > tlen ||
(todrop == tlen && !(thflags & TH_FIN))) {
thflags &= ~TH_FIN;
tp->t_flags |= TF_ACKNOW;
todrop = tlen;
tcpstat.tcps_rcvduppack++;
tcpstat.tcps_rcvdupbyte += todrop;
} else {
tcpstat.tcps_rcvpartduppack++;
tcpstat.tcps_rcvpartdupbyte += todrop;
}
drop_hdrlen += todrop;
th->th_seq += todrop;
tlen -= todrop;
if (th->th_urp > todrop)
th->th_urp -= todrop;
else {
thflags &= ~TH_URG;
th->th_urp = 0;
}
}
if ((so->so_state & SS_NOFDREF) &&
tp->t_state > TCPS_CLOSE_WAIT && tlen) {
tp = tcp_close(tp);
tcpstat.tcps_rcvafterclose++;
rstreason = BANDLIM_UNLIMITED;
goto dropwithreset;
}
todrop = (th->th_seq + tlen) - (tp->rcv_nxt + tp->rcv_wnd);
if (todrop > 0) {
tcpstat.tcps_rcvpackafterwin++;
if (todrop >= tlen) {
tcpstat.tcps_rcvbyteafterwin += tlen;
if (tp->rcv_wnd == 0 && th->th_seq == tp->rcv_nxt) {
tp->t_flags |= TF_ACKNOW;
tcpstat.tcps_rcvwinprobe++;
} else
goto dropafterack;
} else
tcpstat.tcps_rcvbyteafterwin += todrop;
m_adj(m, -todrop);
tlen -= todrop;
thflags &= ~(TH_PUSH | TH_FIN);
}
if ((to.to_flags & TOF_TS) && SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
SEQ_LEQ(tp->last_ack_sent, (th->th_seq + tlen
+ ((thflags & TH_SYN) != 0)
+ ((thflags & TH_FIN) != 0)))) {
tp->ts_recent_age = ticks;
tp->ts_recent = to.to_tsval;
}
if (thflags & TH_SYN) {
tp = tcp_drop(tp, ECONNRESET);
rstreason = BANDLIM_UNLIMITED;
goto dropwithreset;
}
if (!(thflags & TH_ACK)) {
if (tp->t_state == TCPS_SYN_RECEIVED ||
(tp->t_flags & TF_NEEDSYN))
goto step6;
else
goto drop;
}
switch (tp->t_state) {
case TCPS_SYN_RECEIVED:
tcpstat.tcps_connects++;
soisconnected(so);
if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
(TF_RCVD_SCALE | TF_REQ_SCALE))
tp->rcv_scale = tp->request_r_scale;
tp->t_starttime = ticks;
if (tp->t_flags & TF_NEEDFIN) {
TCP_STATE_CHANGE(tp, TCPS_FIN_WAIT_1);
tp->t_flags &= ~TF_NEEDFIN;
} else {
tcp_established(tp);
}
if (tlen == 0 && !(thflags & TH_FIN))
tcp_reass(tp, NULL, NULL, NULL);
case TCPS_ESTABLISHED:
case TCPS_FIN_WAIT_1:
case TCPS_FIN_WAIT_2:
case TCPS_CLOSE_WAIT:
case TCPS_CLOSING:
case TCPS_LAST_ACK:
case TCPS_TIME_WAIT:
if (SEQ_LEQ(th->th_ack, tp->snd_una)) {
boolean_t maynotdup = FALSE;
if (TCP_DO_SACK(tp))
tcp_sack_update_scoreboard(tp, &to);
if (tlen != 0 || tiwin != tp->snd_wnd ||
((thflags & TH_FIN) && !(tp->t_flags & TF_SAWFIN)))
maynotdup = TRUE;
if (!tcp_callout_active(tp, tp->tt_rexmt) ||
th->th_ack != tp->snd_una) {
if (!maynotdup)
tcpstat.tcps_rcvdupack++;
tp->t_dupacks = 0;
break;
}
#define DELAY_DUPACK \
do { \
delayed_dupack = TRUE; \
th_dupack = th->th_ack; \
to_flags = to.to_flags; \
} while (0)
if (maynotdup) {
if (!tcp_do_rfc6675 ||
!TCP_DO_SACK(tp) ||
(to.to_flags &
(TOF_SACK | TOF_SACK_REDUNDANT))
!= TOF_SACK) {
tp->t_dupacks = 0;
} else {
DELAY_DUPACK;
}
break;
}
if ((thflags & TH_FIN) && !(tp->t_flags & TF_QUEDFIN)) {
DELAY_DUPACK;
break;
}
#undef DELAY_DUPACK
if (tcp_recv_dupack(tp, th->th_ack, to.to_flags))
goto drop;
else
break;
}
KASSERT(SEQ_GT(th->th_ack, tp->snd_una), ("th_ack <= snd_una"));
tp->t_dupacks = 0;
if (SEQ_GT(th->th_ack, tp->snd_max)) {
tp->snd_cwnd = tp->t_maxseg;
tp->snd_wacked = 0;
tcpstat.tcps_rcvacktoomuch++;
goto dropafterack;
}
if (tp->t_flags & TF_NEEDSYN) {
tp->t_flags &= ~TF_NEEDSYN;
tp->snd_una++;
if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
(TF_RCVD_SCALE | TF_REQ_SCALE))
tp->rcv_scale = tp->request_r_scale;
}
process_ACK:
acked = th->th_ack - tp->snd_una;
tcpstat.tcps_rcvackpack++;
tcpstat.tcps_rcvackbyte += acked;
if (tcp_do_eifel_detect && acked > 0 &&
(to.to_flags & TOF_TS) && (to.to_tsecr != 0) &&
(tp->rxt_flags & TRXT_F_FIRSTACCACK)) {
if (to.to_tsecr < tp->t_rexmtTS) {
++tcpstat.tcps_eifeldetected;
tcp_revert_congestion_state(tp);
if (tp->t_rxtshift != 1 ||
ticks >= tp->t_badrxtwin)
++tcpstat.tcps_rttcantdetect;
}
} else if (tp->t_rxtshift == 1 && ticks < tp->t_badrxtwin) {
tcp_revert_congestion_state(tp);
++tcpstat.tcps_rttdetected;
}
if ((to.to_flags & TOF_TS) && (to.to_tsecr != 0))
tcp_xmit_timer(tp, ticks - to.to_tsecr + 1,
th->th_ack);
else if (tp->t_rtttime && SEQ_GT(th->th_ack, tp->t_rtseq))
tcp_xmit_timer(tp, ticks - tp->t_rtttime + 1,
th->th_ack);
tcp_xmit_bandwidth_limit(tp, th->th_ack);
if (acked == 0)
goto step6;
tp->rxt_flags &= ~(TRXT_F_FIRSTACCACK |
TRXT_F_FASTREXMT | TRXT_F_EARLYREXMT);
if (acked > so->so_snd.ssb_cc) {
tp->snd_wnd -= so->so_snd.ssb_cc;
sbdrop(&so->so_snd.sb, (int)so->so_snd.ssb_cc);
ourfinisacked = TRUE;
} else {
sbdrop(&so->so_snd.sb, acked);
tp->snd_wnd -= acked;
ourfinisacked = FALSE;
}
sowwakeup(so);
if (acceptable_window_update(tp, th, tiwin)) {
if (tlen == 0 && tp->snd_wl2 == th->th_ack &&
tiwin > tp->snd_wnd)
tcpstat.tcps_rcvwinupd++;
tp->snd_wnd = tiwin;
tp->snd_wl1 = th->th_seq;
tp->snd_wl2 = th->th_ack;
if (tp->snd_wnd > tp->max_sndwnd)
tp->max_sndwnd = tp->snd_wnd;
needoutput = TRUE;
}
tp->snd_una = th->th_ack;
if (TCP_DO_SACK(tp))
tcp_sack_update_scoreboard(tp, &to);
if (IN_FASTRECOVERY(tp)) {
if (SEQ_GEQ(th->th_ack, tp->snd_recover)) {
EXIT_FASTRECOVERY(tp);
needoutput = TRUE;
if (!TCP_DO_SACK(tp))
tp->snd_cwnd = tp->snd_ssthresh;
if (SEQ_GT(th->th_ack + tp->snd_cwnd,
tp->snd_max + 2 * tp->t_maxseg))
tp->snd_cwnd =
(tp->snd_max - tp->snd_una) +
2 * tp->t_maxseg;
tp->snd_wacked = 0;
} else {
if (TCP_DO_SACK(tp)) {
tp->snd_max_rexmt = tp->snd_max;
tcp_sack_rexmt(tp,
tp->snd_una == tp->rexmt_high);
} else {
tcp_newreno_partial_ack(tp, th, acked);
}
needoutput = FALSE;
}
} else {
if (tp->snd_cwnd <= tp->snd_ssthresh) {
u_int abc_sslimit =
(SEQ_LT(tp->snd_nxt, tp->snd_max) ?
tp->t_maxseg : 2 * tp->t_maxseg);
tp->snd_cwnd += tcp_do_abc ?
min(acked, abc_sslimit) : tp->t_maxseg;
} else {
tp->snd_wacked += tcp_do_abc ? acked :
tp->t_maxseg;
if (tp->snd_wacked >= tp->snd_cwnd) {
tp->snd_wacked -= tp->snd_cwnd;
tp->snd_cwnd += tp->t_maxseg;
}
}
tp->snd_cwnd = min(tp->snd_cwnd,
TCP_MAXWIN << tp->snd_scale);
tp->snd_recover = th->th_ack - 1;
}
if (SEQ_LT(tp->snd_nxt, tp->snd_una))
tp->snd_nxt = tp->snd_una;
if (th->th_ack == tp->snd_max) {
tcp_callout_stop(tp, tp->tt_rexmt);
needoutput = TRUE;
} else if (!tcp_callout_active(tp, tp->tt_persist)) {
tcp_callout_reset(tp, tp->tt_rexmt, tp->t_rxtcur,
tcp_timer_rexmt);
}
switch (tp->t_state) {
case TCPS_FIN_WAIT_1:
if (ourfinisacked) {
if (so->so_state & SS_CANTRCVMORE) {
soisdisconnected(so);
tcp_callout_reset(tp, tp->tt_2msl,
tp->t_maxidle, tcp_timer_2msl);
}
TCP_STATE_CHANGE(tp, TCPS_FIN_WAIT_2);
}
break;
case TCPS_CLOSING:
if (ourfinisacked) {
TCP_STATE_CHANGE(tp, TCPS_TIME_WAIT);
tcp_canceltimers(tp);
tcp_callout_reset(tp, tp->tt_2msl,
2 * tcp_rmx_msl(tp),
tcp_timer_2msl);
soisdisconnected(so);
}
break;
case TCPS_LAST_ACK:
if (ourfinisacked) {
tp = tcp_close(tp);
goto drop;
}
break;
case TCPS_TIME_WAIT:
tcp_callout_reset(tp, tp->tt_2msl, 2 * tcp_rmx_msl(tp),
tcp_timer_2msl);
goto dropafterack;
}
}
step6:
if ((thflags & TH_ACK) &&
acceptable_window_update(tp, th, tiwin)) {
if (tlen == 0 && tp->snd_wl2 == th->th_ack &&
tiwin > tp->snd_wnd)
tcpstat.tcps_rcvwinupd++;
tp->snd_wnd = tiwin;
tp->snd_wl1 = th->th_seq;
tp->snd_wl2 = th->th_ack;
if (tp->snd_wnd > tp->max_sndwnd)
tp->max_sndwnd = tp->snd_wnd;
needoutput = TRUE;
}
if ((thflags & TH_URG) && th->th_urp &&
!TCPS_HAVERCVDFIN(tp->t_state)) {
if (th->th_urp + so->so_rcv.ssb_cc > sb_max) {
th->th_urp = 0;
thflags &= ~TH_URG;
goto dodata;
}
if (SEQ_GT(th->th_seq + th->th_urp, tp->rcv_up)) {
tp->rcv_up = th->th_seq + th->th_urp;
so->so_oobmark = so->so_rcv.ssb_cc +
(tp->rcv_up - tp->rcv_nxt) - 1;
if (so->so_oobmark == 0)
sosetstate(so, SS_RCVATMARK);
sohasoutofband(so);
tp->t_oobflags &= ~(TCPOOB_HAVEDATA | TCPOOB_HADDATA);
}
if (th->th_urp <= (u_long)tlen &&
!(so->so_options & SO_OOBINLINE)) {
tcp_pulloutofband(so, th, m, drop_hdrlen);
}
} else {
if (SEQ_GT(tp->rcv_nxt, tp->rcv_up))
tp->rcv_up = tp->rcv_nxt;
}
dodata:
if ((tlen || (thflags & TH_FIN)) && !TCPS_HAVERCVDFIN(tp->t_state)) {
if (thflags & TH_FIN)
tp->t_flags |= TF_SAWFIN;
m_adj(m, drop_hdrlen);
if (th->th_seq == tp->rcv_nxt &&
TAILQ_EMPTY(&tp->t_segq) &&
TCPS_HAVEESTABLISHED(tp->t_state)) {
if (thflags & TH_FIN)
tp->t_flags |= TF_QUEDFIN;
if (DELAY_ACK(tp)) {
tcp_callout_reset(tp, tp->tt_delack,
tcp_delacktime, tcp_timer_delack);
} else {
tp->t_flags |= TF_ACKNOW;
}
tp->rcv_nxt += tlen;
thflags = th->th_flags & TH_FIN;
tcpstat.tcps_rcvpack++;
tcpstat.tcps_rcvbyte += tlen;
ND6_HINT(tp);
if (so->so_state & SS_CANTRCVMORE) {
m_freem(m);
} else {
lwkt_gettoken(&so->so_rcv.ssb_token);
ssb_appendstream(&so->so_rcv, m);
lwkt_reltoken(&so->so_rcv.ssb_token);
}
sorwakeup(so);
} else {
if (!(tp->sack_flags & TSACK_F_DUPSEG)) {
tp->reportblk.rblk_start = th->th_seq;
tp->reportblk.rblk_end = TCP_SACK_BLKEND(
th->th_seq + tlen, thflags);
}
thflags = tcp_reass(tp, th, &tlen, m);
tp->t_flags |= TF_ACKNOW;
}
len = so->so_rcv.ssb_hiwat - (tp->rcv_adv - tp->rcv_nxt);
} else {
m_freem(m);
thflags &= ~TH_FIN;
}
if (thflags & TH_FIN) {
if (!TCPS_HAVERCVDFIN(tp->t_state)) {
socantrcvmore(so);
if (DELAY_ACK(tp) && (tp->t_flags & TF_NEEDSYN)) {
tcp_callout_reset(tp, tp->tt_delack,
tcp_delacktime, tcp_timer_delack);
} else {
tp->t_flags |= TF_ACKNOW;
}
tp->rcv_nxt++;
}
switch (tp->t_state) {
case TCPS_SYN_RECEIVED:
tp->t_starttime = ticks;
case TCPS_ESTABLISHED:
TCP_STATE_CHANGE(tp, TCPS_CLOSE_WAIT);
break;
case TCPS_FIN_WAIT_1:
TCP_STATE_CHANGE(tp, TCPS_CLOSING);
break;
case TCPS_FIN_WAIT_2:
TCP_STATE_CHANGE(tp, TCPS_TIME_WAIT);
tcp_canceltimers(tp);
tcp_callout_reset(tp, tp->tt_2msl, 2 * tcp_rmx_msl(tp),
tcp_timer_2msl);
soisdisconnected(so);
break;
case TCPS_TIME_WAIT:
tcp_callout_reset(tp, tp->tt_2msl, 2 * tcp_rmx_msl(tp),
tcp_timer_2msl);
break;
}
}
#ifdef TCPDEBUG
if (so->so_options & SO_DEBUG)
tcp_trace(TA_INPUT, ostate, tp, tcp_saveipgen, &tcp_savetcp, 0);
#endif
if (delayed_dupack && tcp_recv_dupack(tp, th_dupack, to_flags))
needoutput = FALSE;
if ((tp->t_flags & TF_ACKNOW) ||
(needoutput && tcp_sack_report_needed(tp))) {
tcp_output_cancel(tp);
tcp_output_fair(tp);
} else if (needoutput && !tcp_output_pending(tp)) {
tcp_output_fair(tp);
}
tcp_sack_report_cleanup(tp);
return(IPPROTO_DONE);
dropafterack:
if (tp->t_state == TCPS_SYN_RECEIVED && (thflags & TH_ACK) &&
(SEQ_GT(tp->snd_una, th->th_ack) ||
SEQ_GT(th->th_ack, tp->snd_max)) ) {
rstreason = BANDLIM_RST_OPENPORT;
goto dropwithreset;
}
#ifdef TCPDEBUG
if (so->so_options & SO_DEBUG)
tcp_trace(TA_DROP, ostate, tp, tcp_saveipgen, &tcp_savetcp, 0);
#endif
m_freem(m);
tp->t_flags |= TF_ACKNOW;
tcp_output(tp);
tcp_sack_report_cleanup(tp);
return(IPPROTO_DONE);
dropwithreset:
if ((thflags & TH_RST) || m->m_flags & (M_BCAST | M_MCAST))
goto drop;
if (isipv6) {
if (IN6_IS_ADDR_MULTICAST(&ip6->ip6_dst) ||
IN6_IS_ADDR_MULTICAST(&ip6->ip6_src))
goto drop;
} else {
if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr)) ||
IN_MULTICAST(ntohl(ip->ip_src.s_addr)) ||
ip->ip_src.s_addr == htonl(INADDR_BROADCAST) ||
in_broadcast(ip->ip_dst, m->m_pkthdr.rcvif))
goto drop;
}
#ifdef ICMP_BANDLIM
if (badport_bandlim(rstreason) < 0)
goto drop;
#endif
#ifdef TCPDEBUG
if (tp == NULL || (tp->t_inpcb->inp_socket->so_options & SO_DEBUG))
tcp_trace(TA_DROP, ostate, tp, tcp_saveipgen, &tcp_savetcp, 0);
#endif
if (thflags & TH_ACK)
tcp_respond(tp, mtod(m, void *), th, m, (tcp_seq)0, th->th_ack,
TH_RST);
else {
if (thflags & TH_SYN)
tlen++;
tcp_respond(tp, mtod(m, void *), th, m, th->th_seq + tlen,
(tcp_seq)0, TH_RST | TH_ACK);
}
if (tp != NULL)
tcp_sack_report_cleanup(tp);
return(IPPROTO_DONE);
drop:
#ifdef TCPDEBUG
if (tp == NULL || (tp->t_inpcb->inp_socket->so_options & SO_DEBUG))
tcp_trace(TA_DROP, ostate, tp, tcp_saveipgen, &tcp_savetcp, 0);
#endif
m_freem(m);
if (tp != NULL)
tcp_sack_report_cleanup(tp);
return(IPPROTO_DONE);
}
static void
tcp_dooptions(struct tcpopt *to, u_char *cp, int cnt, boolean_t is_syn,
tcp_seq ack)
{
int opt, optlen, i;
to->to_flags = 0;
for (; cnt > 0; cnt -= optlen, cp += optlen) {
opt = cp[0];
if (opt == TCPOPT_EOL)
break;
if (opt == TCPOPT_NOP)
optlen = 1;
else {
if (cnt < 2)
break;
optlen = cp[1];
if (optlen < 2 || optlen > cnt)
break;
}
switch (opt) {
case TCPOPT_MAXSEG:
if (optlen != TCPOLEN_MAXSEG)
continue;
if (!is_syn)
continue;
to->to_flags |= TOF_MSS;
bcopy(cp + 2, &to->to_mss, sizeof to->to_mss);
to->to_mss = ntohs(to->to_mss);
break;
case TCPOPT_WINDOW:
if (optlen != TCPOLEN_WINDOW)
continue;
if (!is_syn)
continue;
to->to_flags |= TOF_SCALE;
to->to_requested_s_scale = min(cp[2], TCP_MAX_WINSHIFT);
break;
case TCPOPT_TIMESTAMP:
if (optlen != TCPOLEN_TIMESTAMP)
continue;
to->to_flags |= TOF_TS;
bcopy(cp + 2, &to->to_tsval, sizeof to->to_tsval);
to->to_tsval = ntohl(to->to_tsval);
bcopy(cp + 6, &to->to_tsecr, sizeof to->to_tsecr);
to->to_tsecr = ntohl(to->to_tsecr);
if (to->to_tsecr != 0 && TSTMP_GT(to->to_tsecr, ticks))
to->to_tsecr = 0;
break;
case TCPOPT_SACK_PERMITTED:
if (optlen != TCPOLEN_SACK_PERMITTED)
continue;
if (!is_syn)
continue;
to->to_flags |= TOF_SACK_PERMITTED;
break;
case TCPOPT_SACK:
if ((optlen - 2) & 0x07)
continue;
to->to_nsackblocks = (optlen - 2) / 8;
to->to_sackblocks = (struct raw_sackblock *) (cp + 2);
to->to_flags |= TOF_SACK;
for (i = 0; i < to->to_nsackblocks; i++) {
struct raw_sackblock *r = &to->to_sackblocks[i];
r->rblk_start = ntohl(r->rblk_start);
r->rblk_end = ntohl(r->rblk_end);
if (SEQ_LEQ(r->rblk_end, r->rblk_start)) {
tcpstat.tcps_rcvbadsackopt++;
to->to_nsackblocks = 0;
to->to_sackblocks = NULL;
to->to_flags &= ~TOF_SACK;
break;
}
}
if ((to->to_flags & TOF_SACK) &&
tcp_sack_ndsack_blocks(to->to_sackblocks,
to->to_nsackblocks, ack))
to->to_flags |= TOF_DSACK;
break;
#ifdef TCP_SIGNATURE
case TCPOPT_SIGNATURE:
if (optlen != TCPOLEN_SIGNATURE)
continue;
to->to_flags |= (TOF_SIGNATURE | TOF_SIGLEN);
break;
#endif
default:
continue;
}
}
}
static void
tcp_pulloutofband(struct socket *so, struct tcphdr *th, struct mbuf *m, int off)
{
int cnt = off + th->th_urp - 1;
while (cnt >= 0) {
if (m->m_len > cnt) {
char *cp = mtod(m, caddr_t) + cnt;
struct tcpcb *tp = sototcpcb(so);
tp->t_iobc = *cp;
tp->t_oobflags |= TCPOOB_HAVEDATA;
bcopy(cp + 1, cp, m->m_len - cnt - 1);
m->m_len--;
if (m->m_flags & M_PKTHDR)
m->m_pkthdr.len--;
return;
}
cnt -= m->m_len;
m = m->m_next;
if (m == NULL)
break;
}
panic("tcp_pulloutofband");
}
static void
tcp_xmit_timer(struct tcpcb *tp, int rtt, tcp_seq ack)
{
int rebaserto = 0;
tcpstat.tcps_rttupdated++;
tp->t_rttupdated++;
if ((tp->rxt_flags & TRXT_F_REBASERTO) &&
SEQ_GT(ack, tp->snd_max_prev)) {
#ifdef DEBUG_EIFEL_RESPONSE
kprintf("srtt/rttvar, prev %d/%d, cur %d/%d, ",
tp->t_srtt_prev, tp->t_rttvar_prev,
tp->t_srtt, tp->t_rttvar);
#endif
tcpstat.tcps_eifelresponse++;
rebaserto = 1;
tp->rxt_flags &= ~TRXT_F_REBASERTO;
tp->t_srtt = max(tp->t_srtt_prev, (rtt << TCP_RTT_SHIFT));
tp->t_rttvar = max(tp->t_rttvar_prev,
(rtt << (TCP_RTTVAR_SHIFT - 1)));
if (tp->t_rttbest > tp->t_srtt + tp->t_rttvar)
tp->t_rttbest = tp->t_srtt + tp->t_rttvar;
#ifdef DEBUG_EIFEL_RESPONSE
kprintf("new %d/%d ", tp->t_srtt, tp->t_rttvar);
#endif
} else if (tp->t_srtt != 0) {
int delta;
delta = ((rtt - 1) << TCP_DELTA_SHIFT)
- (tp->t_srtt >> (TCP_RTT_SHIFT - TCP_DELTA_SHIFT));
if ((tp->t_srtt += delta) <= 0)
tp->t_srtt = 1;
if (delta < 0)
delta = -delta;
delta -= tp->t_rttvar >> (TCP_RTTVAR_SHIFT - TCP_DELTA_SHIFT);
if ((tp->t_rttvar += delta) <= 0)
tp->t_rttvar = 1;
if (tp->t_rttbest > tp->t_srtt + tp->t_rttvar)
tp->t_rttbest = tp->t_srtt + tp->t_rttvar;
} else {
tp->t_srtt = rtt << TCP_RTT_SHIFT;
tp->t_rttvar = rtt << (TCP_RTTVAR_SHIFT - 1);
tp->t_rttbest = tp->t_srtt + tp->t_rttvar;
}
tp->t_rtttime = 0;
tp->t_rxtshift = 0;
#ifdef DEBUG_EIFEL_RESPONSE
if (rebaserto) {
kprintf("| rxtcur prev %d, old %d, ",
tp->t_rxtcur_prev, tp->t_rxtcur);
}
#endif
TCPT_RANGESET(tp->t_rxtcur, TCP_REXMTVAL(tp),
max(tp->t_rttmin, rtt + 2), TCPTV_REXMTMAX);
if (rebaserto) {
if (tp->t_rxtcur < tp->t_rxtcur_prev + tcp_eifel_rtoinc) {
tp->t_rxtcur = tp->t_rxtcur_prev + tcp_eifel_rtoinc;
}
#ifdef DEBUG_EIFEL_RESPONSE
kprintf("new %d\n", tp->t_rxtcur);
#endif
}
tp->t_softerror = 0;
}
static void
tcp_rmx_mss(struct tcpcb *tp, struct rtentry *rt, int offer)
{
struct ifnet *ifp;
int mss;
u_long bufsize;
struct inpcb *inp = tp->t_inpcb;
struct socket *so;
#ifdef INET6
boolean_t isipv6 = INP_ISIPV6(inp);
size_t min_protoh = isipv6 ?
sizeof(struct ip6_hdr) + sizeof(struct tcphdr) :
sizeof(struct tcpiphdr);
#else
const boolean_t isipv6 = FALSE;
const size_t min_protoh = sizeof(struct tcpiphdr);
#endif
if (rt == NULL) {
tp->t_maxopd = tp->t_maxseg =
(isipv6 ? tcp_v6mssdflt : tcp_mssdflt);
return;
}
ifp = rt->rt_ifp;
so = inp->inp_socket;
if (offer == 0) {
if (isipv6) {
if (in6_localaddr(&inp->in6p_faddr))
offer = IN6_LINKMTU(rt->rt_ifp) - min_protoh;
else
offer = tcp_v6mssdflt;
} else {
if (in_localaddr(inp->inp_faddr))
offer = ifp->if_mtu - min_protoh;
else
offer = tcp_mssdflt;
}
}
offer = max(offer, tcp_minmss);
offer = max(offer, 64);
rt->rt_rmx.rmx_mssopt = offer;
if (rt->rt_rmx.rmx_mtu) {
mss = rt->rt_rmx.rmx_mtu;
} else {
if (isipv6)
mss = IN6_LINKMTU(rt->rt_ifp);
else
mss = ifp->if_mtu;
}
mss -= min_protoh;
mss = min(mss, offer);
tp->t_maxopd = mss;
if ((tp->t_flags & (TF_REQ_TSTMP | TF_NOOPT)) == TF_REQ_TSTMP &&
((tp->t_flags & TF_RCVD_TSTMP) == TF_RCVD_TSTMP))
mss -= TCPOLEN_TSTAMP_APPA;
#if (MCLBYTES & (MCLBYTES - 1)) == 0
if (mss > MCLBYTES)
mss &= ~(MCLBYTES-1);
#else
if (mss > MCLBYTES)
mss = rounddown(mss, MCLBYTES);
#endif
#ifdef RTV_SPIPE
if ((bufsize = rt->rt_rmx.rmx_sendpipe) == 0)
#endif
bufsize = so->so_snd.ssb_hiwat;
if (bufsize < mss)
mss = bufsize;
else {
bufsize = roundup(bufsize, mss);
if (bufsize > sb_max)
bufsize = sb_max;
if (bufsize > so->so_snd.ssb_hiwat)
ssb_reserve(&so->so_snd, bufsize, so, NULL);
}
tp->t_maxseg = mss;
#ifdef RTV_RPIPE
if ((bufsize = rt->rt_rmx.rmx_recvpipe) == 0)
#endif
bufsize = so->so_rcv.ssb_hiwat;
if (bufsize > mss) {
bufsize = roundup(bufsize, mss);
if (bufsize > sb_max)
bufsize = sb_max;
if (bufsize > so->so_rcv.ssb_hiwat) {
lwkt_gettoken(&so->so_rcv.ssb_token);
ssb_reserve(&so->so_rcv, bufsize, so, NULL);
lwkt_reltoken(&so->so_rcv.ssb_token);
}
}
tp->snd_cwnd = tcp_initial_window(tp);
if (rt->rt_rmx.rmx_ssthresh) {
tp->snd_ssthresh = max(2 * mss, rt->rt_rmx.rmx_ssthresh);
tcpstat.tcps_usedssthresh++;
}
}
static void
tcp_rmx_rtt(struct tcpcb *tp, struct rtentry *rt)
{
int rtt;
if (rt == NULL)
return;
if (tp->t_srtt == 0 && (rtt = rt->rt_rmx.rmx_rtt)) {
if (rt->rt_rmx.rmx_locks & RTV_RTT)
tp->t_rttmin = rtt / (RTM_RTTUNIT / hz);
tp->t_srtt = rtt / (RTM_RTTUNIT / (hz * TCP_RTT_SCALE));
tp->t_rttbest = tp->t_srtt + TCP_RTT_SCALE;
tcpstat.tcps_usedrtt++;
if (rt->rt_rmx.rmx_rttvar) {
tp->t_rttvar = rt->rt_rmx.rmx_rttvar /
(RTM_RTTUNIT / (hz * TCP_RTTVAR_SCALE));
tcpstat.tcps_usedrttvar++;
} else {
tp->t_rttvar =
tp->t_srtt * TCP_RTTVAR_SCALE / TCP_RTT_SCALE;
}
TCPT_RANGESET(tp->t_rxtcur,
((tp->t_srtt >> 2) + tp->t_rttvar) >> 1,
tp->t_rttmin, TCPTV_REXMTMAX);
}
}
void
tcp_rmx_init(struct tcpcb *tp, int offer)
{
struct inpcb *inp = tp->t_inpcb;
#ifdef INET6
boolean_t isipv6 = INP_ISIPV6(inp);
#else
const boolean_t isipv6 = FALSE;
#endif
struct rtentry *rt;
if (isipv6)
rt = tcp_rtlookup6(&inp->inp_inc);
else
rt = tcp_rtlookup(&inp->inp_inc);
tcp_rmx_mss(tp, rt, offer);
tcp_rmx_rtt(tp, rt);
if (rt != NULL && !tcp_ncr_linklocal && (rt->rt_flags & RTF_LLINFO)) {
tp->t_flags &= ~TF_NCR;
}
}
int
tcp_mssopt(struct tcpcb *tp)
{
struct rtentry *rt;
#ifdef INET6
boolean_t isipv6 = INP_ISIPV6(tp->t_inpcb);
int min_protoh = isipv6 ?
sizeof(struct ip6_hdr) + sizeof(struct tcphdr) :
sizeof(struct tcpiphdr);
#else
const boolean_t isipv6 = FALSE;
const size_t min_protoh = sizeof(struct tcpiphdr);
#endif
if (isipv6)
rt = tcp_rtlookup6(&tp->t_inpcb->inp_inc);
else
rt = tcp_rtlookup(&tp->t_inpcb->inp_inc);
if (rt == NULL)
return (isipv6 ? tcp_v6mssdflt : tcp_mssdflt);
#ifdef INET6
return ((isipv6 ? IN6_LINKMTU(rt->rt_ifp) : rt->rt_ifp->if_mtu) -
min_protoh);
#else
return (rt->rt_ifp->if_mtu - min_protoh);
#endif
}
static void
tcp_newreno_partial_ack(struct tcpcb *tp, struct tcphdr *th, int acked)
{
tcp_seq old_snd_nxt = tp->snd_nxt;
u_long ocwnd = tp->snd_cwnd;
tcp_callout_stop(tp, tp->tt_rexmt);
tp->t_rtttime = 0;
tp->snd_nxt = th->th_ack;
tp->snd_cwnd = tp->t_maxseg;
tp->t_flags |= TF_ACKNOW;
tcp_output(tp);
if (SEQ_GT(old_snd_nxt, tp->snd_nxt))
tp->snd_nxt = old_snd_nxt;
if (ocwnd > acked)
tp->snd_cwnd = ocwnd - acked + tp->t_maxseg;
else
tp->snd_cwnd = tp->t_maxseg;
}
static void
tcp_sack_rexmt(struct tcpcb *tp, boolean_t force)
{
tcp_seq old_snd_nxt = tp->snd_nxt;
u_long ocwnd = tp->snd_cwnd;
uint32_t pipe;
int nseg = 0;
int nseg_rexmt = 0;
int maxrexmt = 0;
if (force) {
uint32_t unsacked = tcp_sack_first_unsacked_len(tp);
maxrexmt = howmany(unsacked, tp->t_maxseg);
if (maxrexmt > tcp_force_sackrxt)
maxrexmt = tcp_force_sackrxt;
}
tp->t_rtttime = 0;
pipe = tcp_sack_compute_pipe(tp);
while (((tcp_seq_diff_t)(ocwnd - pipe) >= (tcp_seq_diff_t)tp->t_maxseg
|| (force && nseg_rexmt < maxrexmt && nseg == 0)) &&
(!tcp_do_smartsack || nseg < TCP_SACK_MAXBURST)) {
tcp_seq old_snd_max, old_rexmt_high, nextrexmt;
uint32_t sent, seglen;
boolean_t rescue;
int error;
old_rexmt_high = tp->rexmt_high;
if (!tcp_sack_nextseg(tp, &nextrexmt, &seglen, &rescue)) {
tp->rexmt_high = old_rexmt_high;
break;
}
if (rescue && (nseg_rexmt > 0 || nseg > 0)) {
tp->rexmt_high = old_rexmt_high;
break;
}
if (nextrexmt == tp->snd_max)
++nseg;
else
++nseg_rexmt;
tp->snd_nxt = nextrexmt;
tp->snd_cwnd = nextrexmt - tp->snd_una + seglen;
old_snd_max = tp->snd_max;
if (nextrexmt == tp->snd_una)
tcp_callout_stop(tp, tp->tt_rexmt);
tp->t_flags |= TF_XMITNOW;
error = tcp_output(tp);
if (error != 0) {
tp->rexmt_high = old_rexmt_high;
break;
}
sent = tp->snd_nxt - nextrexmt;
if (sent <= 0) {
tp->rexmt_high = old_rexmt_high;
break;
}
pipe += sent;
tcpstat.tcps_sndsackpack++;
tcpstat.tcps_sndsackbyte += sent;
if (rescue) {
tcpstat.tcps_sackrescue++;
tp->rexmt_rescue = tp->snd_nxt;
tp->sack_flags |= TSACK_F_SACKRESCUED;
break;
}
if (SEQ_LT(nextrexmt, old_snd_max) &&
SEQ_LT(tp->rexmt_high, tp->snd_nxt)) {
tp->rexmt_high = seq_min(tp->snd_nxt, old_snd_max);
if (tcp_aggressive_rescuesack &&
(tp->sack_flags & TSACK_F_SACKRESCUED) &&
SEQ_LT(tp->rexmt_rescue, tp->rexmt_high)) {
tp->rexmt_rescue = tp->rexmt_high;
}
}
}
if (SEQ_GT(old_snd_nxt, tp->snd_nxt))
tp->snd_nxt = old_snd_nxt;
tp->snd_cwnd = ocwnd;
}
static boolean_t
tcp_sack_limitedxmit(struct tcpcb *tp)
{
tcp_seq oldsndnxt = tp->snd_nxt;
tcp_seq oldsndmax = tp->snd_max;
u_long ocwnd = tp->snd_cwnd;
uint32_t pipe, sent;
boolean_t ret = FALSE;
tcp_seq_diff_t cwnd_left;
tcp_seq next;
tp->rexmt_high = tp->snd_una - 1;
pipe = tcp_sack_compute_pipe(tp);
cwnd_left = (tcp_seq_diff_t)(ocwnd - pipe);
if (cwnd_left < (tcp_seq_diff_t)tp->t_maxseg)
return FALSE;
if (tcp_do_smartsack)
cwnd_left = ulmin(cwnd_left, tp->t_maxseg * TCP_SACK_MAXBURST);
next = tp->snd_nxt = tp->snd_max;
tp->snd_cwnd = tp->snd_nxt - tp->snd_una +
rounddown(cwnd_left, tp->t_maxseg);
tp->t_flags |= TF_XMITNOW;
tcp_output(tp);
sent = tp->snd_nxt - next;
if (sent > 0) {
tcpstat.tcps_sndlimited += howmany(sent, tp->t_maxseg);
ret = TRUE;
}
if (SEQ_LT(oldsndnxt, oldsndmax)) {
KASSERT(SEQ_GEQ(oldsndnxt, tp->snd_una),
("snd_una moved in other threads"));
tp->snd_nxt = oldsndnxt;
}
tp->snd_cwnd = ocwnd;
if (ret && TCP_DO_NCR(tp))
tcp_ncr_update_rxtthresh(tp);
return ret;
}
void
tcp_timer_keep_activity(struct tcpcb *tp, int thflags)
{
if (TCPS_HAVEESTABLISHED(tp->t_state)) {
if ((thflags & (TH_SYN | TH_ACK)) == TH_SYN) {
tp->t_flags |= TF_KEEPALIVE;
tcp_callout_reset(tp, tp->tt_keep, hz / 2,
tcp_timer_keep);
} else {
tp->t_rcvtime = ticks;
tp->t_flags &= ~TF_KEEPALIVE;
tcp_callout_reset(tp, tp->tt_keep,
tp->t_keepidle,
tcp_timer_keep);
}
}
}
static int
tcp_rmx_msl(const struct tcpcb *tp)
{
struct rtentry *rt;
struct inpcb *inp = tp->t_inpcb;
int msl;
#ifdef INET6
boolean_t isipv6 = INP_ISIPV6(inp);
#else
const boolean_t isipv6 = FALSE;
#endif
if (isipv6)
rt = tcp_rtlookup6(&inp->inp_inc);
else
rt = tcp_rtlookup(&inp->inp_inc);
if (rt == NULL || rt->rt_rmx.rmx_msl == 0)
return tcp_msl;
msl = (rt->rt_rmx.rmx_msl * hz) / 1000;
if (msl == 0)
msl = 1;
return msl;
}
static void
tcp_established(struct tcpcb *tp)
{
TCP_STATE_CHANGE(tp, TCPS_ESTABLISHED);
tcp_callout_reset(tp, tp->tt_keep, tp->t_keepidle, tcp_timer_keep);
if (tp->t_rxtsyn > 0) {
if (tp->t_rxtcur < TCPTV_RTOBASE3)
tp->t_rxtcur = TCPTV_RTOBASE3;
}
}
static boolean_t
tcp_recv_dupack(struct tcpcb *tp, tcp_seq th_ack, u_int to_flags)
{
boolean_t fast_sack_rexmt = TRUE;
tcpstat.tcps_rcvdupack++;
if (IN_FASTRECOVERY(tp)) {
if (TCP_DO_SACK(tp)) {
boolean_t force = FALSE;
if (tp->snd_una == tp->rexmt_high &&
(to_flags & (TOF_SACK | TOF_SACK_REDUNDANT)) ==
TOF_SACK) {
force = TRUE;
}
tcp_sack_rexmt(tp, force);
} else {
tp->snd_cwnd += tp->t_maxseg;
tcp_output(tp);
}
return TRUE;
} else if (SEQ_LT(th_ack, tp->snd_recover)) {
tp->t_dupacks = 0;
return FALSE;
} else if (tcp_ignore_redun_dsack && TCP_DO_SACK(tp) &&
(to_flags & (TOF_DSACK | TOF_SACK_REDUNDANT)) ==
(TOF_DSACK | TOF_SACK_REDUNDANT)) {
return TRUE;
} else if (tp->t_dupacks == 0 && TCP_DO_NCR(tp)) {
tcp_ncr_update_rxtthresh(tp);
}
if (++tp->t_dupacks == tp->t_rxtthresh) {
tcp_seq old_snd_nxt;
u_int win;
fastretransmit:
if (tcp_do_eifel_detect && (tp->t_flags & TF_RCVD_TSTMP)) {
tcp_save_congestion_state(tp);
tp->rxt_flags |= TRXT_F_FASTREXMT;
}
win = min(tp->snd_wnd, tp->snd_cwnd) / 2 / tp->t_maxseg;
if (win < 2)
win = 2;
tp->snd_ssthresh = win * tp->t_maxseg;
ENTER_FASTRECOVERY(tp);
tp->snd_recover = tp->snd_max;
tcp_callout_stop(tp, tp->tt_rexmt);
tp->t_rtttime = 0;
old_snd_nxt = tp->snd_nxt;
tp->snd_nxt = th_ack;
if (TCP_DO_SACK(tp)) {
uint32_t rxtlen;
rxtlen = tcp_sack_first_unsacked_len(tp);
if (rxtlen > tp->t_maxseg)
rxtlen = tp->t_maxseg;
tp->snd_cwnd = rxtlen;
} else {
tp->snd_cwnd = tp->t_maxseg;
}
tcp_output(tp);
++tcpstat.tcps_sndfastrexmit;
tp->snd_cwnd = tp->snd_ssthresh;
tp->rexmt_high = tp->snd_nxt;
tp->sack_flags &= ~TSACK_F_SACKRESCUED;
if (SEQ_GT(old_snd_nxt, tp->snd_nxt))
tp->snd_nxt = old_snd_nxt;
KASSERT(tp->snd_limited <= 2, ("tp->snd_limited too big"));
if (TCP_DO_SACK(tp)) {
if (fast_sack_rexmt)
tcp_sack_rexmt(tp, FALSE);
} else {
tp->snd_cwnd += tp->t_maxseg *
(tp->t_dupacks - tp->snd_limited);
}
} else if ((tcp_do_rfc6675 && TCP_DO_SACK(tp)) || TCP_DO_NCR(tp)) {
if (tcp_rfc6675_rxt && tcp_do_rfc6675 &&
tcp_sack_islost(&tp->scb, tp->snd_una))
goto fastretransmit;
if (tcp_do_limitedtransmit || TCP_DO_NCR(tp)) {
if (!tcp_sack_limitedxmit(tp)) {
uint32_t ownd = tp->snd_max - tp->snd_una;
if (need_early_retransmit(tp, ownd)) {
++tcpstat.tcps_sndearlyrexmit;
tp->rxt_flags |= TRXT_F_EARLYREXMT;
goto fastretransmit;
} else if (tcp_do_rfc6675 &&
tcp_sack_islost(&tp->scb, tp->snd_una)) {
fast_sack_rexmt = FALSE;
goto fastretransmit;
}
}
}
} else if (tcp_do_limitedtransmit) {
u_long oldcwnd = tp->snd_cwnd;
tcp_seq oldsndmax = tp->snd_max;
tcp_seq oldsndnxt = tp->snd_nxt;
uint32_t ownd = tp->snd_max - tp->snd_una;
u_int sent;
KASSERT(tp->t_dupacks == 1 || tp->t_dupacks == 2,
("dupacks not 1 or 2"));
if (tp->t_dupacks == 1)
tp->snd_limited = 0;
tp->snd_nxt = tp->snd_max;
tp->snd_cwnd = ownd +
(tp->t_dupacks - tp->snd_limited) * tp->t_maxseg;
tp->t_flags |= TF_XMITNOW;
tcp_output(tp);
if (SEQ_LT(oldsndnxt, oldsndmax)) {
KASSERT(SEQ_GEQ(oldsndnxt, tp->snd_una),
("snd_una moved in other threads"));
tp->snd_nxt = oldsndnxt;
}
tp->snd_cwnd = oldcwnd;
sent = tp->snd_max - oldsndmax;
if (sent > tp->t_maxseg) {
KASSERT((tp->t_dupacks == 2 && tp->snd_limited == 0) ||
(sent == tp->t_maxseg + 1 &&
(tp->t_flags & TF_SENTFIN)),
("sent too much"));
KASSERT(sent <= tp->t_maxseg * 2,
("sent too many segments"));
tp->snd_limited = 2;
tcpstat.tcps_sndlimited += 2;
} else if (sent > 0) {
++tp->snd_limited;
++tcpstat.tcps_sndlimited;
} else if (need_early_retransmit(tp, ownd)) {
++tcpstat.tcps_sndearlyrexmit;
tp->rxt_flags |= TRXT_F_EARLYREXMT;
goto fastretransmit;
}
}
return TRUE;
}