bbr
struct tcp_bbr_info bbr;
bbr->pacing_gain = (bbr->lt_use_bw ?
bbr_pacing_gain[bbr->cycle_idx]);
bbr->cwnd_gain = bbr_cwnd_gain;
bbr->pacing_gain = BBR_UNIT;
bbr->cwnd_gain = BBR_UNIT;
WARN_ONCE(1, "BBR bad mode: %u\n", bbr->mode);
struct bbr *bbr = inet_csk_ca(sk);
bbr_set_pacing_rate(sk, bw, bbr->pacing_gain);
bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain);
struct bbr *bbr = inet_csk_ca(sk);
bbr->prior_cwnd = 0;
bbr->rtt_cnt = 0;
bbr->next_rtt_delivered = tp->delivered;
bbr->prev_ca_state = TCP_CA_Open;
bbr->packet_conservation = 0;
bbr->probe_rtt_done_stamp = 0;
bbr->probe_rtt_round_done = 0;
bbr->min_rtt_us = tcp_min_rtt(tp);
bbr->min_rtt_stamp = tcp_jiffies32;
minmax_reset(&bbr->bw, bbr->rtt_cnt, 0); /* init max bw to 0 */
bbr->has_seen_rtt = 0;
bbr->round_start = 0;
bbr->idle_restart = 0;
bbr->full_bw_reached = 0;
bbr->full_bw = 0;
bbr->full_bw_cnt = 0;
bbr->cycle_mstamp = 0;
bbr->cycle_idx = 0;
bbr->ack_epoch_mstamp = tp->tcp_mstamp;
bbr->ack_epoch_acked = 0;
bbr->extra_acked_win_rtts = 0;
bbr->extra_acked_win_idx = 0;
bbr->extra_acked[0] = 0;
bbr->extra_acked[1] = 0;
struct bbr *bbr = inet_csk_ca(sk);
bbr->full_bw = 0; /* spurious slow-down; reset full pipe detection */
bbr->full_bw_cnt = 0;
struct bbr *bbr = inet_csk_ca(sk);
memset(&info->bbr, 0, sizeof(info->bbr));
info->bbr.bbr_bw_lo = (u32)bw;
info->bbr.bbr_bw_hi = (u32)(bw >> 32);
info->bbr.bbr_min_rtt = bbr->min_rtt_us;
info->bbr.bbr_pacing_gain = bbr->pacing_gain;
info->bbr.bbr_cwnd_gain = bbr->cwnd_gain;
return sizeof(info->bbr);
struct bbr *bbr = inet_csk_ca(sk);
bbr->prev_ca_state = TCP_CA_Loss;
bbr->full_bw = 0;
bbr->round_start = 1; /* treat RTO like end of a round */
BUILD_BUG_ON(sizeof(struct bbr) > ICSK_CA_PRIV_SIZE);
const struct bbr *bbr = inet_csk_ca(sk);
return bbr->full_bw_reached;
struct bbr *bbr = inet_csk_ca(sk);
return minmax_get(&bbr->bw);
struct bbr *bbr = inet_csk_ca(sk);
return bbr->lt_use_bw ? bbr->lt_bw : bbr_max_bw(sk);
struct bbr *bbr = inet_csk_ca(sk);
return max(bbr->extra_acked[0], bbr->extra_acked[1]);
struct bbr *bbr = inet_csk_ca(sk);
bbr->has_seen_rtt = 1;
struct bbr *bbr = inet_csk_ca(sk);
if (unlikely(!bbr->has_seen_rtt && tp->srtt_us))
struct bbr *bbr = inet_csk_ca(sk);
if (bbr->prev_ca_state < TCP_CA_Recovery && bbr->mode != BBR_PROBE_RTT)
bbr->prior_cwnd = tcp_snd_cwnd(tp); /* this cwnd is good enough */
bbr->prior_cwnd = max(bbr->prior_cwnd, tcp_snd_cwnd(tp));
struct bbr *bbr = inet_csk_ca(sk);
bbr->idle_restart = 1;
bbr->ack_epoch_mstamp = tp->tcp_mstamp;
bbr->ack_epoch_acked = 0;
if (bbr->mode == BBR_PROBE_BW)
else if (bbr->mode == BBR_PROBE_RTT)
struct bbr *bbr = inet_csk_ca(sk);
if (unlikely(bbr->min_rtt_us == ~0U)) /* no valid RTT samples yet? */
w = (u64)bw * bbr->min_rtt_us;
struct bbr *bbr = inet_csk_ca(sk);
if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == 0)
struct bbr *bbr = inet_csk_ca(sk);
if (bbr->pacing_gain > BBR_UNIT) /* increasing inflight */
struct bbr *bbr = inet_csk_ca(sk);
u8 prev_state = bbr->prev_ca_state, state = inet_csk(sk)->icsk_ca_state;
bbr->packet_conservation = 1;
bbr->next_rtt_delivered = tp->delivered; /* start round now */
cwnd = max(cwnd, bbr->prior_cwnd);
bbr->packet_conservation = 0;
bbr->prev_ca_state = state;
if (bbr->packet_conservation) {
struct bbr *bbr = inet_csk_ca(sk);
if (bbr->mode == BBR_PROBE_RTT) /* drain queue, refresh min_rtt */
struct bbr *bbr = inet_csk_ca(sk);
tcp_stamp_us_delta(tp->delivered_mstamp, bbr->cycle_mstamp) >
bbr->min_rtt_us;
if (bbr->pacing_gain == BBR_UNIT)
if (bbr->pacing_gain > BBR_UNIT)
inflight >= bbr_inflight(sk, bw, bbr->pacing_gain));
struct bbr *bbr = inet_csk_ca(sk);
bbr->cycle_idx = (bbr->cycle_idx + 1) & (CYCLE_LEN - 1);
bbr->cycle_mstamp = tp->delivered_mstamp;
struct bbr *bbr = inet_csk_ca(sk);
if (bbr->mode == BBR_PROBE_BW && bbr_is_next_cycle_phase(sk, rs))
struct bbr *bbr = inet_csk_ca(sk);
bbr->mode = BBR_STARTUP;
struct bbr *bbr = inet_csk_ca(sk);
bbr->mode = BBR_PROBE_BW;
bbr->cycle_idx = CYCLE_LEN - 1 - get_random_u32_below(bbr_cycle_rand);
struct bbr *bbr = inet_csk_ca(sk);
bbr->lt_last_stamp = div_u64(tp->delivered_mstamp, USEC_PER_MSEC);
bbr->lt_last_delivered = tp->delivered;
bbr->lt_last_lost = tp->lost;
bbr->lt_rtt_cnt = 0;
struct bbr *bbr = inet_csk_ca(sk);
bbr->lt_bw = 0;
bbr->lt_use_bw = 0;
bbr->lt_is_sampling = false;
struct bbr *bbr = inet_csk_ca(sk);
if (bbr->lt_bw) { /* do we have bw from a previous interval? */
diff = abs(bw - bbr->lt_bw);
if ((diff * BBR_UNIT <= bbr_lt_bw_ratio * bbr->lt_bw) ||
bbr->lt_bw = (bw + bbr->lt_bw) >> 1; /* avg 2 intvls */
bbr->lt_use_bw = 1;
bbr->pacing_gain = BBR_UNIT; /* try to avoid drops */
bbr->lt_rtt_cnt = 0;
bbr->lt_bw = bw;
struct bbr *bbr = inet_csk_ca(sk);
if (bbr->lt_use_bw) { /* already using long-term rate, lt_bw? */
if (bbr->mode == BBR_PROBE_BW && bbr->round_start &&
++bbr->lt_rtt_cnt >= bbr_lt_bw_max_rtts) {
if (!bbr->lt_is_sampling) {
bbr->lt_is_sampling = true;
if (bbr->round_start)
bbr->lt_rtt_cnt++; /* count round trips in this interval */
if (bbr->lt_rtt_cnt < bbr_lt_intvl_min_rtts)
if (bbr->lt_rtt_cnt > 4 * bbr_lt_intvl_min_rtts) {
lost = tp->lost - bbr->lt_last_lost;
delivered = tp->delivered - bbr->lt_last_delivered;
t = div_u64(tp->delivered_mstamp, USEC_PER_MSEC) - bbr->lt_last_stamp;
struct bbr *bbr = inet_csk_ca(sk);
bbr->round_start = 0;
if (!before(rs->prior_delivered, bbr->next_rtt_delivered)) {
bbr->next_rtt_delivered = tp->delivered;
bbr->rtt_cnt++;
bbr->round_start = 1;
bbr->packet_conservation = 0;
minmax_running_max(&bbr->bw, bbr_bw_rtts, bbr->rtt_cnt, bw);
struct bbr *bbr = inet_csk_ca(sk);
if (bbr->round_start) {
bbr->extra_acked_win_rtts = min(0x1F,
bbr->extra_acked_win_rtts + 1);
if (bbr->extra_acked_win_rtts >= bbr_extra_acked_win_rtts) {
bbr->extra_acked_win_rtts = 0;
bbr->extra_acked_win_idx = bbr->extra_acked_win_idx ?
bbr->extra_acked[bbr->extra_acked_win_idx] = 0;
bbr->ack_epoch_mstamp);
if (bbr->ack_epoch_acked <= expected_acked ||
(bbr->ack_epoch_acked + rs->acked_sacked >=
bbr->ack_epoch_acked = 0;
bbr->ack_epoch_mstamp = tp->delivered_mstamp;
bbr->ack_epoch_acked = min_t(u32, 0xFFFFF,
bbr->ack_epoch_acked + rs->acked_sacked);
extra_acked = bbr->ack_epoch_acked - expected_acked;
if (extra_acked > bbr->extra_acked[bbr->extra_acked_win_idx])
bbr->extra_acked[bbr->extra_acked_win_idx] = extra_acked;
struct bbr *bbr = inet_csk_ca(sk);
if (bbr_full_bw_reached(sk) || !bbr->round_start || rs->is_app_limited)
bw_thresh = (u64)bbr->full_bw * bbr_full_bw_thresh >> BBR_SCALE;
bbr->full_bw = bbr_max_bw(sk);
bbr->full_bw_cnt = 0;
++bbr->full_bw_cnt;
bbr->full_bw_reached = bbr->full_bw_cnt >= bbr_full_bw_cnt;
struct bbr *bbr = inet_csk_ca(sk);
if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) {
bbr->mode = BBR_DRAIN; /* drain queue we created */
if (bbr->mode == BBR_DRAIN &&
struct bbr *bbr = inet_csk_ca(sk);
if (!(bbr->probe_rtt_done_stamp &&
after(tcp_jiffies32, bbr->probe_rtt_done_stamp)))
bbr->min_rtt_stamp = tcp_jiffies32; /* wait a while until PROBE_RTT */
tcp_snd_cwnd_set(tp, max(tcp_snd_cwnd(tp), bbr->prior_cwnd));
struct bbr *bbr = inet_csk_ca(sk);
bbr->min_rtt_stamp + bbr_min_rtt_win_sec * HZ);
(rs->rtt_us < bbr->min_rtt_us ||
bbr->min_rtt_us = rs->rtt_us;
bbr->min_rtt_stamp = tcp_jiffies32;
!bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) {
bbr->mode = BBR_PROBE_RTT; /* dip, drain queue */
bbr->probe_rtt_done_stamp = 0;
if (bbr->mode == BBR_PROBE_RTT) {
if (!bbr->probe_rtt_done_stamp &&
bbr->probe_rtt_done_stamp = tcp_jiffies32 +
bbr->probe_rtt_round_done = 0;
bbr->next_rtt_delivered = tp->delivered;
} else if (bbr->probe_rtt_done_stamp) {
if (bbr->round_start)
bbr->probe_rtt_round_done = 1;
if (bbr->probe_rtt_round_done)
bbr->idle_restart = 0;
struct bbr *bbr = inet_csk_ca(sk);
switch (bbr->mode) {
bbr->pacing_gain = bbr_high_gain;
bbr->cwnd_gain = bbr_high_gain;
bbr->pacing_gain = bbr_drain_gain; /* slow, to drain */
bbr->cwnd_gain = bbr_high_gain; /* keep cwnd */