diff --git a/net/sched/sch_taprio.c b/net/sched/sch_taprio.c index 39ac5b97aa3a..901dfd2484e1 100644 --- a/net/sched/sch_taprio.c +++ b/net/sched/sch_taprio.c @@ -83,6 +83,10 @@ struct sched_gate_list { s64 cycle_time; s64 cycle_time_extension; s64 base_time; + /* min(cycle_time, sum of intervals): the software schedule restarts + * the list after the last entry even when cycle_time is not up yet. + */ + s64 period; }; struct taprio_sched { @@ -871,12 +875,13 @@ static struct sk_buff *taprio_dequeue(struct Qdisc *sch) } static bool should_restart_cycle(const struct sched_gate_list *oper, - const struct sched_entry *entry) + const struct sched_entry *entry, + ktime_t end_time) { if (list_is_last(&entry->list, &oper->entries)) return true; - if (ktime_compare(entry->end_time, oper->cycle_end_time) == 0) + if (ktime_compare(end_time, oper->cycle_end_time) == 0) return true; return false; @@ -925,8 +930,9 @@ static enum hrtimer_restart advance_sched(struct hrtimer *timer) int num_tc = netdev_get_num_tc(dev); struct sched_entry *entry, *next; struct Qdisc *sch = q->root; - ktime_t end_time; - int tc; + ktime_t end_time, next_start, now; + int budget, tc; + s64 behind; spin_lock(&q->current_entry_lock); entry = rcu_dereference_protected(q->current_entry, @@ -952,23 +958,49 @@ static enum hrtimer_restart advance_sched(struct hrtimer *timer) goto first_run; } - if (should_restart_cycle(oper, entry)) { - next = list_first_entry(&oper->entries, struct sched_entry, - list); - oper->cycle_end_time = ktime_add_ns(oper->cycle_end_time, - oper->cycle_time); - } else { - next = list_next_entry(entry, list); + now = hrtimer_cb_get_time(timer); + end_time = entry->end_time; + behind = ktime_sub(now, end_time); + + /* Behind, e.g. delayed timer or stepped clock: skip whole periods + * arithmetically and walk at most one more to the entry covering + * now, instead of replaying the backlog one expiry at a time. The + * cap bounds the walk; a leftover is picked up by the next expiry. + */ + if (unlikely(behind >= oper->period)) { + s64 jump = div64_s64(behind, oper->period) * oper->period; + + end_time = ktime_add_ns(end_time, jump); + oper->cycle_end_time = ktime_add_ns(oper->cycle_end_time, jump); } - end_time = ktime_add_ns(entry->end_time, next->interval); - end_time = min_t(ktime_t, end_time, oper->cycle_end_time); + budget = 2 * oper->num_entries; + do { + if (should_restart_cycle(oper, entry, end_time)) { + next = list_first_entry(&oper->entries, + struct sched_entry, list); + oper->cycle_end_time = ktime_add_ns(oper->cycle_end_time, + oper->period); + } else { + next = list_next_entry(entry, list); + } + + next_start = end_time; + end_time = ktime_add_ns(next_start, next->interval); + end_time = min_t(ktime_t, end_time, oper->cycle_end_time); + entry = next; + } while (unlikely(ktime_compare(end_time, now) <= 0) && budget--); + /* next can be the entry already published as q->current_entry (a + * single-entry schedule, or a catch-up of whole periods), so the + * close times and budgets below are rewritten in place while + * taprio_dequeue_from_txq() may be reading them. + */ for (tc = 0; tc < num_tc; tc++) { if (next->gate_duration[tc] == oper->cycle_time) next->gate_close_time[tc] = KTIME_MAX; else - next->gate_close_time[tc] = ktime_add_ns(entry->end_time, + next->gate_close_time[tc] = ktime_add_ns(next_start, next->gate_duration[tc]); } @@ -1130,6 +1162,8 @@ static int parse_taprio_schedule(struct taprio_sched *q, struct nlattr **tb, struct sched_gate_list *new, struct netlink_ext_ack *extack) { + struct sched_entry *entry; + ktime_t cycle = 0; int err = 0; if (tb[TCA_TAPRIO_ATTR_SCHED_SINGLE_ENTRY]) { @@ -1152,13 +1186,10 @@ static int parse_taprio_schedule(struct taprio_sched *q, struct nlattr **tb, if (err < 0) return err; - if (!new->cycle_time) { - struct sched_entry *entry; - ktime_t cycle = 0; - - list_for_each_entry(entry, &new->entries, list) - cycle = ktime_add_ns(cycle, entry->interval); + list_for_each_entry(entry, &new->entries, list) + cycle = ktime_add_ns(cycle, entry->interval); + if (!new->cycle_time) { if (cycle < 0 || cycle > INT_MAX) { NL_SET_ERR_MSG(extack, "'cycle_time' is too big"); return -EINVAL; @@ -1172,6 +1203,7 @@ static int parse_taprio_schedule(struct taprio_sched *q, struct nlattr **tb, return -EINVAL; } + new->period = min(new->cycle_time, cycle); taprio_calculate_gate_durations(q, new); return 0;