1 /*
2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2017-2018, Broadcom Limited. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34 #ifndef NET_DIM_H
35 #define NET_DIM_H
36
37 #include <linux/module.h>
38
39 struct net_dim_cq_moder {
40 u16 usec;
41 u16 pkts;
42 u8 cq_period_mode;
43 };
44
45 struct net_dim_sample {
46 ktime_t time;
47 u32 pkt_ctr;
48 u32 byte_ctr;
49 u16 event_ctr;
50 };
51
52 struct net_dim_stats {
53 int ppms; /* packets per msec */
54 int bpms; /* bytes per msec */
55 int epms; /* events per msec */
56 };
57
58 struct net_dim { /* Adaptive Moderation */
59 u8 state;
60 struct net_dim_stats prev_stats;
61 struct net_dim_sample start_sample;
62 struct work_struct work;
63 u8 profile_ix;
64 u8 mode;
65 u8 tune_state;
66 u8 steps_right;
67 u8 steps_left;
68 u8 tired;
69 };
70
71 enum {
72 NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0,
73 NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1,
74 NET_DIM_CQ_PERIOD_NUM_MODES
75 };
76
77 /* Adaptive moderation logic */
78 enum {
79 NET_DIM_START_MEASURE,
80 NET_DIM_MEASURE_IN_PROGRESS,
81 NET_DIM_APPLY_NEW_PROFILE,
82 };
83
84 enum {
85 NET_DIM_PARKING_ON_TOP,
86 NET_DIM_PARKING_TIRED,
87 NET_DIM_GOING_RIGHT,
88 NET_DIM_GOING_LEFT,
89 };
90
91 enum {
92 NET_DIM_STATS_WORSE,
93 NET_DIM_STATS_SAME,
94 NET_DIM_STATS_BETTER,
95 };
96
97 enum {
98 NET_DIM_STEPPED,
99 NET_DIM_TOO_TIRED,
100 NET_DIM_ON_EDGE,
101 };
102
103 #define NET_DIM_PARAMS_NUM_PROFILES 5
104 /* Adaptive moderation profiles */
105 #define NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256
106 #define NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE 128
107 #define NET_DIM_DEF_PROFILE_CQE 1
108 #define NET_DIM_DEF_PROFILE_EQE 1
109
110 /* All profiles sizes must be NET_PARAMS_DIM_NUM_PROFILES */
111 #define NET_DIM_RX_EQE_PROFILES { \
112 {1, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
113 {8, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
114 {64, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
115 {128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
116 {256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
117 }
118
119 #define NET_DIM_RX_CQE_PROFILES { \
120 {2, 256}, \
121 {8, 128}, \
122 {16, 64}, \
123 {32, 64}, \
124 {64, 64} \
125 }
126
127 #define NET_DIM_TX_EQE_PROFILES { \
128 {1, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
129 {8, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
130 {32, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
131 {64, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
132 {128, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE} \
133 }
134
135 #define NET_DIM_TX_CQE_PROFILES { \
136 {5, 128}, \
137 {8, 64}, \
138 {16, 32}, \
139 {32, 32}, \
140 {64, 32} \
141 }
142
143 static const struct net_dim_cq_moder
144 rx_profile[NET_DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {
145 NET_DIM_RX_EQE_PROFILES,
146 NET_DIM_RX_CQE_PROFILES,
147 };
148
149 static const struct net_dim_cq_moder
150 tx_profile[NET_DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {
151 NET_DIM_TX_EQE_PROFILES,
152 NET_DIM_TX_CQE_PROFILES,
153 };
154
155 static inline struct net_dim_cq_moder
net_dim_get_rx_moderation(u8 cq_period_mode,int ix)156 net_dim_get_rx_moderation(u8 cq_period_mode, int ix)
157 {
158 struct net_dim_cq_moder cq_moder = rx_profile[cq_period_mode][ix];
159
160 cq_moder.cq_period_mode = cq_period_mode;
161 return cq_moder;
162 }
163
164 static inline struct net_dim_cq_moder
net_dim_get_def_rx_moderation(u8 cq_period_mode)165 net_dim_get_def_rx_moderation(u8 cq_period_mode)
166 {
167 u8 profile_ix = cq_period_mode == NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE ?
168 NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE;
169
170 return net_dim_get_rx_moderation(cq_period_mode, profile_ix);
171 }
172
173 static inline struct net_dim_cq_moder
net_dim_get_tx_moderation(u8 cq_period_mode,int ix)174 net_dim_get_tx_moderation(u8 cq_period_mode, int ix)
175 {
176 struct net_dim_cq_moder cq_moder = tx_profile[cq_period_mode][ix];
177
178 cq_moder.cq_period_mode = cq_period_mode;
179 return cq_moder;
180 }
181
182 static inline struct net_dim_cq_moder
net_dim_get_def_tx_moderation(u8 cq_period_mode)183 net_dim_get_def_tx_moderation(u8 cq_period_mode)
184 {
185 u8 profile_ix = cq_period_mode == NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE ?
186 NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE;
187
188 return net_dim_get_tx_moderation(cq_period_mode, profile_ix);
189 }
190
net_dim_on_top(struct net_dim * dim)191 static inline bool net_dim_on_top(struct net_dim *dim)
192 {
193 switch (dim->tune_state) {
194 case NET_DIM_PARKING_ON_TOP:
195 case NET_DIM_PARKING_TIRED:
196 return true;
197 case NET_DIM_GOING_RIGHT:
198 return (dim->steps_left > 1) && (dim->steps_right == 1);
199 default: /* NET_DIM_GOING_LEFT */
200 return (dim->steps_right > 1) && (dim->steps_left == 1);
201 }
202 }
203
net_dim_turn(struct net_dim * dim)204 static inline void net_dim_turn(struct net_dim *dim)
205 {
206 switch (dim->tune_state) {
207 case NET_DIM_PARKING_ON_TOP:
208 case NET_DIM_PARKING_TIRED:
209 break;
210 case NET_DIM_GOING_RIGHT:
211 dim->tune_state = NET_DIM_GOING_LEFT;
212 dim->steps_left = 0;
213 break;
214 case NET_DIM_GOING_LEFT:
215 dim->tune_state = NET_DIM_GOING_RIGHT;
216 dim->steps_right = 0;
217 break;
218 }
219 }
220
net_dim_step(struct net_dim * dim)221 static inline int net_dim_step(struct net_dim *dim)
222 {
223 if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2))
224 return NET_DIM_TOO_TIRED;
225
226 switch (dim->tune_state) {
227 case NET_DIM_PARKING_ON_TOP:
228 case NET_DIM_PARKING_TIRED:
229 break;
230 case NET_DIM_GOING_RIGHT:
231 if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1))
232 return NET_DIM_ON_EDGE;
233 dim->profile_ix++;
234 dim->steps_right++;
235 break;
236 case NET_DIM_GOING_LEFT:
237 if (dim->profile_ix == 0)
238 return NET_DIM_ON_EDGE;
239 dim->profile_ix--;
240 dim->steps_left++;
241 break;
242 }
243
244 dim->tired++;
245 return NET_DIM_STEPPED;
246 }
247
net_dim_park_on_top(struct net_dim * dim)248 static inline void net_dim_park_on_top(struct net_dim *dim)
249 {
250 dim->steps_right = 0;
251 dim->steps_left = 0;
252 dim->tired = 0;
253 dim->tune_state = NET_DIM_PARKING_ON_TOP;
254 }
255
net_dim_park_tired(struct net_dim * dim)256 static inline void net_dim_park_tired(struct net_dim *dim)
257 {
258 dim->steps_right = 0;
259 dim->steps_left = 0;
260 dim->tune_state = NET_DIM_PARKING_TIRED;
261 }
262
net_dim_exit_parking(struct net_dim * dim)263 static inline void net_dim_exit_parking(struct net_dim *dim)
264 {
265 dim->tune_state = dim->profile_ix ? NET_DIM_GOING_LEFT :
266 NET_DIM_GOING_RIGHT;
267 net_dim_step(dim);
268 }
269
270 #define IS_SIGNIFICANT_DIFF(val, ref) \
271 (((100UL * abs((val) - (ref))) / (ref)) > 10) /* more than 10% difference */
272
net_dim_stats_compare(struct net_dim_stats * curr,struct net_dim_stats * prev)273 static inline int net_dim_stats_compare(struct net_dim_stats *curr,
274 struct net_dim_stats *prev)
275 {
276 if (!prev->bpms)
277 return curr->bpms ? NET_DIM_STATS_BETTER :
278 NET_DIM_STATS_SAME;
279
280 if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms))
281 return (curr->bpms > prev->bpms) ? NET_DIM_STATS_BETTER :
282 NET_DIM_STATS_WORSE;
283
284 if (!prev->ppms)
285 return curr->ppms ? NET_DIM_STATS_BETTER :
286 NET_DIM_STATS_SAME;
287
288 if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms))
289 return (curr->ppms > prev->ppms) ? NET_DIM_STATS_BETTER :
290 NET_DIM_STATS_WORSE;
291
292 if (!prev->epms)
293 return NET_DIM_STATS_SAME;
294
295 if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms))
296 return (curr->epms < prev->epms) ? NET_DIM_STATS_BETTER :
297 NET_DIM_STATS_WORSE;
298
299 return NET_DIM_STATS_SAME;
300 }
301
net_dim_decision(struct net_dim_stats * curr_stats,struct net_dim * dim)302 static inline bool net_dim_decision(struct net_dim_stats *curr_stats,
303 struct net_dim *dim)
304 {
305 int prev_state = dim->tune_state;
306 int prev_ix = dim->profile_ix;
307 int stats_res;
308 int step_res;
309
310 switch (dim->tune_state) {
311 case NET_DIM_PARKING_ON_TOP:
312 stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats);
313 if (stats_res != NET_DIM_STATS_SAME)
314 net_dim_exit_parking(dim);
315 break;
316
317 case NET_DIM_PARKING_TIRED:
318 dim->tired--;
319 if (!dim->tired)
320 net_dim_exit_parking(dim);
321 break;
322
323 case NET_DIM_GOING_RIGHT:
324 case NET_DIM_GOING_LEFT:
325 stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats);
326 if (stats_res != NET_DIM_STATS_BETTER)
327 net_dim_turn(dim);
328
329 if (net_dim_on_top(dim)) {
330 net_dim_park_on_top(dim);
331 break;
332 }
333
334 step_res = net_dim_step(dim);
335 switch (step_res) {
336 case NET_DIM_ON_EDGE:
337 net_dim_park_on_top(dim);
338 break;
339 case NET_DIM_TOO_TIRED:
340 net_dim_park_tired(dim);
341 break;
342 }
343
344 break;
345 }
346
347 if ((prev_state != NET_DIM_PARKING_ON_TOP) ||
348 (dim->tune_state != NET_DIM_PARKING_ON_TOP))
349 dim->prev_stats = *curr_stats;
350
351 return dim->profile_ix != prev_ix;
352 }
353
net_dim_sample(u16 event_ctr,u64 packets,u64 bytes,struct net_dim_sample * s)354 static inline void net_dim_sample(u16 event_ctr,
355 u64 packets,
356 u64 bytes,
357 struct net_dim_sample *s)
358 {
359 s->time = ktime_get();
360 s->pkt_ctr = packets;
361 s->byte_ctr = bytes;
362 s->event_ctr = event_ctr;
363 }
364
365 #define NET_DIM_NEVENTS 64
366 #define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1))
367
net_dim_calc_stats(struct net_dim_sample * start,struct net_dim_sample * end,struct net_dim_stats * curr_stats)368 static inline void net_dim_calc_stats(struct net_dim_sample *start,
369 struct net_dim_sample *end,
370 struct net_dim_stats *curr_stats)
371 {
372 /* u32 holds up to 71 minutes, should be enough */
373 u32 delta_us = ktime_us_delta(end->time, start->time);
374 u32 npkts = BIT_GAP(BITS_PER_TYPE(u32), end->pkt_ctr, start->pkt_ctr);
375 u32 nbytes = BIT_GAP(BITS_PER_TYPE(u32), end->byte_ctr,
376 start->byte_ctr);
377
378 if (!delta_us)
379 return;
380
381 curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us);
382 curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us);
383 curr_stats->epms = DIV_ROUND_UP(NET_DIM_NEVENTS * USEC_PER_MSEC,
384 delta_us);
385 }
386
net_dim(struct net_dim * dim,struct net_dim_sample end_sample)387 static inline void net_dim(struct net_dim *dim,
388 struct net_dim_sample end_sample)
389 {
390 struct net_dim_stats curr_stats;
391 u16 nevents;
392
393 switch (dim->state) {
394 case NET_DIM_MEASURE_IN_PROGRESS:
395 nevents = BIT_GAP(BITS_PER_TYPE(u16),
396 end_sample.event_ctr,
397 dim->start_sample.event_ctr);
398 if (nevents < NET_DIM_NEVENTS)
399 break;
400 net_dim_calc_stats(&dim->start_sample, &end_sample,
401 &curr_stats);
402 if (net_dim_decision(&curr_stats, dim)) {
403 dim->state = NET_DIM_APPLY_NEW_PROFILE;
404 schedule_work(&dim->work);
405 break;
406 }
407 /* fall through */
408 case NET_DIM_START_MEASURE:
409 net_dim_sample(end_sample.event_ctr, end_sample.pkt_ctr, end_sample.byte_ctr,
410 &dim->start_sample);
411 dim->state = NET_DIM_MEASURE_IN_PROGRESS;
412 break;
413 case NET_DIM_APPLY_NEW_PROFILE:
414 break;
415 }
416 }
417
418 #endif /* NET_DIM_H */
419