1/*
2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15/*
16 * pg_input.c: buffer generator input
17 *
18 * Copyright (c) 2008 Eliot Dresselhaus
19 *
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
27 *
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
30 *
31 *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 */
39
40  /*
41   * To be honest, the packet generator needs an extreme
42   * makeover. Two key assumptions which drove the current implementation
43   * are no longer true. First, buffer managers implement a
44   * post-TX recycle list. Second, that packet generator performance
45   * is first-order important.
46   */
47
48#include <vlib/vlib.h>
49#include <vnet/pg/pg.h>
50#include <vnet/vnet.h>
51#include <vnet/ethernet/ethernet.h>
52#include <vnet/feature/feature.h>
53#include <vnet/ip/ip4_packet.h>
54#include <vnet/ip/ip6_packet.h>
55#include <vnet/udp/udp_packet.h>
56#include <vnet/devices/devices.h>
57
58static int
59validate_buffer_data2 (vlib_buffer_t * b, pg_stream_t * s,
60		       u32 data_offset, u32 n_bytes)
61{
62  u8 *bd, *pd, *pm;
63  u32 i;
64
65  bd = b->data;
66  pd = s->fixed_packet_data + data_offset;
67  pm = s->fixed_packet_data_mask + data_offset;
68
69  if (pd + n_bytes >= vec_end (s->fixed_packet_data))
70    n_bytes = (pd < vec_end (s->fixed_packet_data)
71	       ? vec_end (s->fixed_packet_data) - pd : 0);
72
73  for (i = 0; i < n_bytes; i++)
74    if ((bd[i] & pm[i]) != pd[i])
75      break;
76
77  if (i >= n_bytes)
78    return 1;
79
80  clib_warning ("buffer %U", format_vnet_buffer, b);
81  clib_warning ("differ at index %d", i);
82  clib_warning ("is     %U", format_hex_bytes, bd, n_bytes);
83  clib_warning ("mask   %U", format_hex_bytes, pm, n_bytes);
84  clib_warning ("expect %U", format_hex_bytes, pd, n_bytes);
85  return 0;
86}
87
88static int
89validate_buffer_data (vlib_buffer_t * b, pg_stream_t * s)
90{
91  return validate_buffer_data2 (b, s, 0, s->buffer_bytes);
92}
93
94always_inline void
95set_1 (void *a0,
96       u64 v0, u64 v_min, u64 v_max, u32 n_bits, u32 is_net_byte_order)
97{
98  ASSERT (v0 >= v_min && v0 <= v_max);
99  if (n_bits == BITS (u8))
100    {
101      ((u8 *) a0)[0] = v0;
102    }
103  else if (n_bits == BITS (u16))
104    {
105      if (is_net_byte_order)
106	v0 = clib_host_to_net_u16 (v0);
107      clib_mem_unaligned (a0, u16) = v0;
108    }
109  else if (n_bits == BITS (u32))
110    {
111      if (is_net_byte_order)
112	v0 = clib_host_to_net_u32 (v0);
113      clib_mem_unaligned (a0, u32) = v0;
114    }
115  else if (n_bits == BITS (u64))
116    {
117      if (is_net_byte_order)
118	v0 = clib_host_to_net_u64 (v0);
119      clib_mem_unaligned (a0, u64) = v0;
120    }
121}
122
123always_inline void
124set_2 (void *a0, void *a1,
125       u64 v0, u64 v1,
126       u64 v_min, u64 v_max,
127       u32 n_bits, u32 is_net_byte_order, u32 is_increment)
128{
129  ASSERT (v0 >= v_min && v0 <= v_max);
130  ASSERT (v1 >= v_min && v1 <= (v_max + is_increment));
131  if (n_bits == BITS (u8))
132    {
133      ((u8 *) a0)[0] = v0;
134      ((u8 *) a1)[0] = v1;
135    }
136  else if (n_bits == BITS (u16))
137    {
138      if (is_net_byte_order)
139	{
140	  v0 = clib_host_to_net_u16 (v0);
141	  v1 = clib_host_to_net_u16 (v1);
142	}
143      clib_mem_unaligned (a0, u16) = v0;
144      clib_mem_unaligned (a1, u16) = v1;
145    }
146  else if (n_bits == BITS (u32))
147    {
148      if (is_net_byte_order)
149	{
150	  v0 = clib_host_to_net_u32 (v0);
151	  v1 = clib_host_to_net_u32 (v1);
152	}
153      clib_mem_unaligned (a0, u32) = v0;
154      clib_mem_unaligned (a1, u32) = v1;
155    }
156  else if (n_bits == BITS (u64))
157    {
158      if (is_net_byte_order)
159	{
160	  v0 = clib_host_to_net_u64 (v0);
161	  v1 = clib_host_to_net_u64 (v1);
162	}
163      clib_mem_unaligned (a0, u64) = v0;
164      clib_mem_unaligned (a1, u64) = v1;
165    }
166}
167
168static_always_inline void
169do_set_fixed (pg_main_t * pg,
170	      pg_stream_t * s,
171	      u32 * buffers,
172	      u32 n_buffers,
173	      u32 n_bits,
174	      u32 byte_offset, u32 is_net_byte_order, u64 v_min, u64 v_max)
175{
176  vlib_main_t *vm = vlib_get_main ();
177
178  while (n_buffers >= 4)
179    {
180      vlib_buffer_t *b0, *b1, *b2, *b3;
181      void *a0, *a1;
182
183      b0 = vlib_get_buffer (vm, buffers[0]);
184      b1 = vlib_get_buffer (vm, buffers[1]);
185      b2 = vlib_get_buffer (vm, buffers[2]);
186      b3 = vlib_get_buffer (vm, buffers[3]);
187      buffers += 2;
188      n_buffers -= 2;
189
190      a0 = (void *) b0 + byte_offset;
191      a1 = (void *) b1 + byte_offset;
192      CLIB_PREFETCH ((void *) b2 + byte_offset, sizeof (v_min), WRITE);
193      CLIB_PREFETCH ((void *) b3 + byte_offset, sizeof (v_min), WRITE);
194
195      set_2 (a0, a1, v_min, v_min, v_min, v_max, n_bits, is_net_byte_order,
196	     /* is_increment */ 0);
197
198      ASSERT (validate_buffer_data (b0, s));
199      ASSERT (validate_buffer_data (b1, s));
200    }
201
202  while (n_buffers > 0)
203    {
204      vlib_buffer_t *b0;
205      void *a0;
206
207      b0 = vlib_get_buffer (vm, buffers[0]);
208      buffers += 1;
209      n_buffers -= 1;
210
211      a0 = (void *) b0 + byte_offset;
212
213      set_1 (a0, v_min, v_min, v_max, n_bits, is_net_byte_order);
214
215      ASSERT (validate_buffer_data (b0, s));
216    }
217}
218
219static_always_inline u64
220do_set_increment (pg_main_t * pg,
221		  pg_stream_t * s,
222		  u32 * buffers,
223		  u32 n_buffers,
224		  u32 n_bits,
225		  u32 byte_offset,
226		  u32 is_net_byte_order,
227		  u32 want_sum, u64 * sum_result, u64 v_min, u64 v_max, u64 v)
228{
229  vlib_main_t *vm = vlib_get_main ();
230  u64 sum = 0;
231
232  ASSERT (v >= v_min && v <= v_max);
233
234  while (n_buffers >= 4)
235    {
236      vlib_buffer_t *b0, *b1, *b2, *b3;
237      void *a0, *a1;
238      u64 v_old;
239
240      b0 = vlib_get_buffer (vm, buffers[0]);
241      b1 = vlib_get_buffer (vm, buffers[1]);
242      b2 = vlib_get_buffer (vm, buffers[2]);
243      b3 = vlib_get_buffer (vm, buffers[3]);
244      buffers += 2;
245      n_buffers -= 2;
246
247      a0 = (void *) b0 + byte_offset;
248      a1 = (void *) b1 + byte_offset;
249      CLIB_PREFETCH ((void *) b2 + byte_offset, sizeof (v_min), WRITE);
250      CLIB_PREFETCH ((void *) b3 + byte_offset, sizeof (v_min), WRITE);
251
252      v_old = v;
253      v = v_old + 2;
254      v = v > v_max ? v_min : v;
255      set_2 (a0, a1,
256	     v_old + 0, v_old + 1, v_min, v_max, n_bits, is_net_byte_order,
257	     /* is_increment */ 1);
258
259      if (want_sum)
260	sum += 2 * v_old + 1;
261
262      if (PREDICT_FALSE (v_old + 1 > v_max))
263	{
264	  if (want_sum)
265	    sum -= 2 * v_old + 1;
266
267	  v = v_old;
268	  set_1 (a0, v + 0, v_min, v_max, n_bits, is_net_byte_order);
269	  if (want_sum)
270	    sum += v;
271	  v += 1;
272
273	  v = v > v_max ? v_min : v;
274	  set_1 (a1, v + 0, v_min, v_max, n_bits, is_net_byte_order);
275	  if (want_sum)
276	    sum += v;
277	  v += 1;
278	}
279
280      ASSERT (validate_buffer_data (b0, s));
281      ASSERT (validate_buffer_data (b1, s));
282    }
283
284  while (n_buffers > 0)
285    {
286      vlib_buffer_t *b0;
287      void *a0;
288      u64 v_old;
289
290      b0 = vlib_get_buffer (vm, buffers[0]);
291      buffers += 1;
292      n_buffers -= 1;
293
294      a0 = (void *) b0 + byte_offset;
295
296      v_old = v;
297      if (want_sum)
298	sum += v_old;
299      v += 1;
300      v = v > v_max ? v_min : v;
301
302      ASSERT (v_old >= v_min && v_old <= v_max);
303      set_1 (a0, v_old, v_min, v_max, n_bits, is_net_byte_order);
304
305      ASSERT (validate_buffer_data (b0, s));
306    }
307
308  if (want_sum)
309    *sum_result = sum;
310
311  return v;
312}
313
314static_always_inline void
315do_set_random (pg_main_t * pg,
316	       pg_stream_t * s,
317	       u32 * buffers,
318	       u32 n_buffers,
319	       u32 n_bits,
320	       u32 byte_offset,
321	       u32 is_net_byte_order,
322	       u32 want_sum, u64 * sum_result, u64 v_min, u64 v_max)
323{
324  vlib_main_t *vm = vlib_get_main ();
325  u64 v_diff = v_max - v_min + 1;
326  u64 r_mask = max_pow2 (v_diff) - 1;
327  u64 v0, v1;
328  u64 sum = 0;
329  void *random_data;
330
331  random_data = clib_random_buffer_get_data
332    (&vm->random_buffer, n_buffers * n_bits / BITS (u8));
333
334  v0 = v1 = v_min;
335
336  while (n_buffers >= 4)
337    {
338      vlib_buffer_t *b0, *b1, *b2, *b3;
339      void *a0, *a1;
340      u64 r0 = 0, r1 = 0;	/* warnings be gone */
341
342      b0 = vlib_get_buffer (vm, buffers[0]);
343      b1 = vlib_get_buffer (vm, buffers[1]);
344      b2 = vlib_get_buffer (vm, buffers[2]);
345      b3 = vlib_get_buffer (vm, buffers[3]);
346      buffers += 2;
347      n_buffers -= 2;
348
349      a0 = (void *) b0 + byte_offset;
350      a1 = (void *) b1 + byte_offset;
351      CLIB_PREFETCH ((void *) b2 + byte_offset, sizeof (v_min), WRITE);
352      CLIB_PREFETCH ((void *) b3 + byte_offset, sizeof (v_min), WRITE);
353
354      switch (n_bits)
355	{
356#define _(n)					\
357	  case BITS (u##n):			\
358	    {					\
359	      u##n * r = random_data;		\
360	      r0 = r[0];			\
361	      r1 = r[1];			\
362	      random_data = r + 2;		\
363	    }					\
364	  break;
365
366	  _(8);
367	  _(16);
368	  _(32);
369	  _(64);
370
371#undef _
372	}
373
374      /* Add power of 2 sized random number which may be out of range. */
375      v0 += r0 & r_mask;
376      v1 += r1 & r_mask;
377
378      /* Twice should be enough to reduce to v_min .. v_max range. */
379      v0 = v0 > v_max ? v0 - v_diff : v0;
380      v1 = v1 > v_max ? v1 - v_diff : v1;
381      v0 = v0 > v_max ? v0 - v_diff : v0;
382      v1 = v1 > v_max ? v1 - v_diff : v1;
383
384      if (want_sum)
385	sum += v0 + v1;
386
387      set_2 (a0, a1, v0, v1, v_min, v_max, n_bits, is_net_byte_order,
388	     /* is_increment */ 0);
389
390      ASSERT (validate_buffer_data (b0, s));
391      ASSERT (validate_buffer_data (b1, s));
392    }
393
394  while (n_buffers > 0)
395    {
396      vlib_buffer_t *b0;
397      void *a0;
398      u64 r0 = 0;		/* warnings be gone */
399
400      b0 = vlib_get_buffer (vm, buffers[0]);
401      buffers += 1;
402      n_buffers -= 1;
403
404      a0 = (void *) b0 + byte_offset;
405
406      switch (n_bits)
407	{
408#define _(n)					\
409	  case BITS (u##n):			\
410	    {					\
411	      u##n * r = random_data;		\
412	      r0 = r[0];			\
413	      random_data = r + 1;		\
414	    }					\
415	  break;
416
417	  _(8);
418	  _(16);
419	  _(32);
420	  _(64);
421
422#undef _
423	}
424
425      /* Add power of 2 sized random number which may be out of range. */
426      v0 += r0 & r_mask;
427
428      /* Twice should be enough to reduce to v_min .. v_max range. */
429      v0 = v0 > v_max ? v0 - v_diff : v0;
430      v0 = v0 > v_max ? v0 - v_diff : v0;
431
432      if (want_sum)
433	sum += v0;
434
435      set_1 (a0, v0, v_min, v_max, n_bits, is_net_byte_order);
436
437      ASSERT (validate_buffer_data (b0, s));
438    }
439
440  if (want_sum)
441    *sum_result = sum;
442}
443
444#define _(i,t)							\
445  clib_mem_unaligned (a##i, t) =				\
446    clib_host_to_net_##t ((clib_net_to_host_mem_##t (a##i) &~ mask)	\
447			  | (v##i << shift))
448
449always_inline void
450setbits_1 (void *a0,
451	   u64 v0,
452	   u64 v_min, u64 v_max,
453	   u32 max_bits, u32 n_bits, u64 mask, u32 shift)
454{
455  ASSERT (v0 >= v_min && v0 <= v_max);
456  if (max_bits == BITS (u8))
457    ((u8 *) a0)[0] = (((u8 *) a0)[0] & ~mask) | (v0 << shift);
458
459  else if (max_bits == BITS (u16))
460    {
461      _(0, u16);
462    }
463  else if (max_bits == BITS (u32))
464    {
465      _(0, u32);
466    }
467  else if (max_bits == BITS (u64))
468    {
469      _(0, u64);
470    }
471}
472
473always_inline void
474setbits_2 (void *a0, void *a1,
475	   u64 v0, u64 v1,
476	   u64 v_min, u64 v_max,
477	   u32 max_bits, u32 n_bits, u64 mask, u32 shift, u32 is_increment)
478{
479  ASSERT (v0 >= v_min && v0 <= v_max);
480  ASSERT (v1 >= v_min && v1 <= v_max + is_increment);
481  if (max_bits == BITS (u8))
482    {
483      ((u8 *) a0)[0] = (((u8 *) a0)[0] & ~mask) | (v0 << shift);
484      ((u8 *) a1)[0] = (((u8 *) a1)[0] & ~mask) | (v1 << shift);
485    }
486
487  else if (max_bits == BITS (u16))
488    {
489      _(0, u16);
490      _(1, u16);
491    }
492  else if (max_bits == BITS (u32))
493    {
494      _(0, u32);
495      _(1, u32);
496    }
497  else if (max_bits == BITS (u64))
498    {
499      _(0, u64);
500      _(1, u64);
501    }
502}
503
504#undef _
505
506static_always_inline void
507do_setbits_fixed (pg_main_t * pg,
508		  pg_stream_t * s,
509		  u32 * buffers,
510		  u32 n_buffers,
511		  u32 max_bits,
512		  u32 n_bits,
513		  u32 byte_offset, u64 v_min, u64 v_max, u64 mask, u32 shift)
514{
515  vlib_main_t *vm = vlib_get_main ();
516
517  while (n_buffers >= 4)
518    {
519      vlib_buffer_t *b0, *b1, *b2, *b3;
520      void *a0, *a1;
521
522      b0 = vlib_get_buffer (vm, buffers[0]);
523      b1 = vlib_get_buffer (vm, buffers[1]);
524      b2 = vlib_get_buffer (vm, buffers[2]);
525      b3 = vlib_get_buffer (vm, buffers[3]);
526      buffers += 2;
527      n_buffers -= 2;
528
529      a0 = (void *) b0 + byte_offset;
530      a1 = (void *) b1 + byte_offset;
531      CLIB_PREFETCH ((void *) b2 + byte_offset, sizeof (v_min), WRITE);
532      CLIB_PREFETCH ((void *) b3 + byte_offset, sizeof (v_min), WRITE);
533
534      setbits_2 (a0, a1,
535		 v_min, v_min, v_min, v_max, max_bits, n_bits, mask, shift,
536		 /* is_increment */ 0);
537
538      ASSERT (validate_buffer_data (b0, s));
539      ASSERT (validate_buffer_data (b1, s));
540    }
541
542  while (n_buffers > 0)
543    {
544      vlib_buffer_t *b0;
545      void *a0;
546
547      b0 = vlib_get_buffer (vm, buffers[0]);
548      buffers += 1;
549      n_buffers -= 1;
550
551      a0 = (void *) b0 + byte_offset;
552
553      setbits_1 (a0, v_min, v_min, v_max, max_bits, n_bits, mask, shift);
554      ASSERT (validate_buffer_data (b0, s));
555    }
556}
557
558static_always_inline u64
559do_setbits_increment (pg_main_t * pg,
560		      pg_stream_t * s,
561		      u32 * buffers,
562		      u32 n_buffers,
563		      u32 max_bits,
564		      u32 n_bits,
565		      u32 byte_offset,
566		      u64 v_min, u64 v_max, u64 v, u64 mask, u32 shift)
567{
568  vlib_main_t *vm = vlib_get_main ();
569
570  ASSERT (v >= v_min && v <= v_max);
571
572  while (n_buffers >= 4)
573    {
574      vlib_buffer_t *b0, *b1, *b2, *b3;
575      void *a0, *a1;
576      u64 v_old;
577
578      b0 = vlib_get_buffer (vm, buffers[0]);
579      b1 = vlib_get_buffer (vm, buffers[1]);
580      b2 = vlib_get_buffer (vm, buffers[2]);
581      b3 = vlib_get_buffer (vm, buffers[3]);
582      buffers += 2;
583      n_buffers -= 2;
584
585      a0 = (void *) b0 + byte_offset;
586      a1 = (void *) b1 + byte_offset;
587      CLIB_PREFETCH ((void *) b2 + byte_offset, sizeof (v_min), WRITE);
588      CLIB_PREFETCH ((void *) b3 + byte_offset, sizeof (v_min), WRITE);
589
590      v_old = v;
591      v = v_old + 2;
592      v = v > v_max ? v_min : v;
593      setbits_2 (a0, a1,
594		 v_old + 0, v_old + 1,
595		 v_min, v_max, max_bits, n_bits, mask, shift,
596		 /* is_increment */ 1);
597
598      if (PREDICT_FALSE (v_old + 1 > v_max))
599	{
600	  v = v_old;
601	  setbits_1 (a0, v + 0, v_min, v_max, max_bits, n_bits, mask, shift);
602	  v += 1;
603
604	  v = v > v_max ? v_min : v;
605	  setbits_1 (a1, v + 0, v_min, v_max, max_bits, n_bits, mask, shift);
606	  v += 1;
607	}
608      ASSERT (validate_buffer_data (b0, s));
609      ASSERT (validate_buffer_data (b1, s));
610    }
611
612  while (n_buffers > 0)
613    {
614      vlib_buffer_t *b0;
615      void *a0;
616      u64 v_old;
617
618      b0 = vlib_get_buffer (vm, buffers[0]);
619      buffers += 1;
620      n_buffers -= 1;
621
622      a0 = (void *) b0 + byte_offset;
623
624      v_old = v;
625      v = v_old + 1;
626      v = v > v_max ? v_min : v;
627
628      ASSERT (v_old >= v_min && v_old <= v_max);
629      setbits_1 (a0, v_old, v_min, v_max, max_bits, n_bits, mask, shift);
630
631      ASSERT (validate_buffer_data (b0, s));
632    }
633
634  return v;
635}
636
637static_always_inline void
638do_setbits_random (pg_main_t * pg,
639		   pg_stream_t * s,
640		   u32 * buffers,
641		   u32 n_buffers,
642		   u32 max_bits,
643		   u32 n_bits,
644		   u32 byte_offset, u64 v_min, u64 v_max, u64 mask, u32 shift)
645{
646  vlib_main_t *vm = vlib_get_main ();
647  u64 v_diff = v_max - v_min + 1;
648  u64 r_mask = max_pow2 (v_diff) - 1;
649  u64 v0, v1;
650  void *random_data;
651
652  random_data = clib_random_buffer_get_data
653    (&vm->random_buffer, n_buffers * max_bits / BITS (u8));
654  v0 = v1 = v_min;
655
656  while (n_buffers >= 4)
657    {
658      vlib_buffer_t *b0, *b1, *b2, *b3;
659      void *a0, *a1;
660      u64 r0 = 0, r1 = 0;	/* warnings be gone */
661
662      b0 = vlib_get_buffer (vm, buffers[0]);
663      b1 = vlib_get_buffer (vm, buffers[1]);
664      b2 = vlib_get_buffer (vm, buffers[2]);
665      b3 = vlib_get_buffer (vm, buffers[3]);
666      buffers += 2;
667      n_buffers -= 2;
668
669      a0 = (void *) b0 + byte_offset;
670      a1 = (void *) b1 + byte_offset;
671      CLIB_PREFETCH ((void *) b2 + byte_offset, sizeof (v_min), WRITE);
672      CLIB_PREFETCH ((void *) b3 + byte_offset, sizeof (v_min), WRITE);
673
674      switch (max_bits)
675	{
676#define _(n)					\
677	  case BITS (u##n):			\
678	    {					\
679	      u##n * r = random_data;		\
680	      r0 = r[0];			\
681	      r1 = r[1];			\
682	      random_data = r + 2;		\
683	    }					\
684	  break;
685
686	  _(8);
687	  _(16);
688	  _(32);
689	  _(64);
690
691#undef _
692	}
693
694      /* Add power of 2 sized random number which may be out of range. */
695      v0 += r0 & r_mask;
696      v1 += r1 & r_mask;
697
698      /* Twice should be enough to reduce to v_min .. v_max range. */
699      v0 = v0 > v_max ? v0 - v_diff : v0;
700      v1 = v1 > v_max ? v1 - v_diff : v1;
701      v0 = v0 > v_max ? v0 - v_diff : v0;
702      v1 = v1 > v_max ? v1 - v_diff : v1;
703
704      setbits_2 (a0, a1, v0, v1, v_min, v_max, max_bits, n_bits, mask, shift,
705		 /* is_increment */ 0);
706
707      ASSERT (validate_buffer_data (b0, s));
708      ASSERT (validate_buffer_data (b1, s));
709    }
710
711  while (n_buffers > 0)
712    {
713      vlib_buffer_t *b0;
714      void *a0;
715      u64 r0 = 0;		/* warnings be gone */
716
717      b0 = vlib_get_buffer (vm, buffers[0]);
718      buffers += 1;
719      n_buffers -= 1;
720
721      a0 = (void *) b0 + byte_offset;
722
723      switch (max_bits)
724	{
725#define _(n)					\
726	  case BITS (u##n):			\
727	    {					\
728	      u##n * r = random_data;		\
729	      r0 = r[0];			\
730	      random_data = r + 1;		\
731	    }					\
732	  break;
733
734	  _(8);
735	  _(16);
736	  _(32);
737	  _(64);
738
739#undef _
740	}
741
742      /* Add power of 2 sized random number which may be out of range. */
743      v0 += r0 & r_mask;
744
745      /* Twice should be enough to reduce to v_min .. v_max range. */
746      v0 = v0 > v_max ? v0 - v_diff : v0;
747      v0 = v0 > v_max ? v0 - v_diff : v0;
748
749      setbits_1 (a0, v0, v_min, v_max, max_bits, n_bits, mask, shift);
750
751      ASSERT (validate_buffer_data (b0, s));
752    }
753}
754
755static u64
756do_it (pg_main_t * pg,
757       pg_stream_t * s,
758       u32 * buffers,
759       u32 n_buffers,
760       u32 lo_bit, u32 hi_bit,
761       u64 v_min, u64 v_max, u64 v, pg_edit_type_t edit_type)
762{
763  u32 max_bits, l0, l1, h1, start_bit;
764
765  if (v_min == v_max)
766    edit_type = PG_EDIT_FIXED;
767
768  l0 = lo_bit / BITS (u8);
769  l1 = lo_bit % BITS (u8);
770  h1 = hi_bit % BITS (u8);
771
772  start_bit = l0 * BITS (u8);
773
774  max_bits = hi_bit - start_bit;
775  ASSERT (max_bits <= 64);
776
777#define _(n)						\
778  case (n):						\
779    if (edit_type == PG_EDIT_INCREMENT)			\
780      v = do_set_increment (pg, s, buffers, n_buffers,	\
781			    BITS (u##n),		\
782			    l0,				\
783			    /* is_net_byte_order */ 1,	\
784			    /* want sum */ 0, 0,	\
785			    v_min, v_max,		\
786			    v);				\
787    else if (edit_type == PG_EDIT_RANDOM)		\
788      do_set_random (pg, s, buffers, n_buffers,		\
789		     BITS (u##n),			\
790		     l0,				\
791		     /* is_net_byte_order */ 1,		\
792		     /* want sum */ 0, 0,		\
793		     v_min, v_max);			\
794    else /* edit_type == PG_EDIT_FIXED */		\
795      do_set_fixed (pg, s, buffers, n_buffers,		\
796		    BITS (u##n),			\
797		    l0,					\
798		    /* is_net_byte_order */ 1,		\
799		    v_min, v_max);			\
800  goto done;
801
802  if (l1 == 0 && h1 == 0)
803    {
804      switch (max_bits)
805	{
806	  _(8);
807	  _(16);
808	  _(32);
809	  _(64);
810	}
811    }
812
813#undef _
814
815  {
816    u64 mask;
817    u32 shift = l1;
818    u32 n_bits = max_bits;
819
820    max_bits = clib_max (max_pow2 (n_bits), 8);
821
822    mask = ((u64) 1 << (u64) n_bits) - 1;
823    mask &= ~(((u64) 1 << (u64) shift) - 1);
824
825    mask <<= max_bits - n_bits;
826    shift += max_bits - n_bits;
827
828    switch (max_bits)
829      {
830#define _(n)								\
831	case (n):							\
832	  if (edit_type == PG_EDIT_INCREMENT)				\
833	    v = do_setbits_increment (pg, s, buffers, n_buffers,	\
834				      BITS (u##n), n_bits,		\
835				      l0, v_min, v_max, v,		\
836				      mask, shift);			\
837	  else if (edit_type == PG_EDIT_RANDOM)				\
838	    do_setbits_random (pg, s, buffers, n_buffers,		\
839			       BITS (u##n), n_bits,			\
840			       l0, v_min, v_max,			\
841			       mask, shift);				\
842	  else /* edit_type == PG_EDIT_FIXED */				\
843	    do_setbits_fixed (pg, s, buffers, n_buffers,		\
844			      BITS (u##n), n_bits,			\
845			      l0, v_min, v_max,				\
846			      mask, shift);				\
847	goto done;
848
849	_(8);
850	_(16);
851	_(32);
852	_(64);
853
854#undef _
855      }
856  }
857
858done:
859  return v;
860}
861
862static void
863pg_generate_set_lengths (pg_main_t * pg,
864			 pg_stream_t * s, u32 * buffers, u32 n_buffers)
865{
866  u64 v_min, v_max, length_sum;
867  pg_edit_type_t edit_type;
868
869  v_min = s->min_packet_bytes;
870  v_max = s->max_packet_bytes;
871  edit_type = s->packet_size_edit_type;
872
873  if (edit_type == PG_EDIT_INCREMENT)
874    s->last_increment_packet_size
875      = do_set_increment (pg, s, buffers, n_buffers,
876			  8 * STRUCT_SIZE_OF (vlib_buffer_t, current_length),
877			  STRUCT_OFFSET_OF (vlib_buffer_t, current_length),
878			  /* is_net_byte_order */ 0,
879			  /* want sum */ 1, &length_sum,
880			  v_min, v_max, s->last_increment_packet_size);
881
882  else if (edit_type == PG_EDIT_RANDOM)
883    do_set_random (pg, s, buffers, n_buffers,
884		   8 * STRUCT_SIZE_OF (vlib_buffer_t, current_length),
885		   STRUCT_OFFSET_OF (vlib_buffer_t, current_length),
886		   /* is_net_byte_order */ 0,
887		   /* want sum */ 1, &length_sum,
888		   v_min, v_max);
889
890  else				/* edit_type == PG_EDIT_FIXED */
891    {
892      do_set_fixed (pg, s, buffers, n_buffers,
893		    8 * STRUCT_SIZE_OF (vlib_buffer_t, current_length),
894		    STRUCT_OFFSET_OF (vlib_buffer_t, current_length),
895		    /* is_net_byte_order */ 0,
896		    v_min, v_max);
897      length_sum = v_min * n_buffers;
898    }
899
900  {
901    vnet_main_t *vnm = vnet_get_main ();
902    vnet_interface_main_t *im = &vnm->interface_main;
903    vnet_sw_interface_t *si =
904      vnet_get_sw_interface (vnm, s->sw_if_index[VLIB_RX]);
905
906    vlib_increment_combined_counter (im->combined_sw_if_counters
907				     + VNET_INTERFACE_COUNTER_RX,
908				     vlib_get_thread_index (),
909				     si->sw_if_index, n_buffers, length_sum);
910  }
911
912}
913
914static void
915pg_generate_fix_multi_buffer_lengths (pg_main_t * pg,
916				      pg_stream_t * s,
917				      u32 * buffers, u32 n_buffers)
918{
919  vlib_main_t *vm = vlib_get_main ();
920  pg_buffer_index_t *pbi;
921  uword n_bytes_left;
922  static u32 *unused_buffers = 0;
923
924  while (n_buffers > 0)
925    {
926      vlib_buffer_t *b;
927      u32 bi;
928
929      bi = buffers[0];
930      b = vlib_get_buffer (vm, bi);
931
932      /* Current length here is length of whole packet. */
933      n_bytes_left = b->current_length;
934
935      pbi = s->buffer_indices;
936      while (1)
937	{
938	  uword n = clib_min (n_bytes_left, s->buffer_bytes);
939
940	  b->current_length = n;
941	  n_bytes_left -= n;
942	  if (n_bytes_left > 0)
943	    b->flags |= VLIB_BUFFER_NEXT_PRESENT;
944	  else
945	    b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
946
947	  /* Return unused buffers to fifos. */
948	  if (n == 0)
949	    vec_add1 (unused_buffers, bi);
950
951	  pbi++;
952	  if (pbi >= vec_end (s->buffer_indices))
953	    break;
954
955	  bi = b->next_buffer;
956	  b = vlib_get_buffer (vm, bi);
957	}
958      ASSERT (n_bytes_left == 0);
959
960      buffers += 1;
961      n_buffers -= 1;
962    }
963
964  if (vec_len (unused_buffers) > 0)
965    {
966      vlib_buffer_free_no_next (vm, unused_buffers, vec_len (unused_buffers));
967      _vec_len (unused_buffers) = 0;
968    }
969}
970
971static void
972pg_generate_edit (pg_main_t * pg,
973		  pg_stream_t * s, u32 * buffers, u32 n_buffers)
974{
975  pg_edit_t *e;
976
977  vec_foreach (e, s->non_fixed_edits)
978  {
979    switch (e->type)
980      {
981      case PG_EDIT_RANDOM:
982      case PG_EDIT_INCREMENT:
983	{
984	  u32 lo_bit, hi_bit;
985	  u64 v_min, v_max;
986
987	  v_min = pg_edit_get_value (e, PG_EDIT_LO);
988	  v_max = pg_edit_get_value (e, PG_EDIT_HI);
989
990	  hi_bit = (BITS (u8) * STRUCT_OFFSET_OF (vlib_buffer_t, data)
991		    + BITS (u8) + e->lsb_bit_offset);
992	  lo_bit = hi_bit - e->n_bits;
993
994	  e->last_increment_value
995	    = do_it (pg, s, buffers, n_buffers, lo_bit, hi_bit, v_min, v_max,
996		     e->last_increment_value, e->type);
997	}
998	break;
999
1000      case PG_EDIT_UNSPECIFIED:
1001	break;
1002
1003      default:
1004	/* Should not be any fixed edits left. */
1005	ASSERT (0);
1006	break;
1007      }
1008  }
1009
1010  /* Call any edit functions to e.g. completely IP lengths, checksums, ... */
1011  {
1012    int i;
1013    for (i = vec_len (s->edit_groups) - 1; i >= 0; i--)
1014      {
1015	pg_edit_group_t *g = s->edit_groups + i;
1016	if (g->edit_function)
1017	  g->edit_function (pg, s, g, buffers, n_buffers);
1018      }
1019  }
1020}
1021
1022static void
1023pg_set_next_buffer_pointers (pg_main_t * pg,
1024			     pg_stream_t * s,
1025			     u32 * buffers, u32 * next_buffers, u32 n_buffers)
1026{
1027  vlib_main_t *vm = vlib_get_main ();
1028
1029  while (n_buffers >= 4)
1030    {
1031      u32 ni0, ni1;
1032      vlib_buffer_t *b0, *b1;
1033
1034      b0 = vlib_get_buffer (vm, buffers[0]);
1035      b1 = vlib_get_buffer (vm, buffers[1]);
1036      ni0 = next_buffers[0];
1037      ni1 = next_buffers[1];
1038
1039      vlib_prefetch_buffer_with_index (vm, buffers[2], WRITE);
1040      vlib_prefetch_buffer_with_index (vm, buffers[3], WRITE);
1041
1042      b0->flags |= VLIB_BUFFER_NEXT_PRESENT;
1043      b1->flags |= VLIB_BUFFER_NEXT_PRESENT;
1044      b0->next_buffer = ni0;
1045      b1->next_buffer = ni1;
1046
1047      buffers += 2;
1048      next_buffers += 2;
1049      n_buffers -= 2;
1050    }
1051
1052  while (n_buffers > 0)
1053    {
1054      u32 ni0;
1055      vlib_buffer_t *b0;
1056
1057      b0 = vlib_get_buffer (vm, buffers[0]);
1058      ni0 = next_buffers[0];
1059      buffers += 1;
1060      next_buffers += 1;
1061      n_buffers -= 1;
1062
1063      b0->flags |= VLIB_BUFFER_NEXT_PRESENT;
1064      b0->next_buffer = ni0;
1065    }
1066}
1067
1068static_always_inline void
1069init_buffers_inline (vlib_main_t * vm,
1070		     pg_stream_t * s,
1071		     u32 * buffers,
1072		     u32 n_buffers, u32 data_offset, u32 n_data, u32 set_data)
1073{
1074  u32 n_left, *b;
1075  u8 *data, *mask;
1076
1077  ASSERT (s->replay_packet_templates == 0);
1078
1079  data = s->fixed_packet_data + data_offset;
1080  mask = s->fixed_packet_data_mask + data_offset;
1081  if (data + n_data >= vec_end (s->fixed_packet_data))
1082    n_data = (data < vec_end (s->fixed_packet_data)
1083	      ? vec_end (s->fixed_packet_data) - data : 0);
1084  if (n_data > 0)
1085    {
1086      ASSERT (data + n_data <= vec_end (s->fixed_packet_data));
1087      ASSERT (mask + n_data <= vec_end (s->fixed_packet_data_mask));
1088    }
1089
1090  n_left = n_buffers;
1091  b = buffers;
1092
1093  while (n_left >= 4)
1094    {
1095      u32 bi0, bi1;
1096      vlib_buffer_t *b0, *b1;
1097
1098      /* Prefetch next iteration. */
1099      vlib_prefetch_buffer_with_index (vm, b[2], STORE);
1100      vlib_prefetch_buffer_with_index (vm, b[3], STORE);
1101
1102      bi0 = b[0];
1103      bi1 = b[1];
1104      b += 2;
1105      n_left -= 2;
1106
1107      b0 = vlib_get_buffer (vm, bi0);
1108      b1 = vlib_get_buffer (vm, bi1);
1109      b0->flags |= s->buffer_flags;
1110      b1->flags |= s->buffer_flags;
1111
1112      vnet_buffer (b0)->sw_if_index[VLIB_RX] =
1113	vnet_buffer (b1)->sw_if_index[VLIB_RX] = s->sw_if_index[VLIB_RX];
1114
1115      vnet_buffer (b0)->sw_if_index[VLIB_TX] =
1116	vnet_buffer (b1)->sw_if_index[VLIB_TX] = s->sw_if_index[VLIB_TX];
1117
1118      if (set_data)
1119	{
1120	  clib_memcpy_fast (b0->data, data, n_data);
1121	  clib_memcpy_fast (b1->data, data, n_data);
1122	}
1123      else
1124	{
1125	  ASSERT (validate_buffer_data2 (b0, s, data_offset, n_data));
1126	  ASSERT (validate_buffer_data2 (b1, s, data_offset, n_data));
1127	}
1128    }
1129
1130  while (n_left >= 1)
1131    {
1132      u32 bi0;
1133      vlib_buffer_t *b0;
1134
1135      bi0 = b[0];
1136      b += 1;
1137      n_left -= 1;
1138
1139      b0 = vlib_get_buffer (vm, bi0);
1140      b0->flags |= s->buffer_flags;
1141      vnet_buffer (b0)->sw_if_index[VLIB_RX] = s->sw_if_index[VLIB_RX];
1142      vnet_buffer (b0)->sw_if_index[VLIB_TX] = s->sw_if_index[VLIB_TX];
1143
1144      if (set_data)
1145	clib_memcpy_fast (b0->data, data, n_data);
1146      else
1147	ASSERT (validate_buffer_data2 (b0, s, data_offset, n_data));
1148    }
1149}
1150
1151static u32
1152pg_stream_fill_helper (pg_main_t * pg,
1153		       pg_stream_t * s,
1154		       pg_buffer_index_t * bi,
1155		       u32 * buffers, u32 * next_buffers, u32 n_alloc)
1156{
1157  vlib_main_t *vm = vlib_get_main ();
1158  uword is_start_of_packet = bi == s->buffer_indices;
1159  u32 n_allocated;
1160
1161  ASSERT (vec_len (s->replay_packet_templates) == 0);
1162
1163  n_allocated = vlib_buffer_alloc (vm, buffers, n_alloc);
1164  if (n_allocated == 0)
1165    return 0;
1166
1167  /*
1168   * We can't assume we got all the buffers we asked for...
1169   * This never worked until recently.
1170   */
1171  n_alloc = n_allocated;
1172
1173  /* Reinitialize buffers */
1174  init_buffers_inline
1175    (vm, s,
1176     buffers,
1177     n_alloc, (bi - s->buffer_indices) * s->buffer_bytes /* data offset */ ,
1178     s->buffer_bytes,
1179     /* set_data */ 1);
1180
1181  if (next_buffers)
1182    pg_set_next_buffer_pointers (pg, s, buffers, next_buffers, n_alloc);
1183
1184  if (is_start_of_packet)
1185    {
1186      pg_generate_set_lengths (pg, s, buffers, n_alloc);
1187      if (vec_len (s->buffer_indices) > 1)
1188	pg_generate_fix_multi_buffer_lengths (pg, s, buffers, n_alloc);
1189
1190      pg_generate_edit (pg, s, buffers, n_alloc);
1191    }
1192
1193  return n_alloc;
1194}
1195
1196static u32
1197pg_stream_fill_replay (pg_main_t * pg, pg_stream_t * s, u32 n_alloc)
1198{
1199  pg_buffer_index_t *bi;
1200  u32 n_left, i, l;
1201  u32 buffer_alloc_request = 0;
1202  u32 buffer_alloc_result;
1203  u32 current_buffer_index;
1204  u32 *buffers;
1205  vlib_main_t *vm = vlib_get_main ();
1206  vnet_main_t *vnm = vnet_get_main ();
1207  u32 buf_sz = vlib_buffer_get_default_data_size (vm);
1208  vnet_interface_main_t *im = &vnm->interface_main;
1209  vnet_sw_interface_t *si;
1210
1211  buffers = pg->replay_buffers_by_thread[vm->thread_index];
1212  vec_reset_length (buffers);
1213  bi = s->buffer_indices;
1214
1215  n_left = n_alloc;
1216  i = s->current_replay_packet_index;
1217  l = vec_len (s->replay_packet_templates);
1218
1219  /* Figure out how many buffers we need */
1220  while (n_left > 0)
1221    {
1222      u8 *d0;
1223
1224      d0 = vec_elt (s->replay_packet_templates, i);
1225      buffer_alloc_request += (vec_len (d0) + (buf_sz - 1)) / buf_sz;
1226
1227      i = ((i + 1) == l) ? 0 : i + 1;
1228      n_left--;
1229    }
1230
1231  ASSERT (buffer_alloc_request > 0);
1232  vec_validate (buffers, buffer_alloc_request - 1);
1233
1234  /* Allocate that many buffers */
1235  buffer_alloc_result = vlib_buffer_alloc (vm, buffers, buffer_alloc_request);
1236  if (buffer_alloc_result < buffer_alloc_request)
1237    {
1238      clib_warning ("alloc failure, got %d not %d", buffer_alloc_result,
1239		    buffer_alloc_request);
1240      vlib_buffer_free_no_next (vm, buffers, buffer_alloc_result);
1241      pg->replay_buffers_by_thread[vm->thread_index] = buffers;
1242      return 0;
1243    }
1244
1245  /* Now go generate the buffers, and add them to the FIFO */
1246  n_left = n_alloc;
1247
1248  current_buffer_index = 0;
1249  i = s->current_replay_packet_index;
1250  l = vec_len (s->replay_packet_templates);
1251  while (n_left > 0)
1252    {
1253      u8 *d0;
1254      int not_last;
1255      u32 data_offset;
1256      u32 bytes_to_copy, bytes_this_chunk;
1257      vlib_buffer_t *b;
1258
1259      d0 = vec_elt (s->replay_packet_templates, i);
1260      data_offset = 0;
1261      bytes_to_copy = vec_len (d0);
1262
1263      /* Add head chunk to pg fifo */
1264      clib_fifo_add1 (bi->buffer_fifo, buffers[current_buffer_index]);
1265
1266      /* Copy the data */
1267      while (bytes_to_copy)
1268	{
1269	  bytes_this_chunk = clib_min (bytes_to_copy, buf_sz);
1270	  ASSERT (current_buffer_index < vec_len (buffers));
1271	  b = vlib_get_buffer (vm, buffers[current_buffer_index]);
1272	  clib_memcpy_fast (b->data, d0 + data_offset, bytes_this_chunk);
1273	  vnet_buffer (b)->sw_if_index[VLIB_RX] = s->sw_if_index[VLIB_RX];
1274	  vnet_buffer (b)->sw_if_index[VLIB_TX] = s->sw_if_index[VLIB_TX];
1275	  b->flags = s->buffer_flags;
1276	  b->next_buffer = 0;
1277	  b->current_data = 0;
1278	  b->current_length = bytes_this_chunk;
1279
1280	  not_last = bytes_this_chunk < bytes_to_copy;
1281	  if (not_last)
1282	    {
1283	      ASSERT (current_buffer_index < (vec_len (buffers) - 1));
1284	      b->flags |= VLIB_BUFFER_NEXT_PRESENT;
1285	      b->next_buffer = buffers[current_buffer_index + 1];
1286	    }
1287	  bytes_to_copy -= bytes_this_chunk;
1288	  data_offset += bytes_this_chunk;
1289	  current_buffer_index++;
1290	}
1291
1292      i = ((i + 1) == l) ? 0 : i + 1;
1293      n_left--;
1294    }
1295
1296  /* Update the interface counters */
1297  si = vnet_get_sw_interface (vnm, s->sw_if_index[VLIB_RX]);
1298  l = 0;
1299  for (i = 0; i < n_alloc; i++)
1300    l += vlib_buffer_index_length_in_chain (vm, buffers[i]);
1301  vlib_increment_combined_counter (im->combined_sw_if_counters
1302				   + VNET_INTERFACE_COUNTER_RX,
1303				   vlib_get_thread_index (),
1304				   si->sw_if_index, n_alloc, l);
1305
1306  s->current_replay_packet_index += n_alloc;
1307  s->current_replay_packet_index %= vec_len (s->replay_packet_templates);
1308
1309  pg->replay_buffers_by_thread[vm->thread_index] = buffers;
1310  return n_alloc;
1311}
1312
1313
1314static u32
1315pg_stream_fill (pg_main_t * pg, pg_stream_t * s, u32 n_buffers)
1316{
1317  pg_buffer_index_t *bi;
1318  word i, n_in_fifo, n_alloc, n_free, n_added;
1319  u32 *tail, *start, *end, *last_tail, *last_start;
1320
1321  bi = s->buffer_indices;
1322
1323  n_in_fifo = clib_fifo_elts (bi->buffer_fifo);
1324  if (n_in_fifo >= n_buffers)
1325    return n_in_fifo;
1326
1327  n_alloc = n_buffers - n_in_fifo;
1328
1329  /* Round up, but never generate more than limit. */
1330  n_alloc = clib_max (VLIB_FRAME_SIZE, n_alloc);
1331
1332  if (s->n_packets_limit > 0
1333      && s->n_packets_generated + n_in_fifo + n_alloc >= s->n_packets_limit)
1334    {
1335      n_alloc = s->n_packets_limit - s->n_packets_generated - n_in_fifo;
1336      if (n_alloc < 0)
1337	n_alloc = 0;
1338    }
1339
1340  /*
1341   * Handle pcap replay directly
1342   */
1343  if (s->replay_packet_templates)
1344    return pg_stream_fill_replay (pg, s, n_alloc);
1345
1346  /* All buffer fifos should have the same size. */
1347  if (CLIB_DEBUG > 0)
1348    {
1349      uword l = ~0, e;
1350      vec_foreach (bi, s->buffer_indices)
1351      {
1352	e = clib_fifo_elts (bi->buffer_fifo);
1353	if (bi == s->buffer_indices)
1354	  l = e;
1355	ASSERT (l == e);
1356      }
1357    }
1358
1359  last_tail = last_start = 0;
1360  n_added = n_alloc;
1361
1362  for (i = vec_len (s->buffer_indices) - 1; i >= 0; i--)
1363    {
1364      bi = vec_elt_at_index (s->buffer_indices, i);
1365
1366      n_free = clib_fifo_free_elts (bi->buffer_fifo);
1367      if (n_free < n_alloc)
1368	clib_fifo_resize (bi->buffer_fifo, n_alloc - n_free);
1369
1370      tail = clib_fifo_advance_tail (bi->buffer_fifo, n_alloc);
1371      start = bi->buffer_fifo;
1372      end = clib_fifo_end (bi->buffer_fifo);
1373
1374      if (tail + n_alloc <= end)
1375	{
1376	  n_added =
1377	    pg_stream_fill_helper (pg, s, bi, tail, last_tail, n_alloc);
1378	}
1379      else
1380	{
1381	  u32 n = clib_min (end - tail, n_alloc);
1382	  n_added = pg_stream_fill_helper (pg, s, bi, tail, last_tail, n);
1383
1384	  if (n_added == n && n_alloc > n_added)
1385	    {
1386	      n_added += pg_stream_fill_helper
1387		(pg, s, bi, start, last_start, n_alloc - n_added);
1388	    }
1389	}
1390
1391      if (PREDICT_FALSE (n_added < n_alloc))
1392	tail = clib_fifo_advance_tail (bi->buffer_fifo, n_added - n_alloc);
1393
1394      last_tail = tail;
1395      last_start = start;
1396
1397      /* Verify that pkts in the fifo are properly allocated */
1398    }
1399
1400  return n_in_fifo + n_added;
1401}
1402
1403typedef struct
1404{
1405  u32 stream_index;
1406
1407  u32 packet_length;
1408  u32 sw_if_index;
1409
1410  /* Use pre data for packet data. */
1411  vlib_buffer_t buffer;
1412} pg_input_trace_t;
1413
1414static u8 *
1415format_pg_input_trace (u8 * s, va_list * va)
1416{
1417  vlib_main_t *vm = va_arg (*va, vlib_main_t *);
1418  CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
1419  pg_input_trace_t *t = va_arg (*va, pg_input_trace_t *);
1420  pg_main_t *pg = &pg_main;
1421  pg_stream_t *stream;
1422  vlib_node_t *n;
1423  u32 indent = format_get_indent (s);
1424
1425  stream = 0;
1426  if (!pool_is_free_index (pg->streams, t->stream_index))
1427    stream = pool_elt_at_index (pg->streams, t->stream_index);
1428
1429  if (stream)
1430    s = format (s, "stream %v", pg->streams[t->stream_index].name);
1431  else
1432    s = format (s, "stream %d", t->stream_index);
1433
1434  s = format (s, ", %d bytes", t->packet_length);
1435  s = format (s, ", sw_if_index %d", t->sw_if_index);
1436
1437  s = format (s, "\n%U%U",
1438	      format_white_space, indent, format_vnet_buffer, &t->buffer);
1439
1440  s = format (s, "\n%U", format_white_space, indent);
1441
1442  n = 0;
1443  if (stream)
1444    n = vlib_get_node (vm, stream->node_index);
1445
1446  if (n && n->format_buffer)
1447    s = format (s, "%U", n->format_buffer,
1448		t->buffer.pre_data, sizeof (t->buffer.pre_data));
1449  else
1450    s = format (s, "%U",
1451		format_hex_bytes, t->buffer.pre_data,
1452		ARRAY_LEN (t->buffer.pre_data));
1453  return s;
1454}
1455
1456static void
1457pg_input_trace (pg_main_t * pg,
1458		vlib_node_runtime_t * node, u32 stream_index, u32 next_index,
1459		u32 * buffers, u32 n_buffers)
1460{
1461  vlib_main_t *vm = vlib_get_main ();
1462  u32 *b, n_left;
1463
1464  n_left = n_buffers;
1465  b = buffers;
1466
1467  while (n_left >= 2)
1468    {
1469      u32 bi0, bi1;
1470      vlib_buffer_t *b0, *b1;
1471      pg_input_trace_t *t0, *t1;
1472
1473      bi0 = b[0];
1474      bi1 = b[1];
1475      b += 2;
1476      n_left -= 2;
1477
1478      b0 = vlib_get_buffer (vm, bi0);
1479      b1 = vlib_get_buffer (vm, bi1);
1480
1481      vlib_trace_buffer (vm, node, next_index, b0, /* follow_chain */ 1);
1482      vlib_trace_buffer (vm, node, next_index, b1, /* follow_chain */ 1);
1483
1484      t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1485      t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1486
1487      t0->stream_index = stream_index;
1488      t1->stream_index = stream_index;
1489
1490      t0->packet_length = vlib_buffer_length_in_chain (vm, b0);
1491      t1->packet_length = vlib_buffer_length_in_chain (vm, b1);
1492
1493      t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1494      t1->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1495
1496      clib_memcpy_fast (&t0->buffer, b0,
1497			sizeof (b0[0]) - sizeof (b0->pre_data));
1498      clib_memcpy_fast (&t1->buffer, b1,
1499			sizeof (b1[0]) - sizeof (b1->pre_data));
1500
1501      clib_memcpy_fast (t0->buffer.pre_data, b0->data,
1502			sizeof (t0->buffer.pre_data));
1503      clib_memcpy_fast (t1->buffer.pre_data, b1->data,
1504			sizeof (t1->buffer.pre_data));
1505    }
1506
1507  while (n_left >= 1)
1508    {
1509      u32 bi0;
1510      vlib_buffer_t *b0;
1511      pg_input_trace_t *t0;
1512
1513      bi0 = b[0];
1514      b += 1;
1515      n_left -= 1;
1516
1517      b0 = vlib_get_buffer (vm, bi0);
1518
1519      vlib_trace_buffer (vm, node, next_index, b0, /* follow_chain */ 1);
1520      t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1521
1522      t0->stream_index = stream_index;
1523      t0->packet_length = vlib_buffer_length_in_chain (vm, b0);
1524      t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1525      clib_memcpy_fast (&t0->buffer, b0,
1526			sizeof (b0[0]) - sizeof (b0->pre_data));
1527      clib_memcpy_fast (t0->buffer.pre_data, b0->data,
1528			sizeof (t0->buffer.pre_data));
1529    }
1530}
1531
1532static_always_inline void
1533fill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
1534			   int gso_enabled, u32 gso_size)
1535{
1536  for (int i = 0; i < n_buffers; i++)
1537    {
1538      vlib_buffer_t *b0 = vlib_get_buffer (vm, buffers[i]);
1539      u8 l4_proto = 0;
1540
1541      ethernet_header_t *eh =
1542	(ethernet_header_t *) vlib_buffer_get_current (b0);
1543      u16 ethertype = clib_net_to_host_u16 (eh->type);
1544      u16 l2hdr_sz = sizeof (ethernet_header_t);
1545
1546      if (ethernet_frame_is_tagged (ethertype))
1547	{
1548	  ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1);
1549
1550	  ethertype = clib_net_to_host_u16 (vlan->type);
1551	  l2hdr_sz += sizeof (*vlan);
1552	  if (ethertype == ETHERNET_TYPE_VLAN)
1553	    {
1554	      vlan++;
1555	      ethertype = clib_net_to_host_u16 (vlan->type);
1556	      l2hdr_sz += sizeof (*vlan);
1557	    }
1558	}
1559
1560      vnet_buffer (b0)->l2_hdr_offset = 0;
1561      vnet_buffer (b0)->l3_hdr_offset = l2hdr_sz;
1562
1563      if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP4))
1564	{
1565	  ip4_header_t *ip4 =
1566	    (ip4_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz);
1567	  vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4);
1568	  l4_proto = ip4->protocol;
1569	  b0->flags |=
1570	    (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
1571	  b0->flags |= (VNET_BUFFER_F_L2_HDR_OFFSET_VALID
1572			| VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
1573			VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
1574	}
1575      else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
1576	{
1577	  ip6_header_t *ip6 =
1578	    (ip6_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz);
1579	  vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + sizeof (ip6_header_t);
1580	  /* FIXME IPv6 EH traversal */
1581	  l4_proto = ip6->protocol;
1582	  b0->flags |=
1583	    (VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
1584	     VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
1585	     VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
1586	}
1587
1588      if (l4_proto == IP_PROTOCOL_TCP)
1589	{
1590	  b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
1591
1592	  /* only set GSO flag for chained buffers */
1593	  if (gso_enabled && (b0->flags & VLIB_BUFFER_NEXT_PRESENT))
1594	    {
1595	      b0->flags |= VNET_BUFFER_F_GSO;
1596	      tcp_header_t *tcp =
1597		(tcp_header_t *) (vlib_buffer_get_current (b0) +
1598				  vnet_buffer (b0)->l4_hdr_offset);
1599	      vnet_buffer2 (b0)->gso_l4_hdr_sz = tcp_header_bytes (tcp);
1600	      vnet_buffer2 (b0)->gso_size = gso_size;
1601	    }
1602	}
1603      else if (l4_proto == IP_PROTOCOL_UDP)
1604	{
1605	  b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
1606	}
1607    }
1608}
1609
1610static uword
1611pg_generate_packets (vlib_node_runtime_t * node,
1612		     pg_main_t * pg,
1613		     pg_stream_t * s, uword n_packets_to_generate)
1614{
1615  vlib_main_t *vm = vlib_get_main ();
1616  u32 *to_next, n_this_frame, n_left, n_trace, n_packets_in_fifo;
1617  uword n_packets_generated;
1618  pg_buffer_index_t *bi, *bi0;
1619  u32 next_index = s->next_index;
1620  vnet_feature_main_t *fm = &feature_main;
1621  vnet_feature_config_main_t *cm;
1622  u8 feature_arc_index = fm->device_input_feature_arc_index;
1623  cm = &fm->feature_config_mains[feature_arc_index];
1624  u32 current_config_index = ~(u32) 0;
1625  pg_interface_t *pi = pool_elt_at_index (pg->interfaces, s->pg_if_index);
1626  int i;
1627
1628  bi0 = s->buffer_indices;
1629
1630  n_packets_in_fifo = pg_stream_fill (pg, s, n_packets_to_generate);
1631  n_packets_to_generate = clib_min (n_packets_in_fifo, n_packets_to_generate);
1632  n_packets_generated = 0;
1633
1634  if (PREDICT_FALSE
1635      (vnet_have_features (feature_arc_index, s->sw_if_index[VLIB_RX])))
1636    {
1637      current_config_index =
1638	vec_elt (cm->config_index_by_sw_if_index, s->sw_if_index[VLIB_RX]);
1639      vnet_get_config_data (&cm->config_main, &current_config_index,
1640			    &next_index, 0);
1641    }
1642
1643  while (n_packets_to_generate > 0)
1644    {
1645      u32 *head, *start, *end;
1646
1647      if (PREDICT_TRUE (next_index == VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT))
1648	{
1649	  vlib_next_frame_t *nf;
1650	  vlib_frame_t *f;
1651	  ethernet_input_frame_t *ef;
1652	  vlib_get_new_next_frame (vm, node, next_index, to_next, n_left);
1653	  nf = vlib_node_runtime_get_next_frame (vm, node, next_index);
1654	  f = vlib_get_frame (vm, nf->frame);
1655	  f->flags = ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX;
1656
1657	  ef = vlib_frame_scalar_args (f);
1658	  ef->sw_if_index = pi->sw_if_index;
1659	  ef->hw_if_index = pi->hw_if_index;
1660	  vlib_frame_no_append (f);
1661	}
1662      else
1663	vlib_get_next_frame (vm, node, next_index, to_next, n_left);
1664
1665      n_this_frame = n_packets_to_generate;
1666      if (n_this_frame > n_left)
1667	n_this_frame = n_left;
1668
1669      start = bi0->buffer_fifo;
1670      end = clib_fifo_end (bi0->buffer_fifo);
1671      head = clib_fifo_head (bi0->buffer_fifo);
1672
1673      if (head + n_this_frame <= end)
1674	vlib_buffer_copy_indices (to_next, head, n_this_frame);
1675      else
1676	{
1677	  u32 n = end - head;
1678	  vlib_buffer_copy_indices (to_next + 0, head, n);
1679	  vlib_buffer_copy_indices (to_next + n, start, n_this_frame - n);
1680	}
1681
1682      if (s->replay_packet_templates == 0)
1683	{
1684	  vec_foreach (bi, s->buffer_indices)
1685	    clib_fifo_advance_head (bi->buffer_fifo, n_this_frame);
1686	}
1687      else
1688	{
1689	  clib_fifo_advance_head (bi0->buffer_fifo, n_this_frame);
1690	}
1691
1692      if (current_config_index != ~(u32) 0)
1693	for (i = 0; i < n_this_frame; i++)
1694	  {
1695	    vlib_buffer_t *b;
1696	    b = vlib_get_buffer (vm, to_next[i]);
1697	    b->current_config_index = current_config_index;
1698	    vnet_buffer (b)->feature_arc_index = feature_arc_index;
1699	  }
1700
1701      if (pi->gso_enabled ||
1702	  (s->buffer_flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
1703			      VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
1704			      VNET_BUFFER_F_OFFLOAD_IP_CKSUM)))
1705	{
1706	  fill_buffer_offload_flags (vm, to_next, n_this_frame,
1707				     pi->gso_enabled, pi->gso_size);
1708	}
1709
1710      n_trace = vlib_get_trace_count (vm, node);
1711      if (n_trace > 0)
1712	{
1713	  u32 n = clib_min (n_trace, n_this_frame);
1714	  pg_input_trace (pg, node, s - pg->streams, next_index, to_next, n);
1715	  vlib_set_trace_count (vm, node, n_trace - n);
1716	}
1717      n_packets_to_generate -= n_this_frame;
1718      n_packets_generated += n_this_frame;
1719      n_left -= n_this_frame;
1720      if (CLIB_DEBUG > 0)
1721	{
1722	  int i;
1723	  vlib_buffer_t *b;
1724
1725	  for (i = 0; i < n_this_frame; i++)
1726	    {
1727	      b = vlib_get_buffer (vm, to_next[i]);
1728	      ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0 ||
1729		      b->current_length >= VLIB_BUFFER_MIN_CHAIN_SEG_SIZE);
1730	    }
1731	}
1732      vlib_put_next_frame (vm, node, next_index, n_left);
1733    }
1734
1735  return n_packets_generated;
1736}
1737
1738static uword
1739pg_input_stream (vlib_node_runtime_t * node, pg_main_t * pg, pg_stream_t * s)
1740{
1741  vlib_main_t *vm = vlib_get_main ();
1742  uword n_packets;
1743  f64 time_now, dt;
1744
1745  if (s->n_packets_limit > 0 && s->n_packets_generated >= s->n_packets_limit)
1746    {
1747      pg_stream_enable_disable (pg, s, /* want_enabled */ 0);
1748      return 0;
1749    }
1750
1751  /* Apply rate limit. */
1752  time_now = vlib_time_now (vm);
1753  if (s->time_last_generate == 0)
1754    s->time_last_generate = time_now;
1755
1756  dt = time_now - s->time_last_generate;
1757  s->time_last_generate = time_now;
1758
1759  n_packets = VLIB_FRAME_SIZE;
1760  if (s->rate_packets_per_second > 0)
1761    {
1762      s->packet_accumulator += dt * s->rate_packets_per_second;
1763      n_packets = s->packet_accumulator;
1764
1765      /* Never allow accumulator to grow if we get behind. */
1766      s->packet_accumulator -= n_packets;
1767    }
1768
1769  /* Apply fixed limit. */
1770  if (s->n_packets_limit > 0
1771      && s->n_packets_generated + n_packets > s->n_packets_limit)
1772    n_packets = s->n_packets_limit - s->n_packets_generated;
1773
1774  /* Generate up to one frame's worth of packets. */
1775  if (n_packets > s->n_max_frame)
1776    n_packets = s->n_max_frame;
1777
1778  if (n_packets > 0)
1779    n_packets = pg_generate_packets (node, pg, s, n_packets);
1780
1781  s->n_packets_generated += n_packets;
1782
1783  return n_packets;
1784}
1785
1786uword
1787pg_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1788{
1789  uword i;
1790  pg_main_t *pg = &pg_main;
1791  uword n_packets = 0;
1792  u32 worker_index = 0;
1793
1794  if (vlib_num_workers ())
1795    worker_index = vlib_get_current_worker_index ();
1796
1797  /* *INDENT-OFF* */
1798  clib_bitmap_foreach (i, pg->enabled_streams[worker_index], ({
1799    pg_stream_t *s = vec_elt_at_index (pg->streams, i);
1800    n_packets += pg_input_stream (node, pg, s);
1801  }));
1802  /* *INDENT-ON* */
1803
1804  return n_packets;
1805}
1806
1807/* *INDENT-OFF* */
1808VLIB_REGISTER_NODE (pg_input_node) = {
1809  .function = pg_input,
1810  .flags = VLIB_NODE_FLAG_TRACE_SUPPORTED,
1811  .name = "pg-input",
1812  .sibling_of = "device-input",
1813  .type = VLIB_NODE_TYPE_INPUT,
1814
1815  .format_trace = format_pg_input_trace,
1816
1817  /* Input node will be left disabled until a stream is active. */
1818  .state = VLIB_NODE_STATE_DISABLED,
1819};
1820/* *INDENT-ON* */
1821
1822/*
1823 * fd.io coding-style-patch-verification: ON
1824 *
1825 * Local Variables:
1826 * eval: (c-set-style "gnu")
1827 * End:
1828 */
1829