1cb9cadadSEd Warnicke/*
2cb9cadadSEd Warnicke * Copyright (c) 2015 Cisco and/or its affiliates.
3cb9cadadSEd Warnicke * Licensed under the Apache License, Version 2.0 (the "License");
4cb9cadadSEd Warnicke * you may not use this file except in compliance with the License.
5cb9cadadSEd Warnicke * You may obtain a copy of the License at:
6cb9cadadSEd Warnicke *
7cb9cadadSEd Warnicke *     http://www.apache.org/licenses/LICENSE-2.0
8cb9cadadSEd Warnicke *
9cb9cadadSEd Warnicke * Unless required by applicable law or agreed to in writing, software
10cb9cadadSEd Warnicke * distributed under the License is distributed on an "AS IS" BASIS,
11cb9cadadSEd Warnicke * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12cb9cadadSEd Warnicke * See the License for the specific language governing permissions and
13cb9cadadSEd Warnicke * limitations under the License.
14cb9cadadSEd Warnicke */
15cb9cadadSEd Warnicke/*
16cb9cadadSEd Warnicke * pg_input.c: buffer generator input
17cb9cadadSEd Warnicke *
18cb9cadadSEd Warnicke * Copyright (c) 2008 Eliot Dresselhaus
19cb9cadadSEd Warnicke *
20cb9cadadSEd Warnicke * Permission is hereby granted, free of charge, to any person obtaining
21cb9cadadSEd Warnicke * a copy of this software and associated documentation files (the
22cb9cadadSEd Warnicke * "Software"), to deal in the Software without restriction, including
23cb9cadadSEd Warnicke * without limitation the rights to use, copy, modify, merge, publish,
24cb9cadadSEd Warnicke * distribute, sublicense, and/or sell copies of the Software, and to
25cb9cadadSEd Warnicke * permit persons to whom the Software is furnished to do so, subject to
26cb9cadadSEd Warnicke * the following conditions:
27cb9cadadSEd Warnicke *
28cb9cadadSEd Warnicke * The above copyright notice and this permission notice shall be
29cb9cadadSEd Warnicke * included in all copies or substantial portions of the Software.
30cb9cadadSEd Warnicke *
38cb9cadadSEd Warnicke */
39cb9cadadSEd Warnicke
403c8e1468SDave Barach  /*
413c8e1468SDave Barach   * To be honest, the packet generator needs an extreme
423c8e1468SDave Barach   * makeover. Two key assumptions which drove the current implementation
433c8e1468SDave Barach   * are no longer true. First, buffer managers implement a
443c8e1468SDave Barach   * post-TX recycle list. Second, that packet generator performance
453c8e1468SDave Barach   * is first-order important.
463c8e1468SDave Barach   */
473c8e1468SDave Barach
48cb9cadadSEd Warnicke#include <vlib/vlib.h>
49cb9cadadSEd Warnicke#include <vnet/pg/pg.h>
50cb9cadadSEd Warnicke#include <vnet/vnet.h>
51650223c0SDamjan Marion#include <vnet/ethernet/ethernet.h>
52d2017f6dSDamjan Marion#include <vnet/feature/feature.h>
5322e9cfd7SMohsin Kazmi#include <vnet/ip/ip4_packet.h>
5422e9cfd7SMohsin Kazmi#include <vnet/ip/ip6_packet.h>
5522e9cfd7SMohsin Kazmi#include <vnet/udp/udp_packet.h>
56d2017f6dSDamjan Marion#include <vnet/devices/devices.h>
57cb9cadadSEd Warnicke
58cb9cadadSEd Warnickestatic int
59cb9cadadSEd Warnickevalidate_buffer_data2 (vlib_buffer_t * b, pg_stream_t * s,
60cb9cadadSEd Warnicke		       u32 data_offset, u32 n_bytes)
61cb9cadadSEd Warnicke{
6271e97c63SCalvin  u8 *bd, *pd, *pm;
63cb9cadadSEd Warnicke  u32 i;
64cb9cadadSEd Warnicke
65cb9cadadSEd Warnicke  bd = b->data;
66cb9cadadSEd Warnicke  pd = s->fixed_packet_data + data_offset;
67cb9cadadSEd Warnicke  pm = s->fixed_packet_data_mask + data_offset;
68cb9cadadSEd Warnicke
69cb9cadadSEd Warnicke  if (pd + n_bytes >= vec_end (s->fixed_packet_data))
70cb9cadadSEd Warnicke    n_bytes = (pd < vec_end (s->fixed_packet_data)
7171e97c63SCalvin	       ? vec_end (s->fixed_packet_data) - pd : 0);
72cb9cadadSEd Warnicke
73cb9cadadSEd Warnicke  for (i = 0; i < n_bytes; i++)
74cb9cadadSEd Warnicke    if ((bd[i] & pm[i]) != pd[i])
75cb9cadadSEd Warnicke      break;
76cb9cadadSEd Warnicke
77cb9cadadSEd Warnicke  if (i >= n_bytes)
78cb9cadadSEd Warnicke    return 1;
79cb9cadadSEd Warnicke
80bd846cdcSDamjan Marion  clib_warning ("buffer %U", format_vnet_buffer, b);
81cb9cadadSEd Warnicke  clib_warning ("differ at index %d", i);
82cb9cadadSEd Warnicke  clib_warning ("is     %U", format_hex_bytes, bd, n_bytes);
83cb9cadadSEd Warnicke  clib_warning ("mask   %U", format_hex_bytes, pm, n_bytes);
84cb9cadadSEd Warnicke  clib_warning ("expect %U", format_hex_bytes, pd, n_bytes);
85cb9cadadSEd Warnicke  return 0;
86cb9cadadSEd Warnicke}
87cb9cadadSEd Warnicke
88cb9cadadSEd Warnickestatic int
89cb9cadadSEd Warnickevalidate_buffer_data (vlib_buffer_t * b, pg_stream_t * s)
9171e97c63SCalvin  return validate_buffer_data2 (b, s, 0, s->buffer_bytes);
93cb9cadadSEd Warnicke
94cb9cadadSEd Warnickealways_inline void
9571e97c63SCalvinset_1 (void *a0,
9671e97c63SCalvin       u64 v0, u64 v_min, u64 v_max, u32 n_bits, u32 is_net_byte_order)
97cb9cadadSEd Warnicke{
98cb9cadadSEd Warnicke  ASSERT (v0 >= v_min && v0 <= v_max);
99cb9cadadSEd Warnicke  if (n_bits == BITS (u8))
100cb9cadadSEd Warnicke    {
101cb9cadadSEd Warnicke      ((u8 *) a0)[0] = v0;
102cb9cadadSEd Warnicke    }
103cb9cadadSEd Warnicke  else if (n_bits == BITS (u16))
104cb9cadadSEd Warnicke    {
105cb9cadadSEd Warnicke      if (is_net_byte_order)
106cb9cadadSEd Warnicke	v0 = clib_host_to_net_u16 (v0);
107cb9cadadSEd Warnicke      clib_mem_unaligned (a0, u16) = v0;
108cb9cadadSEd Warnicke    }
109cb9cadadSEd Warnicke  else if (n_bits == BITS (u32))
110cb9cadadSEd Warnicke    {
111cb9cadadSEd Warnicke      if (is_net_byte_order)
112cb9cadadSEd Warnicke	v0 = clib_host_to_net_u32 (v0);
113cb9cadadSEd Warnicke      clib_mem_unaligned (a0, u32) = v0;
114cb9cadadSEd Warnicke    }
115cb9cadadSEd Warnicke  else if (n_bits == BITS (u64))
116cb9cadadSEd Warnicke    {
117cb9cadadSEd Warnicke      if (is_net_byte_order)
118cb9cadadSEd Warnicke	v0 = clib_host_to_net_u64 (v0);
119cb9cadadSEd Warnicke      clib_mem_unaligned (a0, u64) = v0;
120cb9cadadSEd Warnicke    }
121cb9cadadSEd Warnicke}
122cb9cadadSEd Warnicke
123cb9cadadSEd Warnickealways_inline void
12471e97c63SCalvinset_2 (void *a0, void *a1,
125cb9cadadSEd Warnicke       u64 v0, u64 v1,
126cb9cadadSEd Warnicke       u64 v_min, u64 v_max,
12771e97c63SCalvin       u32 n_bits, u32 is_net_byte_order, u32 is_increment)
128cb9cadadSEd Warnicke{
129cb9cadadSEd Warnicke  ASSERT (v0 >= v_min && v0 <= v_max);
130cb9cadadSEd Warnicke  ASSERT (v1 >= v_min && v1 <= (v_max + is_increment));
131cb9cadadSEd Warnicke  if (n_bits == BITS (u8))
132cb9cadadSEd Warnicke    {
133cb9cadadSEd Warnicke      ((u8 *) a0)[0] = v0;
134cb9cadadSEd Warnicke      ((u8 *) a1)[0] = v1;
135cb9cadadSEd Warnicke    }
136cb9cadadSEd Warnicke  else if (n_bits == BITS (u16))
137cb9cadadSEd Warnicke    {
138cb9cadadSEd Warnicke      if (is_net_byte_order)
139cb9cadadSEd Warnicke	{
140cb9cadadSEd Warnicke	  v0 = clib_host_to_net_u16 (v0);
141cb9cadadSEd Warnicke	  v1 = clib_host_to_net_u16 (v1);
142cb9cadadSEd Warnicke	}
143cb9cadadSEd Warnicke      clib_mem_unaligned (a0, u16) = v0;
144cb9cadadSEd Warnicke      clib_mem_unaligned (a1, u16) = v1;
145cb9cadadSEd Warnicke    }
146cb9cadadSEd Warnicke  else if (n_bits == BITS (u32))
147cb9cadadSEd Warnicke    {
148cb9cadadSEd Warnicke      if (is_net_byte_order)
149cb9cadadSEd Warnicke	{
150cb9cadadSEd Warnicke	  v0 = clib_host_to_net_u32 (v0);
151cb9cadadSEd Warnicke	  v1 = clib_host_to_net_u32 (v1);
152cb9cadadSEd Warnicke	}
153cb9cadadSEd Warnicke      clib_mem_unaligned (a0, u32) = v0;
154cb9cadadSEd Warnicke      clib_mem_unaligned (a1, u32) = v1;
155cb9cadadSEd Warnicke    }
156cb9cadadSEd Warnicke  else if (n_bits == BITS (u64))
157cb9cadadSEd Warnicke    {
158cb9cadadSEd Warnicke      if (is_net_byte_order)
159cb9cadadSEd Warnicke	{
160cb9cadadSEd Warnicke	  v0 = clib_host_to_net_u64 (v0);
161cb9cadadSEd Warnicke	  v1 = clib_host_to_net_u64 (v1);
162cb9cadadSEd Warnicke	}
163cb9cadadSEd Warnicke      clib_mem_unaligned (a0, u64) = v0;
164cb9cadadSEd Warnicke      clib_mem_unaligned (a1, u64) = v1;
165cb9cadadSEd Warnicke    }
166cb9cadadSEd Warnicke}
167cb9cadadSEd Warnicke
168cb9cadadSEd Warnickestatic_always_inline void
169cb9cadadSEd Warnickedo_set_fixed (pg_main_t * pg,
170cb9cadadSEd Warnicke	      pg_stream_t * s,
171cb9cadadSEd Warnicke	      u32 * buffers,
172cb9cadadSEd Warnicke	      u32 n_buffers,
173cb9cadadSEd Warnicke	      u32 n_bits,
17471e97c63SCalvin	      u32 byte_offset, u32 is_net_byte_order, u64 v_min, u64 v_max)
175cb9cadadSEd Warnicke{
17664034367SDamjan Marion  vlib_main_t *vm = vlib_get_main ();
177cb9cadadSEd Warnicke
178cb9cadadSEd Warnicke  while (n_buffers >= 4)
179cb9cadadSEd Warnicke    {
18071e97c63SCalvin      vlib_buffer_t *b0, *b1, *b2, *b3;
18171e97c63SCalvin      void *a0, *a1;
182cb9cadadSEd Warnicke
183cb9cadadSEd Warnicke      b0 = vlib_get_buffer (vm, buffers[0]);
184cb9cadadSEd Warnicke      b1 = vlib_get_buffer (vm, buffers[1]);
185cb9cadadSEd Warnicke      b2 = vlib_get_buffer (vm, buffers[2]);
186cb9cadadSEd Warnicke      b3 = vlib_get_buffer (vm, buffers[3]);
187cb9cadadSEd Warnicke      buffers += 2;
188cb9cadadSEd Warnicke      n_buffers -= 2;
189cb9cadadSEd Warnicke
190cb9cadadSEd Warnicke      a0 = (void *) b0 + byte_offset;
191cb9cadadSEd Warnicke      a1 = (void *) b1 + byte_offset;
192cb9cadadSEd Warnicke      CLIB_PREFETCH ((void *) b2 + byte_offset, sizeof (v_min), WRITE);
193cb9cadadSEd Warnicke      CLIB_PREFETCH ((void *) b3 + byte_offset, sizeof (v_min), WRITE);
194cb9cadadSEd Warnicke
19571e97c63SCalvin      set_2 (a0, a1, v_min, v_min, v_min, v_max, n_bits, is_net_byte_order,
196cb9cadadSEd Warnicke	     /* is_increment */ 0);
197cb9cadadSEd Warnicke
198cb9cadadSEd Warnicke      ASSERT (validate_buffer_data (b0, s));
199cb9cadadSEd Warnicke      ASSERT (validate_buffer_data (b1, s));
200cb9cadadSEd Warnicke    }
201cb9cadadSEd Warnicke
202cb9cadadSEd Warnicke  while (n_buffers > 0)
203cb9cadadSEd Warnicke    {
20471e97c63SCalvin      vlib_buffer_t *b0;
20571e97c63SCalvin      void *a0;
206cb9cadadSEd Warnicke
207cb9cadadSEd Warnicke      b0 = vlib_get_buffer (vm, buffers[0]);
208cb9cadadSEd Warnicke      buffers += 1;
209cb9cadadSEd Warnicke      n_buffers -= 1;
210cb9cadadSEd Warnicke
211cb9cadadSEd Warnicke      a0 = (void *) b0 + byte_offset;
212cb9cadadSEd Warnicke
21371e97c63SCalvin      set_1 (a0, v_min, v_min, v_max, n_bits, is_net_byte_order);
214cb9cadadSEd Warnicke
215cb9cadadSEd Warnicke      ASSERT (validate_buffer_data (b0, s));
216cb9cadadSEd Warnicke    }
217cb9cadadSEd Warnicke}
218cb9cadadSEd Warnicke
219cb9cadadSEd Warnickestatic_always_inline u64
220cb9cadadSEd Warnickedo_set_increment (pg_main_t * pg,
221cb9cadadSEd Warnicke		  pg_stream_t * s,
222cb9cadadSEd Warnicke		  u32 * buffers,
223cb9cadadSEd Warnicke		  u32 n_buffers,
224cb9cadadSEd Warnicke		  u32 n_bits,
225cb9cadadSEd Warnicke		  u32 byte_offset,
226cb9cadadSEd Warnicke		  u32 is_net_byte_order,
22771e97c63SCalvin		  u32 want_sum, u64 * sum_result, u64 v_min, u64 v_max, u64 v)
228cb9cadadSEd Warnicke{
22964034367SDamjan Marion  vlib_main_t *vm = vlib_get_main ();
230cb9cadadSEd Warnicke  u64 sum = 0;
231cb9cadadSEd Warnicke
232cb9cadadSEd Warnicke  ASSERT (v >= v_min && v <= v_max);
233cb9cadadSEd Warnicke
234cb9cadadSEd Warnicke  while (n_buffers >= 4)
235cb9cadadSEd Warnicke    {
23671e97c63SCalvin      vlib_buffer_t *b0, *b1, *b2, *b3;
23771e97c63SCalvin      void *a0, *a1;
238cb9cadadSEd Warnicke      u64 v_old;
239cb9cadadSEd Warnicke
240cb9cadadSEd Warnicke      b0 = vlib_get_buffer (vm, buffers[0]);
241cb9cadadSEd Warnicke      b1 = vlib_get_buffer (vm, buffers[1]);
242cb9cadadSEd Warnicke      b2 = vlib_get_buffer (vm, buffers[2]);
243cb9cadadSEd Warnicke      b3 = vlib_get_buffer (vm, buffers[3]);
244cb9cadadSEd Warnicke      buffers += 2;
245cb9cadadSEd Warnicke      n_buffers -= 2;
246cb9cadadSEd Warnicke
247cb9cadadSEd Warnicke      a0 = (void *) b0 + byte_offset;
248cb9cadadSEd Warnicke      a1 = (void *) b1 + byte_offset;
249cb9cadadSEd Warnicke      CLIB_PREFETCH ((void *) b2 + byte_offset, sizeof (v_min), WRITE);
250cb9cadadSEd Warnicke      CLIB_PREFETCH ((void *) b3 + byte_offset, sizeof (v_min), WRITE);
251cb9cadadSEd Warnicke
252cb9cadadSEd Warnicke      v_old = v;
253cb9cadadSEd Warnicke      v = v_old + 2;
254cb9cadadSEd Warnicke      v = v > v_max ? v_min : v;
255cb9cadadSEd Warnicke      set_2 (a0, a1,
25671e97c63SCalvin	     v_old + 0, v_old + 1, v_min, v_max, n_bits, is_net_byte_order,
257cb9cadadSEd Warnicke	     /* is_increment */ 1);
258cb9cadadSEd Warnicke
259cb9cadadSEd Warnicke      if (want_sum)
26071e97c63SCalvin	sum += 2 * v_old + 1;
261cb9cadadSEd Warnicke
262cb9cadadSEd Warnicke      if (PREDICT_FALSE (v_old + 1 > v_max))
263cb9cadadSEd Warnicke	{
264cb9cadadSEd Warnicke	  if (want_sum)
26571e97c63SCalvin	    sum -= 2 * v_old + 1;
266cb9cadadSEd Warnicke
267cb9cadadSEd Warnicke	  v = v_old;
268cb9cadadSEd Warnicke	  set_1 (a0, v + 0, v_min, v_max, n_bits, is_net_byte_order);
269cb9cadadSEd Warnicke	  if (want_sum)
270cb9cadadSEd Warnicke	    sum += v;
271cb9cadadSEd Warnicke	  v += 1;
272cb9cadadSEd Warnicke
273cb9cadadSEd Warnicke	  v = v > v_max ? v_min : v;
274cb9cadadSEd Warnicke	  set_1 (a1, v + 0, v_min, v_max, n_bits, is_net_byte_order);
275cb9cadadSEd Warnicke	  if (want_sum)
276cb9cadadSEd Warnicke	    sum += v;
277cb9cadadSEd Warnicke	  v += 1;
278cb9cadadSEd Warnicke	}
279cb9cadadSEd Warnicke
280cb9cadadSEd Warnicke      ASSERT (validate_buffer_data (b0, s));
281cb9cadadSEd Warnicke      ASSERT (validate_buffer_data (b1, s));
282cb9cadadSEd Warnicke    }
283cb9cadadSEd Warnicke
284cb9cadadSEd Warnicke  while (n_buffers > 0)
285cb9cadadSEd Warnicke    {
28671e97c63SCalvin      vlib_buffer_t *b0;
28771e97c63SCalvin      void *a0;
288cb9cadadSEd Warnicke      u64 v_old;
289cb9cadadSEd Warnicke
290cb9cadadSEd Warnicke      b0 = vlib_get_buffer (vm, buffers[0]);
291cb9cadadSEd Warnicke      buffers += 1;
292cb9cadadSEd Warnicke      n_buffers -= 1;
293cb9cadadSEd Warnicke
294cb9cadadSEd Warnicke      a0 = (void *) b0 + byte_offset;
295cb9cadadSEd Warnicke
296cb9cadadSEd Warnicke      v_old = v;
297cb9cadadSEd Warnicke      if (want_sum)
298cb9cadadSEd Warnicke	sum += v_old;
299cb9cadadSEd Warnicke      v += 1;
300cb9cadadSEd Warnicke      v = v > v_max ? v_min : v;
301cb9cadadSEd Warnicke
302cb9cadadSEd Warnicke      ASSERT (v_old >= v_min && v_old <= v_max);
303cb9cadadSEd Warnicke      set_1 (a0, v_old, v_min, v_max, n_bits, is_net_byte_order);
304cb9cadadSEd Warnicke
305cb9cadadSEd Warnicke      ASSERT (validate_buffer_data (b0, s));
306cb9cadadSEd Warnicke    }
307cb9cadadSEd Warnicke
308cb9cadadSEd Warnicke  if (want_sum)
309cb9cadadSEd Warnicke    *sum_result = sum;
310cb9cadadSEd Warnicke
311cb9cadadSEd Warnicke  return v;
312cb9cadadSEd Warnicke}
313cb9cadadSEd Warnicke
314cb9cadadSEd Warnickestatic_always_inline void
315cb9cadadSEd Warnickedo_set_random (pg_main_t * pg,
316cb9cadadSEd Warnicke	       pg_stream_t * s,
317cb9cadadSEd Warnicke	       u32 * buffers,
318cb9cadadSEd Warnicke	       u32 n_buffers,
319cb9cadadSEd Warnicke	       u32 n_bits,
320cb9cadadSEd Warnicke	       u32 byte_offset,
321cb9cadadSEd Warnicke	       u32 is_net_byte_order,
32271e97c63SCalvin	       u32 want_sum, u64 * sum_result, u64 v_min, u64 v_max)
323cb9cadadSEd Warnicke{
32464034367SDamjan Marion  vlib_main_t *vm = vlib_get_main ();
325cb9cadadSEd Warnicke  u64 v_diff = v_max - v_min + 1;
326cb9cadadSEd Warnicke  u64 r_mask = max_pow2 (v_diff) - 1;
327cb9cadadSEd Warnicke  u64 v0, v1;
328cb9cadadSEd Warnicke  u64 sum = 0;
32971e97c63SCalvin  void *random_data;
330cb9cadadSEd Warnicke
331cb9cadadSEd Warnicke  random_data = clib_random_buffer_get_data
332cb9cadadSEd Warnicke    (&vm->random_buffer, n_buffers * n_bits / BITS (u8));
333cb9cadadSEd Warnicke
334cb9cadadSEd Warnicke  v0 = v1 = v_min;
335cb9cadadSEd Warnicke
336cb9cadadSEd Warnicke  while (n_buffers >= 4)
337cb9cadadSEd Warnicke    {
33871e97c63SCalvin      vlib_buffer_t *b0, *b1, *b2, *b3;
33971e97c63SCalvin      void *a0, *a1;
34071e97c63SCalvin      u64 r0 = 0, r1 = 0;	/* warnings be gone */
341cb9cadadSEd Warnicke
342cb9cadadSEd Warnicke      b0 = vlib_get_buffer (vm, buffers[0]);
343cb9cadadSEd Warnicke      b1 = vlib_get_buffer (vm, buffers[1]);
344cb9cadadSEd Warnicke      b2 = vlib_get_buffer (vm, buffers[2]);
345cb9cadadSEd Warnicke      b3 = vlib_get_buffer (vm, buffers[3]);
346cb9cadadSEd Warnicke      buffers += 2;
347cb9cadadSEd Warnicke      n_buffers -= 2;
348cb9cadadSEd Warnicke
349cb9cadadSEd Warnicke      a0 = (void *) b0 + byte_offset;
350cb9cadadSEd Warnicke      a1 = (void *) b1 + byte_offset;
351cb9cadadSEd Warnicke      CLIB_PREFETCH ((void *) b2 + byte_offset, sizeof (v_min), WRITE);
352cb9cadadSEd Warnicke      CLIB_PREFETCH ((void *) b3 + byte_offset, sizeof (v_min), WRITE);
353cb9cadadSEd Warnicke
354cb9cadadSEd Warnicke      switch (n_bits)
355cb9cadadSEd Warnicke	{
356cb9cadadSEd Warnicke#define _(n)					\
357cb9cadadSEd Warnicke	  case BITS (u##n):			\
358cb9cadadSEd Warnicke	    {					\
359cb9cadadSEd Warnicke	      u##n * r = random_data;		\
360cb9cadadSEd Warnicke	      r0 = r[0];			\
361cb9cadadSEd Warnicke	      r1 = r[1];			\
362cb9cadadSEd Warnicke	      random_data = r + 2;		\
363cb9cadadSEd Warnicke	    }					\
364cb9cadadSEd Warnicke	  break;
365cb9cadadSEd Warnicke
36671e97c63SCalvin	  _(8);
36771e97c63SCalvin	  _(16);
36871e97c63SCalvin	  _(32);
36971e97c63SCalvin	  _(64);
370cb9cadadSEd Warnicke
371cb9cadadSEd Warnicke#undef _
372cb9cadadSEd Warnicke	}
373cb9cadadSEd Warnicke
374cb9cadadSEd Warnicke      /* Add power of 2 sized random number which may be out of range. */
375cb9cadadSEd Warnicke      v0 += r0 & r_mask;
376cb9cadadSEd Warnicke      v1 += r1 & r_mask;
377cb9cadadSEd Warnicke
378cb9cadadSEd Warnicke      /* Twice should be enough to reduce to v_min .. v_max range. */
379cb9cadadSEd Warnicke      v0 = v0 > v_max ? v0 - v_diff : v0;
380cb9cadadSEd Warnicke      v1 = v1 > v_max ? v1 - v_diff : v1;
381cb9cadadSEd Warnicke      v0 = v0 > v_max ? v0 - v_diff : v0;
382cb9cadadSEd Warnicke      v1 = v1 > v_max ? v1 - v_diff : v1;
383cb9cadadSEd Warnicke
384cb9cadadSEd Warnicke      if (want_sum)
385cb9cadadSEd Warnicke	sum += v0 + v1;
386cb9cadadSEd Warnicke
38771e97c63SCalvin      set_2 (a0, a1, v0, v1, v_min, v_max, n_bits, is_net_byte_order,
388cb9cadadSEd Warnicke	     /* is_increment */ 0);
389cb9cadadSEd Warnicke
390cb9cadadSEd Warnicke      ASSERT (validate_buffer_data (b0, s));
391cb9cadadSEd Warnicke      ASSERT (validate_buffer_data (b1, s));
392cb9cadadSEd Warnicke    }
393cb9cadadSEd Warnicke
394cb9cadadSEd Warnicke  while (n_buffers > 0)
395cb9cadadSEd Warnicke    {
39671e97c63SCalvin      vlib_buffer_t *b0;
39771e97c63SCalvin      void *a0;
39871e97c63SCalvin      u64 r0 = 0;		/* warnings be gone */
399cb9cadadSEd Warnicke
400cb9cadadSEd Warnicke      b0 = vlib_get_buffer (vm, buffers[0]);
401cb9cadadSEd Warnicke      buffers += 1;
402cb9cadadSEd Warnicke      n_buffers -= 1;
403cb9cadadSEd Warnicke
404cb9cadadSEd Warnicke      a0 = (void *) b0 + byte_offset;
405cb9cadadSEd Warnicke
406cb9cadadSEd Warnicke      switch (n_bits)
407cb9cadadSEd Warnicke	{
408cb9cadadSEd Warnicke#define _(n)					\
409cb9cadadSEd Warnicke	  case BITS (u##n):			\
410cb9cadadSEd Warnicke	    {					\
411cb9cadadSEd Warnicke	      u##n * r = random_data;		\
412cb9cadadSEd Warnicke	      r0 = r[0];			\
413cb9cadadSEd Warnicke	      random_data = r + 1;		\
414cb9cadadSEd Warnicke	    }					\
415cb9cadadSEd Warnicke	  break;
416cb9cadadSEd Warnicke
41771e97c63SCalvin	  _(8);
41871e97c63SCalvin	  _(16);
41971e97c63SCalvin	  _(32);
42071e97c63SCalvin	  _(64);
421cb9cadadSEd Warnicke
422cb9cadadSEd Warnicke#undef _
423cb9cadadSEd Warnicke	}
424cb9cadadSEd Warnicke
425cb9cadadSEd Warnicke      /* Add power of 2 sized random number which may be out of range. */
426cb9cadadSEd Warnicke      v0 += r0 & r_mask;
427cb9cadadSEd Warnicke
428cb9cadadSEd Warnicke      /* Twice should be enough to reduce to v_min .. v_max range. */
429cb9cadadSEd Warnicke      v0 = v0 > v_max ? v0 - v_diff : v0;
430cb9cadadSEd Warnicke      v0 = v0 > v_max ? v0 - v_diff : v0;
431cb9cadadSEd Warnicke
432cb9cadadSEd Warnicke      if (want_sum)
433cb9cadadSEd Warnicke	sum += v0;
434cb9cadadSEd Warnicke
435cb9cadadSEd Warnicke      set_1 (a0, v0, v_min, v_max, n_bits, is_net_byte_order);
436cb9cadadSEd Warnicke
437cb9cadadSEd Warnicke      ASSERT (validate_buffer_data (b0, s));
438cb9cadadSEd Warnicke    }
439cb9cadadSEd Warnicke
440cb9cadadSEd Warnicke  if (want_sum)
441cb9cadadSEd Warnicke    *sum_result = sum;
442cb9cadadSEd Warnicke}
443cb9cadadSEd Warnicke
444cb9cadadSEd Warnicke#define _(i,t)							\
445cb9cadadSEd Warnicke  clib_mem_unaligned (a##i, t) =				\
446cb9cadadSEd Warnicke    clib_host_to_net_##t ((clib_net_to_host_mem_##t (a##i) &~ mask)	\
447cb9cadadSEd Warnicke			  | (v##i << shift))
449cb9cadadSEd Warnickealways_inline void
45071e97c63SCalvinsetbits_1 (void *a0,
451cb9cadadSEd Warnicke	   u64 v0,
452cb9cadadSEd Warnicke	   u64 v_min, u64 v_max,
45371e97c63SCalvin	   u32 max_bits, u32 n_bits, u64 mask, u32 shift)
454cb9cadadSEd Warnicke{
455cb9cadadSEd Warnicke  ASSERT (v0 >= v_min && v0 <= v_max);
456cb9cadadSEd Warnicke  if (max_bits == BITS (u8))
45771e97c63SCalvin    ((u8 *) a0)[0] = (((u8 *) a0)[0] & ~mask) | (v0 << shift);
458cb9cadadSEd Warnicke
459cb9cadadSEd Warnicke  else if (max_bits == BITS (u16))
460cb9cadadSEd Warnicke    {
46171e97c63SCalvin      _(0, u16);
462cb9cadadSEd Warnicke    }
463cb9cadadSEd Warnicke  else if (max_bits == BITS (u32))
464cb9cadadSEd Warnicke    {
46571e97c63SCalvin      _(0, u32);
466cb9cadadSEd Warnicke    }
467cb9cadadSEd Warnicke  else if (max_bits == BITS (u64))
468cb9cadadSEd Warnicke    {
46971e97c63SCalvin      _(0, u64);
470cb9cadadSEd Warnicke    }
471cb9cadadSEd Warnicke}
472cb9cadadSEd Warnicke
473cb9cadadSEd Warnickealways_inline void
47471e97c63SCalvinsetbits_2 (void *a0, void *a1,
475cb9cadadSEd Warnicke	   u64 v0, u64 v1,
476cb9cadadSEd Warnicke	   u64 v_min, u64 v_max,
47771e97c63SCalvin	   u32 max_bits, u32 n_bits, u64 mask, u32 shift, u32 is_increment)
478cb9cadadSEd Warnicke{
479cb9cadadSEd Warnicke  ASSERT (v0 >= v_min && v0 <= v_max);
480cb9cadadSEd Warnicke  ASSERT (v1 >= v_min && v1 <= v_max + is_increment);
481cb9cadadSEd Warnicke  if (max_bits == BITS (u8))
482cb9cadadSEd Warnicke    {
48371e97c63SCalvin      ((u8 *) a0)[0] = (((u8 *) a0)[0] & ~mask) | (v0 << shift);
48471e97c63SCalvin      ((u8 *) a1)[0] = (((u8 *) a1)[0] & ~mask) | (v1 << shift);
485cb9cadadSEd Warnicke    }
486cb9cadadSEd Warnicke
487cb9cadadSEd Warnicke  else if (max_bits == BITS (u16))
488cb9cadadSEd Warnicke    {
48971e97c63SCalvin      _(0, u16);
49071e97c63SCalvin      _(1, u16);
491cb9cadadSEd Warnicke    }
492cb9cadadSEd Warnicke  else if (max_bits == BITS (u32))
493cb9cadadSEd Warnicke    {
49471e97c63SCalvin      _(0, u32);
49571e97c63SCalvin      _(1, u32);
496cb9cadadSEd Warnicke    }
497cb9cadadSEd Warnicke  else if (max_bits == BITS (u64))
498cb9cadadSEd Warnicke    {
49971e97c63SCalvin      _(0, u64);
50071e97c63SCalvin      _(1, u64);
501cb9cadadSEd Warnicke    }
502cb9cadadSEd Warnicke}
503cb9cadadSEd Warnicke
504cb9cadadSEd Warnicke#undef _
505cb9cadadSEd Warnicke
506cb9cadadSEd Warnickestatic_always_inline void
507cb9cadadSEd Warnickedo_setbits_fixed (pg_main_t * pg,
508cb9cadadSEd Warnicke		  pg_stream_t * s,
509cb9cadadSEd Warnicke		  u32 * buffers,
510cb9cadadSEd Warnicke		  u32 n_buffers,
511cb9cadadSEd Warnicke		  u32 max_bits,
512cb9cadadSEd Warnicke		  u32 n_bits,
51371e97c63SCalvin		  u32 byte_offset, u64 v_min, u64 v_max, u64 mask, u32 shift)
514cb9cadadSEd Warnicke{
51564034367SDamjan Marion  vlib_main_t *vm = vlib_get_main ();
516cb9cadadSEd Warnicke
517cb9cadadSEd Warnicke  while (n_buffers >= 4)
518cb9cadadSEd Warnicke    {
51971e97c63SCalvin      vlib_buffer_t *b0, *b1, *b2, *b3;
52071e97c63SCalvin      void *a0, *a1;
521cb9cadadSEd Warnicke
522cb9cadadSEd Warnicke      b0 = vlib_get_buffer (vm, buffers[0]);
523cb9cadadSEd Warnicke      b1 = vlib_get_buffer (vm, buffers[1]);
524cb9cadadSEd Warnicke      b2 = vlib_get_buffer (vm, buffers[2]);
525cb9cadadSEd Warnicke      b3 = vlib_get_buffer (vm, buffers[3]);
526cb9cadadSEd Warnicke      buffers += 2;
527cb9cadadSEd Warnicke      n_buffers -= 2;
528cb9cadadSEd Warnicke
529cb9cadadSEd Warnicke      a0 = (void *) b0 + byte_offset;
530cb9cadadSEd Warnicke      a1 = (void *) b1 + byte_offset;
531cb9cadadSEd Warnicke      CLIB_PREFETCH ((void *) b2 + byte_offset, sizeof (v_min), WRITE);
532cb9cadadSEd Warnicke      CLIB_PREFETCH ((void *) b3 + byte_offset, sizeof (v_min), WRITE);
533cb9cadadSEd Warnicke
534cb9cadadSEd Warnicke      setbits_2 (a0, a1,
53571e97c63SCalvin		 v_min, v_min, v_min, v_max, max_bits, n_bits, mask, shift,
536cb9cadadSEd Warnicke		 /* is_increment */ 0);
537cb9cadadSEd Warnicke
538cb9cadadSEd Warnicke      ASSERT (validate_buffer_data (b0, s));
539cb9cadadSEd Warnicke      ASSERT (validate_buffer_data (b1, s));
540cb9cadadSEd Warnicke    }
541cb9cadadSEd Warnicke
542cb9cadadSEd Warnicke  while (n_buffers > 0)
543cb9cadadSEd Warnicke    {
54471e97c63SCalvin      vlib_buffer_t *b0;
54571e97c63SCalvin      void *a0;
546cb9cadadSEd Warnicke
547cb9cadadSEd Warnicke      b0 = vlib_get_buffer (vm, buffers[0]);
548cb9cadadSEd Warnicke      buffers += 1;
549cb9cadadSEd Warnicke      n_buffers -= 1;
550cb9cadadSEd Warnicke
551cb9cadadSEd Warnicke      a0 = (void *) b0 + byte_offset;
552cb9cadadSEd Warnicke
553cb9cadadSEd Warnicke      setbits_1 (a0, v_min, v_min, v_max, max_bits, n_bits, mask, shift);
554cb9cadadSEd Warnicke      ASSERT (validate_buffer_data (b0, s));
555cb9cadadSEd Warnicke    }
556cb9cadadSEd Warnicke}
557cb9cadadSEd Warnicke
558cb9cadadSEd Warnickestatic_always_inline u64
559cb9cadadSEd Warnickedo_setbits_increment (pg_main_t * pg,
560cb9cadadSEd Warnicke		      pg_stream_t * s,
561cb9cadadSEd Warnicke		      u32 * buffers,
562cb9cadadSEd Warnicke		      u32 n_buffers,
563cb9cadadSEd Warnicke		      u32 max_bits,
564cb9cadadSEd Warnicke		      u32 n_bits,
565cb9cadadSEd Warnicke		      u32 byte_offset,
56671e97c63SCalvin		      u64 v_min, u64 v_max, u64 v, u64 mask, u32 shift)
567cb9cadadSEd Warnicke{
56864034367SDamjan Marion  vlib_main_t *vm = vlib_get_main ();
569cb9cadadSEd Warnicke
570cb9cadadSEd Warnicke  ASSERT (v >= v_min && v <= v_max);
571cb9cadadSEd Warnicke
572cb9cadadSEd Warnicke  while (n_buffers >= 4)
573cb9cadadSEd Warnicke    {
57471e97c63SCalvin      vlib_buffer_t *b0, *b1, *b2, *b3;
57571e97c63SCalvin      void *a0, *a1;
576cb9cadadSEd Warnicke      u64 v_old;
577cb9cadadSEd Warnicke
578cb9cadadSEd Warnicke      b0 = vlib_get_buffer (vm, buffers[0]);
579cb9cadadSEd Warnicke      b1 = vlib_get_buffer (vm, buffers[1]);
580cb9cadadSEd Warnicke      b2 = vlib_get_buffer (vm, buffers[2]);
581cb9cadadSEd Warnicke      b3 = vlib_get_buffer (vm, buffers[3]);
582cb9cadadSEd Warnicke      buffers += 2;
583cb9cadadSEd Warnicke      n_buffers -= 2;
584cb9cadadSEd Warnicke
585cb9cadadSEd Warnicke      a0 = (void *) b0 + byte_offset;
586cb9cadadSEd Warnicke      a1 = (void *) b1 + byte_offset;
587cb9cadadSEd Warnicke      CLIB_PREFETCH ((void *) b2 + byte_offset, sizeof (v_min), WRITE);
588cb9cadadSEd Warnicke      CLIB_PREFETCH ((void *) b3 + byte_offset, sizeof (v_min), WRITE);
589cb9cadadSEd Warnicke
590cb9cadadSEd Warnicke      v_old = v;
591cb9cadadSEd Warnicke      v = v_old + 2;
592cb9cadadSEd Warnicke      v = v > v_max ? v_min : v;
593cb9cadadSEd Warnicke      setbits_2 (a0, a1,
594cb9cadadSEd Warnicke		 v_old + 0, v_old + 1,
59571e97c63SCalvin		 v_min, v_max, max_bits, n_bits, mask, shift,
596cb9cadadSEd Warnicke		 /* is_increment */ 1);
597cb9cadadSEd Warnicke
598cb9cadadSEd Warnicke      if (PREDICT_FALSE (v_old + 1 > v_max))
599cb9cadadSEd Warnicke	{
600cb9cadadSEd Warnicke	  v = v_old;
601cb9cadadSEd Warnicke	  setbits_1 (a0, v + 0, v_min, v_max, max_bits, n_bits, mask, shift);
602cb9cadadSEd Warnicke	  v += 1;
603cb9cadadSEd Warnicke
604cb9cadadSEd Warnicke	  v = v > v_max ? v_min : v;
605cb9cadadSEd Warnicke	  setbits_1 (a1, v + 0, v_min, v_max, max_bits, n_bits, mask, shift);
606cb9cadadSEd Warnicke	  v += 1;
607cb9cadadSEd Warnicke	}
608cb9cadadSEd Warnicke      ASSERT (validate_buffer_data (b0, s));
609cb9cadadSEd Warnicke      ASSERT (validate_buffer_data (b1, s));
610cb9cadadSEd Warnicke    }
611cb9cadadSEd Warnicke
612cb9cadadSEd Warnicke  while (n_buffers > 0)
613cb9cadadSEd Warnicke    {
61471e97c63SCalvin      vlib_buffer_t *b0;
61571e97c63SCalvin      void *a0;
616cb9cadadSEd Warnicke      u64 v_old;
617cb9cadadSEd Warnicke
618cb9cadadSEd Warnicke      b0 = vlib_get_buffer (vm, buffers[0]);
619cb9cadadSEd Warnicke      buffers += 1;
620cb9cadadSEd Warnicke      n_buffers -= 1;
621cb9cadadSEd Warnicke
622cb9cadadSEd Warnicke      a0 = (void *) b0 + byte_offset;
623cb9cadadSEd Warnicke
624cb9cadadSEd Warnicke      v_old = v;
625cb9cadadSEd Warnicke      v = v_old + 1;
626cb9cadadSEd Warnicke      v = v > v_max ? v_min : v;
627cb9cadadSEd Warnicke
628cb9cadadSEd Warnicke      ASSERT (v_old >= v_min && v_old <= v_max);
629cb9cadadSEd Warnicke      setbits_1 (a0, v_old, v_min, v_max, max_bits, n_bits, mask, shift);
630cb9cadadSEd Warnicke
631cb9cadadSEd Warnicke      ASSERT (validate_buffer_data (b0, s));
632cb9cadadSEd Warnicke    }
633cb9cadadSEd Warnicke
634cb9cadadSEd Warnicke  return v;
635cb9cadadSEd Warnicke}
636cb9cadadSEd Warnicke
637cb9cadadSEd Warnickestatic_always_inline void
638cb9cadadSEd Warnickedo_setbits_random (pg_main_t * pg,
639cb9cadadSEd Warnicke		   pg_stream_t * s,
640cb9cadadSEd Warnicke		   u32 * buffers,
641cb9cadadSEd Warnicke		   u32 n_buffers,
642cb9cadadSEd Warnicke		   u32 max_bits,
643cb9cadadSEd Warnicke		   u32 n_bits,
64471e97c63SCalvin		   u32 byte_offset, u64 v_min, u64 v_max, u64 mask, u32 shift)
645cb9cadadSEd Warnicke{
64664034367SDamjan Marion  vlib_main_t *vm = vlib_get_main ();
647cb9cadadSEd Warnicke  u64 v_diff = v_max - v_min + 1;
648cb9cadadSEd Warnicke  u64 r_mask = max_pow2 (v_diff) - 1;
649cb9cadadSEd Warnicke  u64 v0, v1;
65071e97c63SCalvin  void *random_data;
651cb9cadadSEd Warnicke
652cb9cadadSEd Warnicke  random_data = clib_random_buffer_get_data
653cb9cadadSEd Warnicke    (&vm->random_buffer, n_buffers * max_bits / BITS (u8));
654cb9cadadSEd Warnicke  v0 = v1 = v_min;
655cb9cadadSEd Warnicke
656cb9cadadSEd Warnicke  while (n_buffers >= 4)
657cb9cadadSEd Warnicke    {
65871e97c63SCalvin      vlib_buffer_t *b0, *b1, *b2, *b3;
65971e97c63SCalvin      void *a0, *a1;
66071e97c63SCalvin      u64 r0 = 0, r1 = 0;	/* warnings be gone */
661cb9cadadSEd Warnicke
662cb9cadadSEd Warnicke      b0 = vlib_get_buffer (vm, buffers[0]);
663cb9cadadSEd Warnicke      b1 = vlib_get_buffer (vm, buffers[1]);
664cb9cadadSEd Warnicke      b2 = vlib_get_buffer (vm, buffers[2]);
665cb9cadadSEd Warnicke      b3 = vlib_get_buffer (vm, buffers[3]);
666cb9cadadSEd Warnicke      buffers += 2;
667cb9cadadSEd Warnicke      n_buffers -= 2;
668cb9cadadSEd Warnicke
669cb9cadadSEd Warnicke      a0 = (void *) b0 + byte_offset;
670cb9cadadSEd Warnicke      a1 = (void *) b1 + byte_offset;
671cb9cadadSEd Warnicke      CLIB_PREFETCH ((void *) b2 + byte_offset, sizeof (v_min), WRITE);
672cb9cadadSEd Warnicke      CLIB_PREFETCH ((void *) b3 + byte_offset, sizeof (v_min), WRITE);
673cb9cadadSEd Warnicke
674cb9cadadSEd Warnicke      switch (max_bits)
675cb9cadadSEd Warnicke	{
676cb9cadadSEd Warnicke#define _(n)					\
677cb9cadadSEd Warnicke	  case BITS (u##n):			\
678cb9cadadSEd Warnicke	    {					\
679cb9cadadSEd Warnicke	      u##n * r = random_data;		\
680cb9cadadSEd Warnicke	      r0 = r[0];			\
681cb9cadadSEd Warnicke	      r1 = r[1];			\
682cb9cadadSEd Warnicke	      random_data = r + 2;		\
683cb9cadadSEd Warnicke	    }					\
684cb9cadadSEd Warnicke	  break;
685cb9cadadSEd Warnicke
68671e97c63SCalvin	  _(8);
68771e97c63SCalvin	  _(16);
68871e97c63SCalvin	  _(32);
68971e97c63SCalvin	  _(64);
690cb9cadadSEd Warnicke
691cb9cadadSEd Warnicke#undef _
692cb9cadadSEd Warnicke	}
693cb9cadadSEd Warnicke
694cb9cadadSEd Warnicke      /* Add power of 2 sized random number which may be out of range. */
695cb9cadadSEd Warnicke      v0 += r0 & r_mask;
696cb9cadadSEd Warnicke      v1 += r1 & r_mask;
697cb9cadadSEd Warnicke
698cb9cadadSEd Warnicke      /* Twice should be enough to reduce to v_min .. v_max range. */
699cb9cadadSEd Warnicke      v0 = v0 > v_max ? v0 - v_diff : v0;
700cb9cadadSEd Warnicke      v1 = v1 > v_max ? v1 - v_diff : v1;
701cb9cadadSEd Warnicke      v0 = v0 > v_max ? v0 - v_diff : v0;
702cb9cadadSEd Warnicke      v1 = v1 > v_max ? v1 - v_diff : v1;
703cb9cadadSEd Warnicke
70471e97c63SCalvin      setbits_2 (a0, a1, v0, v1, v_min, v_max, max_bits, n_bits, mask, shift,
705cb9cadadSEd Warnicke		 /* is_increment */ 0);
706cb9cadadSEd Warnicke
707cb9cadadSEd Warnicke      ASSERT (validate_buffer_data (b0, s));
708cb9cadadSEd Warnicke      ASSERT (validate_buffer_data (b1, s));
709cb9cadadSEd Warnicke    }
710cb9cadadSEd Warnicke
711cb9cadadSEd Warnicke  while (n_buffers > 0)
712cb9cadadSEd Warnicke    {
71371e97c63SCalvin      vlib_buffer_t *b0;
71471e97c63SCalvin      void *a0;
71571e97c63SCalvin      u64 r0 = 0;		/* warnings be gone */
716cb9cadadSEd Warnicke
717cb9cadadSEd Warnicke      b0 = vlib_get_buffer (vm, buffers[0]);
718cb9cadadSEd Warnicke      buffers += 1;
719cb9cadadSEd Warnicke      n_buffers -= 1;
720cb9cadadSEd Warnicke
721cb9cadadSEd Warnicke      a0 = (void *) b0 + byte_offset;
722cb9cadadSEd Warnicke
723cb9cadadSEd Warnicke      switch (max_bits)
724cb9cadadSEd Warnicke	{
725cb9cadadSEd Warnicke#define _(n)					\
726cb9cadadSEd Warnicke	  case BITS (u##n):			\
727cb9cadadSEd Warnicke	    {					\
728cb9cadadSEd Warnicke	      u##n * r = random_data;		\
729cb9cadadSEd Warnicke	      r0 = r[0];			\
730cb9cadadSEd Warnicke	      random_data = r + 1;		\
731cb9cadadSEd Warnicke	    }					\
732cb9cadadSEd Warnicke	  break;
733cb9cadadSEd Warnicke
73471e97c63SCalvin	  _(8);
73571e97c63SCalvin	  _(16);
73671e97c63SCalvin	  _(32);
73771e97c63SCalvin	  _(64);
738cb9cadadSEd Warnicke
739cb9cadadSEd Warnicke#undef _
740cb9cadadSEd Warnicke	}
741cb9cadadSEd Warnicke
742cb9cadadSEd Warnicke      /* Add power of 2 sized random number which may be out of range. */
743cb9cadadSEd Warnicke      v0 += r0 & r_mask;
744cb9cadadSEd Warnicke
745cb9cadadSEd Warnicke      /* Twice should be enough to reduce to v_min .. v_max range. */
746cb9cadadSEd Warnicke      v0 = v0 > v_max ? v0 - v_diff : v0;
747cb9cadadSEd Warnicke      v0 = v0 > v_max ? v0 - v_diff : v0;
748cb9cadadSEd Warnicke
749cb9cadadSEd Warnicke      setbits_1 (a0, v0, v_min, v_max, max_bits, n_bits, mask, shift);
750cb9cadadSEd Warnicke
751cb9cadadSEd Warnicke      ASSERT (validate_buffer_data (b0, s));
752cb9cadadSEd Warnicke    }
753cb9cadadSEd Warnicke}
754cb9cadadSEd Warnicke
75571e97c63SCalvinstatic u64
75671e97c63SCalvindo_it (pg_main_t * pg,
75771e97c63SCalvin       pg_stream_t * s,
75871e97c63SCalvin       u32 * buffers,
75971e97c63SCalvin       u32 n_buffers,
76071e97c63SCalvin       u32 lo_bit, u32 hi_bit,
76171e97c63SCalvin       u64 v_min, u64 v_max, u64 v, pg_edit_type_t edit_type)
762cb9cadadSEd Warnicke{
763cb9cadadSEd Warnicke  u32 max_bits, l0, l1, h1, start_bit;
764cb9cadadSEd Warnicke
765cb9cadadSEd Warnicke  if (v_min == v_max)
766cb9cadadSEd Warnicke    edit_type = PG_EDIT_FIXED;
767cb9cadadSEd Warnicke
768cb9cadadSEd Warnicke  l0 = lo_bit / BITS (u8);
769cb9cadadSEd Warnicke  l1 = lo_bit % BITS (u8);
770cb9cadadSEd Warnicke  h1 = hi_bit % BITS (u8);
771cb9cadadSEd Warnicke
772cb9cadadSEd Warnicke  start_bit = l0 * BITS (u8);
773cb9cadadSEd Warnicke
774cb9cadadSEd Warnicke  max_bits = hi_bit - start_bit;
775cb9cadadSEd Warnicke  ASSERT (max_bits <= 64);
776cb9cadadSEd Warnicke
777cb9cadadSEd Warnicke#define _(n)						\
778cb9cadadSEd Warnicke  case (n):						\
779cb9cadadSEd Warnicke    if (edit_type == PG_EDIT_INCREMENT)			\
780cb9cadadSEd Warnicke      v = do_set_increment (pg, s, buffers, n_buffers,	\
781cb9cadadSEd Warnicke			    BITS (u##n),		\
782cb9cadadSEd Warnicke			    l0,				\
783cb9cadadSEd Warnicke			    /* is_net_byte_order */ 1,	\
784cb9cadadSEd Warnicke			    /* want sum */ 0, 0,	\
785cb9cadadSEd Warnicke			    v_min, v_max,		\
786cb9cadadSEd Warnicke			    v);				\
787cb9cadadSEd Warnicke    else if (edit_type == PG_EDIT_RANDOM)		\
788cb9cadadSEd Warnicke      do_set_random (pg, s, buffers, n_buffers,		\
789cb9cadadSEd Warnicke		     BITS (u##n),			\
790cb9cadadSEd Warnicke		     l0,				\
791cb9cadadSEd Warnicke		     /* is_net_byte_order */ 1,		\
792cb9cadadSEd Warnicke		     /* want sum */ 0, 0,		\
793cb9cadadSEd Warnicke		     v_min, v_max);			\
794cb9cadadSEd Warnicke    else /* edit_type == PG_EDIT_FIXED */		\
795cb9cadadSEd Warnicke      do_set_fixed (pg, s, buffers, n_buffers,		\
796cb9cadadSEd Warnicke		    BITS (u##n),			\
797cb9cadadSEd Warnicke		    l0,					\
798cb9cadadSEd Warnicke		    /* is_net_byte_order */ 1,		\
799cb9cadadSEd Warnicke		    v_min, v_max);			\
800cb9cadadSEd Warnicke  goto done;
801cb9cadadSEd Warnicke
802cb9cadadSEd Warnicke  if (l1 == 0 && h1 == 0)
803cb9cadadSEd Warnicke    {
804cb9cadadSEd Warnicke      switch (max_bits)
805cb9cadadSEd Warnicke	{
80671e97c63SCalvin	  _(8);
80771e97c63SCalvin	  _(16);
80871e97c63SCalvin	  _(32);
80971e97c63SCalvin	  _(64);
810cb9cadadSEd Warnicke	}
811cb9cadadSEd Warnicke    }
812cb9cadadSEd Warnicke
813cb9cadadSEd Warnicke#undef _
814cb9cadadSEd Warnicke
815cb9cadadSEd Warnicke  {
816cb9cadadSEd Warnicke    u64 mask;
817cb9cadadSEd Warnicke    u32 shift = l1;
81871e97c63SCalvin    u32 n_bits = max_bits;
819cb9cadadSEd Warnicke
820cb9cadadSEd Warnicke    max_bits = clib_max (max_pow2 (n_bits), 8);
821cb9cadadSEd Warnicke
822cb9cadadSEd Warnicke    mask = ((u64) 1 << (u64) n_bits) - 1;
823cb9cadadSEd Warnicke    mask &= ~(((u64) 1 << (u64) shift) - 1);
824cb9cadadSEd Warnicke
825cb9cadadSEd Warnicke    mask <<= max_bits - n_bits;
826cb9cadadSEd Warnicke    shift += max_bits - n_bits;
827cb9cadadSEd Warnicke
828cb9cadadSEd Warnicke    switch (max_bits)
829cb9cadadSEd Warnicke      {
830cb9cadadSEd Warnicke#define _(n)								\
831cb9cadadSEd Warnicke	case (n):							\
832cb9cadadSEd Warnicke	  if (edit_type == PG_EDIT_INCREMENT)				\
833cb9cadadSEd Warnicke	    v = do_setbits_increment (pg, s, buffers, n_buffers,	\
834cb9cadadSEd Warnicke				      BITS (u##n), n_bits,		\
835cb9cadadSEd Warnicke				      l0, v_min, v_max, v,		\
836cb9cadadSEd Warnicke				      mask, shift);			\
837cb9cadadSEd Warnicke	  else if (edit_type == PG_EDIT_RANDOM)				\
838cb9cadadSEd Warnicke	    do_setbits_random (pg, s, buffers, n_buffers,		\
839cb9cadadSEd Warnicke			       BITS (u##n), n_bits,			\
840cb9cadadSEd Warnicke			       l0, v_min, v_max,			\
841cb9cadadSEd Warnicke			       mask, shift);				\
842cb9cadadSEd Warnicke	  else /* edit_type == PG_EDIT_FIXED */				\
843cb9cadadSEd Warnicke	    do_setbits_fixed (pg, s, buffers, n_buffers,		\
844cb9cadadSEd Warnicke			      BITS (u##n), n_bits,			\
845cb9cadadSEd Warnicke			      l0, v_min, v_max,				\
846cb9cadadSEd Warnicke			      mask, shift);				\
847cb9cadadSEd Warnicke	goto done;
848cb9cadadSEd Warnicke
84971e97c63SCalvin	_(8);
85071e97c63SCalvin	_(16);
85171e97c63SCalvin	_(32);
85271e97c63SCalvin	_(64);
853cb9cadadSEd Warnicke
854cb9cadadSEd Warnicke#undef _
855cb9cadadSEd Warnicke      }
856cb9cadadSEd Warnicke  }
857cb9cadadSEd Warnicke
859cb9cadadSEd Warnicke  return v;
860cb9cadadSEd Warnicke}
861cb9cadadSEd Warnicke
862cb9cadadSEd Warnickestatic void
863cb9cadadSEd Warnickepg_generate_set_lengths (pg_main_t * pg,
86471e97c63SCalvin			 pg_stream_t * s, u32 * buffers, u32 n_buffers)
865cb9cadadSEd Warnicke{
866cb9cadadSEd Warnicke  u64 v_min, v_max, length_sum;
867cb9cadadSEd Warnicke  pg_edit_type_t edit_type;
868cb9cadadSEd Warnicke
869cb9cadadSEd Warnicke  v_min = s->min_packet_bytes;
870cb9cadadSEd Warnicke  v_max = s->max_packet_bytes;
871cb9cadadSEd Warnicke  edit_type = s->packet_size_edit_type;
872cb9cadadSEd Warnicke
873cb9cadadSEd Warnicke  if (edit_type == PG_EDIT_INCREMENT)
874cb9cadadSEd Warnicke    s->last_increment_packet_size
875cb9cadadSEd Warnicke      = do_set_increment (pg, s, buffers, n_buffers,
876cb9cadadSEd Warnicke			  8 * STRUCT_SIZE_OF (vlib_buffer_t, current_length),
877cb9cadadSEd Warnicke			  STRUCT_OFFSET_OF (vlib_buffer_t, current_length),
878cb9cadadSEd Warnicke			  /* is_net_byte_order */ 0,
879cb9cadadSEd Warnicke			  /* want sum */ 1, &length_sum,
88071e97c63SCalvin			  v_min, v_max, s->last_increment_packet_size);
881cb9cadadSEd Warnicke
882cb9cadadSEd Warnicke  else if (edit_type == PG_EDIT_RANDOM)
883cb9cadadSEd Warnicke    do_set_random (pg, s, buffers, n_buffers,
884cb9cadadSEd Warnicke		   8 * STRUCT_SIZE_OF (vlib_buffer_t, current_length),
885cb9cadadSEd Warnicke		   STRUCT_OFFSET_OF (vlib_buffer_t, current_length),
886cb9cadadSEd Warnicke		   /* is_net_byte_order */ 0,
887cb9cadadSEd Warnicke		   /* want sum */ 1, &length_sum,
888cb9cadadSEd Warnicke		   v_min, v_max);
889cb9cadadSEd Warnicke
89071e97c63SCalvin  else				/* edit_type == PG_EDIT_FIXED */
891cb9cadadSEd Warnicke    {
892cb9cadadSEd Warnicke      do_set_fixed (pg, s, buffers, n_buffers,
893cb9cadadSEd Warnicke		    8 * STRUCT_SIZE_OF (vlib_buffer_t, current_length),
894cb9cadadSEd Warnicke		    STRUCT_OFFSET_OF (vlib_buffer_t, current_length),
895cb9cadadSEd Warnicke		    /* is_net_byte_order */ 0,
896cb9cadadSEd Warnicke		    v_min, v_max);
897cb9cadadSEd Warnicke      length_sum = v_min * n_buffers;
898cb9cadadSEd Warnicke    }
899cb9cadadSEd Warnicke
900cb9cadadSEd Warnicke  {
90171e97c63SCalvin    vnet_main_t *vnm = vnet_get_main ();
90271e97c63SCalvin    vnet_interface_main_t *im = &vnm->interface_main;
90371e97c63SCalvin    vnet_sw_interface_t *si =
90471e97c63SCalvin      vnet_get_sw_interface (vnm, s->sw_if_index[VLIB_RX]);
905cb9cadadSEd Warnicke
906cb9cadadSEd Warnicke    vlib_increment_combined_counter (im->combined_sw_if_counters
907cb9cadadSEd Warnicke				     + VNET_INTERFACE_COUNTER_RX,
908586afd76SDamjan Marion				     vlib_get_thread_index (),
90971e97c63SCalvin				     si->sw_if_index, n_buffers, length_sum);
910cb9cadadSEd Warnicke  }
911cb9cadadSEd Warnicke
912cb9cadadSEd Warnicke}
913cb9cadadSEd Warnicke
914cb9cadadSEd Warnickestatic void
915cb9cadadSEd Warnickepg_generate_fix_multi_buffer_lengths (pg_main_t * pg,
916cb9cadadSEd Warnicke				      pg_stream_t * s,
91771e97c63SCalvin				      u32 * buffers, u32 n_buffers)
918cb9cadadSEd Warnicke{
91964034367SDamjan Marion  vlib_main_t *vm = vlib_get_main ();
92071e97c63SCalvin  pg_buffer_index_t *pbi;
921cb9cadadSEd Warnicke  uword n_bytes_left;
92271e97c63SCalvin  static u32 *unused_buffers = 0;
923cb9cadadSEd Warnicke
924cb9cadadSEd Warnicke  while (n_buffers > 0)
925cb9cadadSEd Warnicke    {
92671e97c63SCalvin      vlib_buffer_t *b;
927cb9cadadSEd Warnicke      u32 bi;
928cb9cadadSEd Warnicke
929cb9cadadSEd Warnicke      bi = buffers[0];
930cb9cadadSEd Warnicke      b = vlib_get_buffer (vm, bi);
931cb9cadadSEd Warnicke
932cb9cadadSEd Warnicke      /* Current length here is length of whole packet. */
933cb9cadadSEd Warnicke      n_bytes_left = b->current_length;
934cb9cadadSEd Warnicke
935cb9cadadSEd Warnicke      pbi = s->buffer_indices;
936cb9cadadSEd Warnicke      while (1)
937cb9cadadSEd Warnicke	{
938cb9cadadSEd Warnicke	  uword n = clib_min (n_bytes_left, s->buffer_bytes);
939cb9cadadSEd Warnicke
940cb9cadadSEd Warnicke	  b->current_length = n;
941cb9cadadSEd Warnicke	  n_bytes_left -= n;
942cb9cadadSEd Warnicke	  if (n_bytes_left > 0)
943cb9cadadSEd Warnicke	    b->flags |= VLIB_BUFFER_NEXT_PRESENT;
944cb9cadadSEd Warnicke	  else
945cb9cadadSEd Warnicke	    b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
946cb9cadadSEd Warnicke
947cb9cadadSEd Warnicke	  /* Return unused buffers to fifos. */
948cb9cadadSEd Warnicke	  if (n == 0)
949cb9cadadSEd Warnicke	    vec_add1 (unused_buffers, bi);
950cb9cadadSEd Warnicke
951cb9cadadSEd Warnicke	  pbi++;
952cb9cadadSEd Warnicke	  if (pbi >= vec_end (s->buffer_indices))
953cb9cadadSEd Warnicke	    break;
954cb9cadadSEd Warnicke
955cb9cadadSEd Warnicke	  bi = b->next_buffer;
956cb9cadadSEd Warnicke	  b = vlib_get_buffer (vm, bi);
957cb9cadadSEd Warnicke	}
958cb9cadadSEd Warnicke      ASSERT (n_bytes_left == 0);
959cb9cadadSEd Warnicke
960cb9cadadSEd Warnicke      buffers += 1;
961cb9cadadSEd Warnicke      n_buffers -= 1;
962cb9cadadSEd Warnicke    }
963cb9cadadSEd Warnicke
964cb9cadadSEd Warnicke  if (vec_len (unused_buffers) > 0)
965cb9cadadSEd Warnicke    {
96671e97c63SCalvin      vlib_buffer_free_no_next (vm, unused_buffers, vec_len (unused_buffers));
967cb9cadadSEd Warnicke      _vec_len (unused_buffers) = 0;
968cb9cadadSEd Warnicke    }
969cb9cadadSEd Warnicke}
970cb9cadadSEd Warnicke
971cb9cadadSEd Warnickestatic void
972cb9cadadSEd Warnickepg_generate_edit (pg_main_t * pg,
97371e97c63SCalvin		  pg_stream_t * s, u32 * buffers, u32 n_buffers)
974cb9cadadSEd Warnicke{
97571e97c63SCalvin  pg_edit_t *e;
976cb9cadadSEd Warnicke
977cb9cadadSEd Warnicke  vec_foreach (e, s->non_fixed_edits)
97871e97c63SCalvin  {
97971e97c63SCalvin    switch (e->type)
98071e97c63SCalvin      {
98171e97c63SCalvin      case PG_EDIT_RANDOM:
98271e97c63SCalvin      case PG_EDIT_INCREMENT:
983cb9cadadSEd Warnicke	{
98471e97c63SCalvin	  u32 lo_bit, hi_bit;
98571e97c63SCalvin	  u64 v_min, v_max;
986cb9cadadSEd Warnicke
98771e97c63SCalvin	  v_min = pg_edit_get_value (e, PG_EDIT_LO);
98871e97c63SCalvin	  v_max = pg_edit_get_value (e, PG_EDIT_HI);
989cb9cadadSEd Warnicke
99071e97c63SCalvin	  hi_bit = (BITS (u8) * STRUCT_OFFSET_OF (vlib_buffer_t, data)
99171e97c63SCalvin		    + BITS (u8) + e->lsb_bit_offset);
99271e97c63SCalvin	  lo_bit = hi_bit - e->n_bits;
99471e97c63SCalvin	  e->last_increment_value
99571e97c63SCalvin	    = do_it (pg, s, buffers, n_buffers, lo_bit, hi_bit, v_min, v_max,
99671e97c63SCalvin		     e->last_increment_value, e->type);
997cb9cadadSEd Warnicke	}
99871e97c63SCalvin	break;
100071e97c63SCalvin      case PG_EDIT_UNSPECIFIED:
100171e97c63SCalvin	break;
100371e97c63SCalvin      default:
100471e97c63SCalvin	/* Should not be any fixed edits left. */
100571e97c63SCalvin	ASSERT (0);
100671e97c63SCalvin	break;
100771e97c63SCalvin      }
100871e97c63SCalvin  }
1009cb9cadadSEd Warnicke
1010cb9cadadSEd Warnicke  /* Call any edit functions to e.g. completely IP lengths, checksums, ... */
1011cb9cadadSEd Warnicke  {
1012cb9cadadSEd Warnicke    int i;
1013cb9cadadSEd Warnicke    for (i = vec_len (s->edit_groups) - 1; i >= 0; i--)
1014cb9cadadSEd Warnicke      {
101571e97c63SCalvin	pg_edit_group_t *g = s->edit_groups + i;
1016cb9cadadSEd Warnicke	if (g->edit_function)
1017cb9cadadSEd Warnicke	  g->edit_function (pg, s, g, buffers, n_buffers);
1018cb9cadadSEd Warnicke      }
1019cb9cadadSEd Warnicke  }
1020cb9cadadSEd Warnicke}
1021cb9cadadSEd Warnicke
1022cb9cadadSEd Warnickestatic void
1023cb9cadadSEd Warnickepg_set_next_buffer_pointers (pg_main_t * pg,
1024cb9cadadSEd Warnicke			     pg_stream_t * s,
102571e97c63SCalvin			     u32 * buffers, u32 * next_buffers, u32 n_buffers)
1026cb9cadadSEd Warnicke{
102764034367SDamjan Marion  vlib_main_t *vm = vlib_get_main ();
1028cb9cadadSEd Warnicke
1029cb9cadadSEd Warnicke  while (n_buffers >= 4)
1030cb9cadadSEd Warnicke    {
1031cb9cadadSEd Warnicke      u32 ni0, ni1;
103271e97c63SCalvin      vlib_buffer_t *b0, *b1;
1033cb9cadadSEd Warnicke
1034cb9cadadSEd Warnicke      b0 = vlib_get_buffer (vm, buffers[0]);
1035cb9cadadSEd Warnicke      b1 = vlib_get_buffer (vm, buffers[1]);
1036cb9cadadSEd Warnicke      ni0 = next_buffers[0];
1037cb9cadadSEd Warnicke      ni1 = next_buffers[1];
1038cb9cadadSEd Warnicke
1039cb9cadadSEd Warnicke      vlib_prefetch_buffer_with_index (vm, buffers[2], WRITE);
1040cb9cadadSEd Warnicke      vlib_prefetch_buffer_with_index (vm, buffers[3], WRITE);
1041cb9cadadSEd Warnicke
1042cb9cadadSEd Warnicke      b0->flags |= VLIB_BUFFER_NEXT_PRESENT;
1043cb9cadadSEd Warnicke      b1->flags |= VLIB_BUFFER_NEXT_PRESENT;
1044cb9cadadSEd Warnicke      b0->next_buffer = ni0;
1045cb9cadadSEd Warnicke      b1->next_buffer = ni1;
1046cb9cadadSEd Warnicke
1047cb9cadadSEd Warnicke      buffers += 2;
1048cb9cadadSEd Warnicke      next_buffers += 2;
1049cb9cadadSEd Warnicke      n_buffers -= 2;
1050cb9cadadSEd Warnicke    }
1051cb9cadadSEd Warnicke
1052cb9cadadSEd Warnicke  while (n_buffers > 0)
1053cb9cadadSEd Warnicke    {
1054cb9cadadSEd Warnicke      u32 ni0;
105571e97c63SCalvin      vlib_buffer_t *b0;
1056cb9cadadSEd Warnicke
1057cb9cadadSEd Warnicke      b0 = vlib_get_buffer (vm, buffers[0]);
1058cb9cadadSEd Warnicke      ni0 = next_buffers[0];
1059cb9cadadSEd Warnicke      buffers += 1;
1060cb9cadadSEd Warnicke      next_buffers += 1;
1061cb9cadadSEd Warnicke      n_buffers -= 1;
1062cb9cadadSEd Warnicke
1063cb9cadadSEd Warnicke      b0->flags |= VLIB_BUFFER_NEXT_PRESENT;
1064cb9cadadSEd Warnicke      b0->next_buffer = ni0;
1065cb9cadadSEd Warnicke    }
1066cb9cadadSEd Warnicke}
1067cb9cadadSEd Warnicke
1068cb9cadadSEd Warnickestatic_always_inline void
1069cb9cadadSEd Warnickeinit_buffers_inline (vlib_main_t * vm,
1070cb9cadadSEd Warnicke		     pg_stream_t * s,
1071cb9cadadSEd Warnicke		     u32 * buffers,
107271e97c63SCalvin		     u32 n_buffers, u32 data_offset, u32 n_data, u32 set_data)
1073cb9cadadSEd Warnicke{
107471e97c63SCalvin  u32 n_left, *b;
107571e97c63SCalvin  u8 *data, *mask;
1076cb9cadadSEd Warnicke
10773c8e1468SDave Barach  ASSERT (s->replay_packet_templates == 0);
1078cb9cadadSEd Warnicke
1079cb9cadadSEd Warnicke  data = s->fixed_packet_data + data_offset;
1080cb9cadadSEd Warnicke  mask = s->fixed_packet_data_mask + data_offset;
1081cb9cadadSEd Warnicke  if (data + n_data >= vec_end (s->fixed_packet_data))
1082cb9cadadSEd Warnicke    n_data = (data < vec_end (s->fixed_packet_data)
108371e97c63SCalvin	      ? vec_end (s->fixed_packet_data) - data : 0);
1084cb9cadadSEd Warnicke  if (n_data > 0)
1085cb9cadadSEd Warnicke    {
1086cb9cadadSEd Warnicke      ASSERT (data + n_data <= vec_end (s->fixed_packet_data));
1087cb9cadadSEd Warnicke      ASSERT (mask + n_data <= vec_end (s->fixed_packet_data_mask));
1088cb9cadadSEd Warnicke    }
1089cb9cadadSEd Warnicke
1090cb9cadadSEd Warnicke  n_left = n_buffers;
1091cb9cadadSEd Warnicke  b = buffers;
1092cb9cadadSEd Warnicke
1093cb9cadadSEd Warnicke  while (n_left >= 4)
1094cb9cadadSEd Warnicke    {
1095cb9cadadSEd Warnicke      u32 bi0, bi1;
109671e97c63SCalvin      vlib_buffer_t *b0, *b1;
1097cb9cadadSEd Warnicke
1098cb9cadadSEd Warnicke      /* Prefetch next iteration. */
1099cb9cadadSEd Warnicke      vlib_prefetch_buffer_with_index (vm, b[2], STORE);
1100cb9cadadSEd Warnicke      vlib_prefetch_buffer_with_index (vm, b[3], STORE);
1101cb9cadadSEd Warnicke
1102cb9cadadSEd Warnicke      bi0 = b[0];
1103cb9cadadSEd Warnicke      bi1 = b[1];
1104cb9cadadSEd Warnicke      b += 2;
1105cb9cadadSEd Warnicke      n_left -= 2;
1106cb9cadadSEd Warnicke
1107cb9cadadSEd Warnicke      b0 = vlib_get_buffer (vm, bi0);
1108cb9cadadSEd Warnicke      b1 = vlib_get_buffer (vm, bi1);
110908eb2bb2SDave Barach      b0->flags |= s->buffer_flags;
111008eb2bb2SDave Barach      b1->flags |= s->buffer_flags;
1111cb9cadadSEd Warnicke
1112cb9cadadSEd Warnicke      vnet_buffer (b0)->sw_if_index[VLIB_RX] =
1113cb9cadadSEd Warnicke	vnet_buffer (b1)->sw_if_index[VLIB_RX] = s->sw_if_index[VLIB_RX];
1114cb9cadadSEd Warnicke
1115cb9cadadSEd Warnicke      vnet_buffer (b0)->sw_if_index[VLIB_TX] =
11167d31ab2aSDave Barach	vnet_buffer (b1)->sw_if_index[VLIB_TX] = s->sw_if_index[VLIB_TX];
1117cb9cadadSEd Warnicke
1118cb9cadadSEd Warnicke      if (set_data)
1119cb9cadadSEd Warnicke	{
1120178cf493SDave Barach	  clib_memcpy_fast (b0->data, data, n_data);
1121178cf493SDave Barach	  clib_memcpy_fast (b1->data, data, n_data);
1122cb9cadadSEd Warnicke	}
1123cb9cadadSEd Warnicke      else
1124cb9cadadSEd Warnicke	{
1125cb9cadadSEd Warnicke	  ASSERT (validate_buffer_data2 (b0, s, data_offset, n_data));
1126cb9cadadSEd Warnicke	  ASSERT (validate_buffer_data2 (b1, s, data_offset, n_data));
1127cb9cadadSEd Warnicke	}
1128cb9cadadSEd Warnicke    }
1129cb9cadadSEd Warnicke
1130cb9cadadSEd Warnicke  while (n_left >= 1)
1131cb9cadadSEd Warnicke    {
1132cb9cadadSEd Warnicke      u32 bi0;
113371e97c63SCalvin      vlib_buffer_t *b0;
1134cb9cadadSEd Warnicke
1135cb9cadadSEd Warnicke      bi0 = b[0];
1136cb9cadadSEd Warnicke      b += 1;
1137cb9cadadSEd Warnicke      n_left -= 1;
1138cb9cadadSEd Warnicke
1139cb9cadadSEd Warnicke      b0 = vlib_get_buffer (vm, bi0);
114008eb2bb2SDave Barach      b0->flags |= s->buffer_flags;
1141cb9cadadSEd Warnicke      vnet_buffer (b0)->sw_if_index[VLIB_RX] = s->sw_if_index[VLIB_RX];
11427d31ab2aSDave Barach      vnet_buffer (b0)->sw_if_index[VLIB_TX] = s->sw_if_index[VLIB_TX];
1143cb9cadadSEd Warnicke
1144cb9cadadSEd Warnicke      if (set_data)
1145178cf493SDave Barach	clib_memcpy_fast (b0->data, data, n_data);
1146cb9cadadSEd Warnicke      else
1147cb9cadadSEd Warnicke	ASSERT (validate_buffer_data2 (b0, s, data_offset, n_data));
1148cb9cadadSEd Warnicke    }
1149cb9cadadSEd Warnicke}
1150cb9cadadSEd Warnicke
1151cb9cadadSEd Warnickestatic u32
1152cb9cadadSEd Warnickepg_stream_fill_helper (pg_main_t * pg,
1153cb9cadadSEd Warnicke		       pg_stream_t * s,
1154cb9cadadSEd Warnicke		       pg_buffer_index_t * bi,
115571e97c63SCalvin		       u32 * buffers, u32 * next_buffers, u32 n_alloc)
1156cb9cadadSEd Warnicke{
115764034367SDamjan Marion  vlib_main_t *vm = vlib_get_main ();
1158cb9cadadSEd Warnicke  uword is_start_of_packet = bi == s->buffer_indices;
1159cb9cadadSEd Warnicke  u32 n_allocated;
1160cb9cadadSEd Warnicke
11613c8e1468SDave Barach  ASSERT (vec_len (s->replay_packet_templates) == 0);
11623c8e1468SDave Barach
1163671e60e6SDamjan Marion  n_allocated = vlib_buffer_alloc (vm, buffers, n_alloc);
1164cb9cadadSEd Warnicke  if (n_allocated == 0)
1165cb9cadadSEd Warnicke    return 0;
1166cb9cadadSEd Warnicke
116771e97c63SCalvin  /*
116871e97c63SCalvin   * We can't assume we got all the buffers we asked for...
1169cb9cadadSEd Warnicke   * This never worked until recently.
1170cb9cadadSEd Warnicke   */
1171cb9cadadSEd Warnicke  n_alloc = n_allocated;
1172cb9cadadSEd Warnicke
1173cb9cadadSEd Warnicke  /* Reinitialize buffers */
1174ef2e5845SDamjan Marion  init_buffers_inline
1175ef2e5845SDamjan Marion    (vm, s,
1176ef2e5845SDamjan Marion     buffers,
1177ef2e5845SDamjan Marion     n_alloc, (bi - s->buffer_indices) * s->buffer_bytes /* data offset */ ,
1178ef2e5845SDamjan Marion     s->buffer_bytes,
1179ef2e5845SDamjan Marion     /* set_data */ 1);
1181cb9cadadSEd Warnicke  if (next_buffers)
1182cb9cadadSEd Warnicke    pg_set_next_buffer_pointers (pg, s, buffers, next_buffers, n_alloc);
1183cb9cadadSEd Warnicke
1184cb9cadadSEd Warnicke  if (is_start_of_packet)
1185cb9cadadSEd Warnicke    {
11863c8e1468SDave Barach      pg_generate_set_lengths (pg, s, buffers, n_alloc);
11873c8e1468SDave Barach      if (vec_len (s->buffer_indices) > 1)
11883c8e1468SDave Barach	pg_generate_fix_multi_buffer_lengths (pg, s, buffers, n_alloc);
11893c8e1468SDave Barach
11903c8e1468SDave Barach      pg_generate_edit (pg, s, buffers, n_alloc);
11913c8e1468SDave Barach    }
11923c8e1468SDave Barach
11933c8e1468SDave Barach  return n_alloc;
11943c8e1468SDave Barach}
11953c8e1468SDave Barach
11963c8e1468SDave Barachstatic u32
11973c8e1468SDave Barachpg_stream_fill_replay (pg_main_t * pg, pg_stream_t * s, u32 n_alloc)
11983c8e1468SDave Barach{
11993c8e1468SDave Barach  pg_buffer_index_t *bi;
12003c8e1468SDave Barach  u32 n_left, i, l;
12013c8e1468SDave Barach  u32 buffer_alloc_request = 0;
12023c8e1468SDave Barach  u32 buffer_alloc_result;
12033c8e1468SDave Barach  u32 current_buffer_index;
12043c8e1468SDave Barach  u32 *buffers;
12053c8e1468SDave Barach  vlib_main_t *vm = vlib_get_main ();
12063c8e1468SDave Barach  vnet_main_t *vnm = vnet_get_main ();
12078934a045SDamjan Marion  u32 buf_sz = vlib_buffer_get_default_data_size (vm);
12083c8e1468SDave Barach  vnet_interface_main_t *im = &vnm->interface_main;
12093c8e1468SDave Barach  vnet_sw_interface_t *si;
12103c8e1468SDave Barach
12113c8e1468SDave Barach  buffers = pg->replay_buffers_by_thread[vm->thread_index];
12123c8e1468SDave Barach  vec_reset_length (buffers);
12133c8e1468SDave Barach  bi = s->buffer_indices;
12143c8e1468SDave Barach
12153c8e1468SDave Barach  n_left = n_alloc;
12163c8e1468SDave Barach  i = s->current_replay_packet_index;
12173c8e1468SDave Barach  l = vec_len (s->replay_packet_templates);
12183c8e1468SDave Barach
12193c8e1468SDave Barach  /* Figure out how many buffers we need */
12203c8e1468SDave Barach  while (n_left > 0)
12213c8e1468SDave Barach    {
12223c8e1468SDave Barach      u8 *d0;
12233c8e1468SDave Barach
12243c8e1468SDave Barach      d0 = vec_elt (s->replay_packet_templates, i);
12255de3fec5SDamjan Marion      buffer_alloc_request += (vec_len (d0) + (buf_sz - 1)) / buf_sz;
12263c8e1468SDave Barach
12273c8e1468SDave Barach      i = ((i + 1) == l) ? 0 : i + 1;
12283c8e1468SDave Barach      n_left--;
12293c8e1468SDave Barach    }
12303c8e1468SDave Barach
12313c8e1468SDave Barach  ASSERT (buffer_alloc_request > 0);
12323c8e1468SDave Barach  vec_validate (buffers, buffer_alloc_request - 1);
12333c8e1468SDave Barach
12343c8e1468SDave Barach  /* Allocate that many buffers */
12353c8e1468SDave Barach  buffer_alloc_result = vlib_buffer_alloc (vm, buffers, buffer_alloc_request);
12363c8e1468SDave Barach  if (buffer_alloc_result < buffer_alloc_request)
12373c8e1468SDave Barach    {
12383c8e1468SDave Barach      clib_warning ("alloc failure, got %d not %d", buffer_alloc_result,
12393c8e1468SDave Barach		    buffer_alloc_request);
12403c8e1468SDave Barach      vlib_buffer_free_no_next (vm, buffers, buffer_alloc_result);
12413c8e1468SDave Barach      pg->replay_buffers_by_thread[vm->thread_index] = buffers;
12423c8e1468SDave Barach      return 0;
12433c8e1468SDave Barach    }
12443c8e1468SDave Barach
12453c8e1468SDave Barach  /* Now go generate the buffers, and add them to the FIFO */
12463c8e1468SDave Barach  n_left = n_alloc;
12473c8e1468SDave Barach
12483c8e1468SDave Barach  current_buffer_index = 0;
12493c8e1468SDave Barach  i = s->current_replay_packet_index;
12503c8e1468SDave Barach  l = vec_len (s->replay_packet_templates);
12513c8e1468SDave Barach  while (n_left > 0)
12523c8e1468SDave Barach    {
12533c8e1468SDave Barach      u8 *d0;
12543c8e1468SDave Barach      int not_last;
12553c8e1468SDave Barach      u32 data_offset;
12563c8e1468SDave Barach      u32 bytes_to_copy, bytes_this_chunk;
12573c8e1468SDave Barach      vlib_buffer_t *b;
12583c8e1468SDave Barach
12593c8e1468SDave Barach      d0 = vec_elt (s->replay_packet_templates, i);
12603c8e1468SDave Barach      data_offset = 0;
12613c8e1468SDave Barach      bytes_to_copy = vec_len (d0);
12623c8e1468SDave Barach
12633c8e1468SDave Barach      /* Add head chunk to pg fifo */
12643c8e1468SDave Barach      clib_fifo_add1 (bi->buffer_fifo, buffers[current_buffer_index]);
12653c8e1468SDave Barach
12663c8e1468SDave Barach      /* Copy the data */
12673c8e1468SDave Barach      while (bytes_to_copy)
1268cb9cadadSEd Warnicke	{
12695de3fec5SDamjan Marion	  bytes_this_chunk = clib_min (bytes_to_copy, buf_sz);
12703c8e1468SDave Barach	  ASSERT (current_buffer_index < vec_len (buffers));
12713c8e1468SDave Barach	  b = vlib_get_buffer (vm, buffers[current_buffer_index]);
12723c8e1468SDave Barach	  clib_memcpy_fast (b->data, d0 + data_offset, bytes_this_chunk);
12733c8e1468SDave Barach	  vnet_buffer (b)->sw_if_index[VLIB_RX] = s->sw_if_index[VLIB_RX];
12747d31ab2aSDave Barach	  vnet_buffer (b)->sw_if_index[VLIB_TX] = s->sw_if_index[VLIB_TX];
127508eb2bb2SDave Barach	  b->flags = s->buffer_flags;
12763c8e1468SDave Barach	  b->next_buffer = 0;
12773c8e1468SDave Barach	  b->current_data = 0;
12783c8e1468SDave Barach	  b->current_length = bytes_this_chunk;
12793c8e1468SDave Barach
12803c8e1468SDave Barach	  not_last = bytes_this_chunk < bytes_to_copy;
12813c8e1468SDave Barach	  if (not_last)
12823c8e1468SDave Barach	    {
12833c8e1468SDave Barach	      ASSERT (current_buffer_index < (vec_len (buffers) - 1));
12843c8e1468SDave Barach	      b->flags |= VLIB_BUFFER_NEXT_PRESENT;
12853c8e1468SDave Barach	      b->next_buffer = buffers[current_buffer_index + 1];
12863c8e1468SDave Barach	    }
12873c8e1468SDave Barach	  bytes_to_copy -= bytes_this_chunk;
12883c8e1468SDave Barach	  data_offset += bytes_this_chunk;
12893c8e1468SDave Barach	  current_buffer_index++;
1290cb9cadadSEd Warnicke	}
1291cb9cadadSEd Warnicke
12923c8e1468SDave Barach      i = ((i + 1) == l) ? 0 : i + 1;
12933c8e1468SDave Barach      n_left--;
1294cb9cadadSEd Warnicke    }
1295cb9cadadSEd Warnicke
12963c8e1468SDave Barach  /* Update the interface counters */
12973c8e1468SDave Barach  si = vnet_get_sw_interface (vnm, s->sw_if_index[VLIB_RX]);
12983c8e1468SDave Barach  l = 0;
12993c8e1468SDave Barach  for (i = 0; i < n_alloc; i++)
13003c8e1468SDave Barach    l += vlib_buffer_index_length_in_chain (vm, buffers[i]);
13013c8e1468SDave Barach  vlib_increment_combined_counter (im->combined_sw_if_counters
13023c8e1468SDave Barach				   + VNET_INTERFACE_COUNTER_RX,
13033c8e1468SDave Barach				   vlib_get_thread_index (),
13043c8e1468SDave Barach				   si->sw_if_index, n_alloc, l);
13053c8e1468SDave Barach
13063c8e1468SDave Barach  s->current_replay_packet_index += n_alloc;
13073c8e1468SDave Barach  s->current_replay_packet_index %= vec_len (s->replay_packet_templates);
13083c8e1468SDave Barach
13093c8e1468SDave Barach  pg->replay_buffers_by_thread[vm->thread_index] = buffers;
1310cb9cadadSEd Warnicke  return n_alloc;
1311cb9cadadSEd Warnicke}
1312cb9cadadSEd Warnicke
13133c8e1468SDave Barach
1314cb9cadadSEd Warnickestatic u32
1315cb9cadadSEd Warnickepg_stream_fill (pg_main_t * pg, pg_stream_t * s, u32 n_buffers)
1316cb9cadadSEd Warnicke{
131771e97c63SCalvin  pg_buffer_index_t *bi;
1318cb9cadadSEd Warnicke  word i, n_in_fifo, n_alloc, n_free, n_added;
131971e97c63SCalvin  u32 *tail, *start, *end, *last_tail, *last_start;
1320cb9cadadSEd Warnicke
1321cb9cadadSEd Warnicke  bi = s->buffer_indices;
1322cb9cadadSEd Warnicke
1323cb9cadadSEd Warnicke  n_in_fifo = clib_fifo_elts (bi->buffer_fifo);
1324cb9cadadSEd Warnicke  if (n_in_fifo >= n_buffers)
1325cb9cadadSEd Warnicke    return n_in_fifo;
1326cb9cadadSEd Warnicke
1327cb9cadadSEd Warnicke  n_alloc = n_buffers - n_in_fifo;
1328cb9cadadSEd Warnicke
1329cb9cadadSEd Warnicke  /* Round up, but never generate more than limit. */
1330cb9cadadSEd Warnicke  n_alloc = clib_max (VLIB_FRAME_SIZE, n_alloc);
1331cb9cadadSEd Warnicke
1332cb9cadadSEd Warnicke  if (s->n_packets_limit > 0
1333cb9cadadSEd Warnicke      && s->n_packets_generated + n_in_fifo + n_alloc >= s->n_packets_limit)
1334cb9cadadSEd Warnicke    {
1335cb9cadadSEd Warnicke      n_alloc = s->n_packets_limit - s->n_packets_generated - n_in_fifo;
1336cb9cadadSEd Warnicke      if (n_alloc < 0)
1337cb9cadadSEd Warnicke	n_alloc = 0;
1338cb9cadadSEd Warnicke    }
1339cb9cadadSEd Warnicke
13403c8e1468SDave Barach  /*
13413c8e1468SDave Barach   * Handle pcap replay directly
13423c8e1468SDave Barach   */
13433c8e1468SDave Barach  if (s->replay_packet_templates)
13443c8e1468SDave Barach    return pg_stream_fill_replay (pg, s, n_alloc);
13453c8e1468SDave Barach
1346cb9cadadSEd Warnicke  /* All buffer fifos should have the same size. */
1347cb9cadadSEd Warnicke  if (CLIB_DEBUG > 0)
1348cb9cadadSEd Warnicke    {
1349cb9cadadSEd Warnicke      uword l = ~0, e;
1350cb9cadadSEd Warnicke      vec_foreach (bi, s->buffer_indices)
135171e97c63SCalvin      {
135271e97c63SCalvin	e = clib_fifo_elts (bi->buffer_fifo);
135371e97c63SCalvin	if (bi == s->buffer_indices)
135471e97c63SCalvin	  l = e;
135571e97c63SCalvin	ASSERT (l == e);
135671e97c63SCalvin      }
1357cb9cadadSEd Warnicke    }
1358cb9cadadSEd Warnicke
1359cb9cadadSEd Warnicke  last_tail = last_start = 0;
1360cb9cadadSEd Warnicke  n_added = n_alloc;
1361cb9cadadSEd Warnicke
1362cb9cadadSEd Warnicke  for (i = vec_len (s->buffer_indices) - 1; i >= 0; i--)
1363cb9cadadSEd Warnicke    {
1364cb9cadadSEd Warnicke      bi = vec_elt_at_index (s->buffer_indices, i);
1365cb9cadadSEd Warnicke
1366cb9cadadSEd Warnicke      n_free = clib_fifo_free_elts (bi->buffer_fifo);
1367cb9cadadSEd Warnicke      if (n_free < n_alloc)
1368cb9cadadSEd Warnicke	clib_fifo_resize (bi->buffer_fifo, n_alloc - n_free);
1369cb9cadadSEd Warnicke
1370cb9cadadSEd Warnicke      tail = clib_fifo_advance_tail (bi->buffer_fifo, n_alloc);
1371cb9cadadSEd Warnicke      start = bi->buffer_fifo;
1372cb9cadadSEd Warnicke      end = clib_fifo_end (bi->buffer_fifo);
1373cb9cadadSEd Warnicke
1374cb9cadadSEd Warnicke      if (tail + n_alloc <= end)
137571e97c63SCalvin	{
137671e97c63SCalvin	  n_added =
137771e97c63SCalvin	    pg_stream_fill_helper (pg, s, bi, tail, last_tail, n_alloc);
137871e97c63SCalvin	}
1379cb9cadadSEd Warnicke      else
1380cb9cadadSEd Warnicke	{
1381cb9cadadSEd Warnicke	  u32 n = clib_min (end - tail, n_alloc);
1382cb9cadadSEd Warnicke	  n_added = pg_stream_fill_helper (pg, s, bi, tail, last_tail, n);
1383cb9cadadSEd Warnicke
1384cb9cadadSEd Warnicke	  if (n_added == n && n_alloc > n_added)
138571e97c63SCalvin	    {
138671e97c63SCalvin	      n_added += pg_stream_fill_helper
138771e97c63SCalvin		(pg, s, bi, start, last_start, n_alloc - n_added);
138871e97c63SCalvin	    }
1389cb9cadadSEd Warnicke	}
1390cb9cadadSEd Warnicke
1391cb9cadadSEd Warnicke      if (PREDICT_FALSE (n_added < n_alloc))
1392cb9cadadSEd Warnicke	tail = clib_fifo_advance_tail (bi->buffer_fifo, n_added - n_alloc);
1393cb9cadadSEd Warnicke
1394cb9cadadSEd Warnicke      last_tail = tail;
1395cb9cadadSEd Warnicke      last_start = start;
1396cb9cadadSEd Warnicke
1397cb9cadadSEd Warnicke      /* Verify that pkts in the fifo are properly allocated */
1398cb9cadadSEd Warnicke    }
1400cb9cadadSEd Warnicke  return n_in_fifo + n_added;
1401cb9cadadSEd Warnicke}
1402cb9cadadSEd Warnicke
140371e97c63SCalvintypedef struct
1405cb9cadadSEd Warnicke  u32 stream_index;
1406cb9cadadSEd Warnicke
1407cb9cadadSEd Warnicke  u32 packet_length;
14083466c302SNeale Ranns  u32 sw_if_index;
1409cb9cadadSEd Warnicke
1410cb9cadadSEd Warnicke  /* Use pre data for packet data. */
1411cb9cadadSEd Warnicke  vlib_buffer_t buffer;
1412cb9cadadSEd Warnicke} pg_input_trace_t;
1413cb9cadadSEd Warnicke
141471e97c63SCalvinstatic u8 *
141571e97c63SCalvinformat_pg_input_trace (u8 * s, va_list * va)
1416cb9cadadSEd Warnicke{
141771e97c63SCalvin  vlib_main_t *vm = va_arg (*va, vlib_main_t *);
1418cb9cadadSEd Warnicke  CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
141971e97c63SCalvin  pg_input_trace_t *t = va_arg (*va, pg_input_trace_t *);
142071e97c63SCalvin  pg_main_t *pg = &pg_main;
142171e97c63SCalvin  pg_stream_t *stream;
142271e97c63SCalvin  vlib_node_t *n;
1423d3c008d1SChristophe Fontaine  u32 indent = format_get_indent (s);
1424cb9cadadSEd Warnicke
1425cb9cadadSEd Warnicke  stream = 0;
142671e97c63SCalvin  if (!pool_is_free_index (pg->streams, t->stream_index))
1427cb9cadadSEd Warnicke    stream = pool_elt_at_index (pg->streams, t->stream_index);
1428cb9cadadSEd Warnicke
1429cb9cadadSEd Warnicke  if (stream)
1430cb9cadadSEd Warnicke    s = format (s, "stream %v", pg->streams[t->stream_index].name);
1431cb9cadadSEd Warnicke  else
1432cb9cadadSEd Warnicke    s = format (s, "stream %d", t->stream_index);
1433cb9cadadSEd Warnicke
1434cb9cadadSEd Warnicke  s = format (s, ", %d bytes", t->packet_length);
14351671d3beSPaul Vinciguerra  s = format (s, ", sw_if_index %d", t->sw_if_index);
1436cb9cadadSEd Warnicke
1437cb9cadadSEd Warnicke  s = format (s, "\n%U%U",
1438bd846cdcSDamjan Marion	      format_white_space, indent, format_vnet_buffer, &t->buffer);
1439cb9cadadSEd Warnicke
144071e97c63SCalvin  s = format (s, "\n%U", format_white_space, indent);
1441cb9cadadSEd Warnicke
1442cb9cadadSEd Warnicke  n = 0;
1443cb9cadadSEd Warnicke  if (stream)
1444cb9cadadSEd Warnicke    n = vlib_get_node (vm, stream->node_index);
1445cb9cadadSEd Warnicke
1446cb9cadadSEd Warnicke  if (n && n->format_buffer)
1447cb9cadadSEd Warnicke    s = format (s, "%U", n->format_buffer,
144871e97c63SCalvin		t->buffer.pre_data, sizeof (t->buffer.pre_data));
1449cb9cadadSEd Warnicke  else
145071e97c63SCalvin    s = format (s, "%U",
1451cb9cadadSEd Warnicke		format_hex_bytes, t->buffer.pre_data,
1452cb9cadadSEd Warnicke		ARRAY_LEN (t->buffer.pre_data));
1453cb9cadadSEd Warnicke  return s;
1454cb9cadadSEd Warnicke}
1455cb9cadadSEd Warnicke
1456cb9cadadSEd Warnickestatic void
1457cb9cadadSEd Warnickepg_input_trace (pg_main_t * pg,
145865cbcfe8SDamjan Marion		vlib_node_runtime_t * node, u32 stream_index, u32 next_index,
145965cbcfe8SDamjan Marion		u32 * buffers, u32 n_buffers)
1460cb9cadadSEd Warnicke{
146164034367SDamjan Marion  vlib_main_t *vm = vlib_get_main ();
146265cbcfe8SDamjan Marion  u32 *b, n_left;
1463cb9cadadSEd Warnicke
1464cb9cadadSEd Warnicke  n_left = n_buffers;
1465cb9cadadSEd Warnicke  b = buffers;
1466cb9cadadSEd Warnicke
1467cb9cadadSEd Warnicke  while (n_left >= 2)
1468cb9cadadSEd Warnicke    {
1469cb9cadadSEd Warnicke      u32 bi0, bi1;
147071e97c63SCalvin      vlib_buffer_t *b0, *b1;
147171e97c63SCalvin      pg_input_trace_t *t0, *t1;
1472cb9cadadSEd Warnicke
1473cb9cadadSEd Warnicke      bi0 = b[0];
1474cb9cadadSEd Warnicke      bi1 = b[1];
1475cb9cadadSEd Warnicke      b += 2;
1476cb9cadadSEd Warnicke      n_left -= 2;
1477cb9cadadSEd Warnicke
1478cb9cadadSEd Warnicke      b0 = vlib_get_buffer (vm, bi0);
1479cb9cadadSEd Warnicke      b1 = vlib_get_buffer (vm, bi1);
1480cb9cadadSEd Warnicke
1481cb9cadadSEd Warnicke      vlib_trace_buffer (vm, node, next_index, b0, /* follow_chain */ 1);
1482cb9cadadSEd Warnicke      vlib_trace_buffer (vm, node, next_index, b1, /* follow_chain */ 1);
1483cb9cadadSEd Warnicke
1484cb9cadadSEd Warnicke      t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1485cb9cadadSEd Warnicke      t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1486cb9cadadSEd Warnicke
1487cb9cadadSEd Warnicke      t0->stream_index = stream_index;
1488cb9cadadSEd Warnicke      t1->stream_index = stream_index;
1489cb9cadadSEd Warnicke
1490cb9cadadSEd Warnicke      t0->packet_length = vlib_buffer_length_in_chain (vm, b0);
1491cb9cadadSEd Warnicke      t1->packet_length = vlib_buffer_length_in_chain (vm, b1);
1492cb9cadadSEd Warnicke
14933466c302SNeale Ranns      t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
14943466c302SNeale Ranns      t1->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_RX];
14953466c302SNeale Ranns
1496178cf493SDave Barach      clib_memcpy_fast (&t0->buffer, b0,
1497178cf493SDave Barach			sizeof (b0[0]) - sizeof (b0->pre_data));
1498178cf493SDave Barach      clib_memcpy_fast (&t1->buffer, b1,
1499178cf493SDave Barach			sizeof (b1[0]) - sizeof (b1->pre_data));
1500cb9cadadSEd Warnicke
1501178cf493SDave Barach      clib_memcpy_fast (t0->buffer.pre_data, b0->data,
1502178cf493SDave Barach			sizeof (t0->buffer.pre_data));
1503178cf493SDave Barach      clib_memcpy_fast (t1->buffer.pre_data, b1->data,
1504178cf493SDave Barach			sizeof (t1->buffer.pre_data));
1505cb9cadadSEd Warnicke    }
1506cb9cadadSEd Warnicke
1507cb9cadadSEd Warnicke  while (n_left >= 1)
1508cb9cadadSEd Warnicke    {
1509cb9cadadSEd Warnicke      u32 bi0;
151071e97c63SCalvin      vlib_buffer_t *b0;
151171e97c63SCalvin      pg_input_trace_t *t0;
1512cb9cadadSEd Warnicke
1513cb9cadadSEd Warnicke      bi0 = b[0];
1514cb9cadadSEd Warnicke      b += 1;
1515cb9cadadSEd Warnicke      n_left -= 1;
1516cb9cadadSEd Warnicke
1517cb9cadadSEd Warnicke      b0 = vlib_get_buffer (vm, bi0);
1518cb9cadadSEd Warnicke
1519cb9cadadSEd Warnicke      vlib_trace_buffer (vm, node, next_index, b0, /* follow_chain */ 1);
1520cb9cadadSEd Warnicke      t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1521cb9cadadSEd Warnicke
1522cb9cadadSEd Warnicke      t0->stream_index = stream_index;
1523cb9cadadSEd Warnicke      t0->packet_length = vlib_buffer_length_in_chain (vm, b0);
15243466c302SNeale Ranns      t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1525178cf493SDave Barach      clib_memcpy_fast (&t0->buffer, b0,
1526178cf493SDave Barach			sizeof (b0[0]) - sizeof (b0->pre_data));
1527178cf493SDave Barach      clib_memcpy_fast (t0->buffer.pre_data, b0->data,
1528178cf493SDave Barach			sizeof (t0->buffer.pre_data));
1529cb9cadadSEd Warnicke    }
1530cb9cadadSEd Warnicke}
1531cb9cadadSEd Warnicke
153222e9cfd7SMohsin Kazmistatic_always_inline void
1533698eb87aSVladimir Isaevfill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
1534698eb87aSVladimir Isaev			   int gso_enabled, u32 gso_size)
153522e9cfd7SMohsin Kazmi{
153622e9cfd7SMohsin Kazmi  for (int i = 0; i < n_buffers; i++)
153722e9cfd7SMohsin Kazmi    {
153822e9cfd7SMohsin Kazmi      vlib_buffer_t *b0 = vlib_get_buffer (vm, buffers[i]);
153922e9cfd7SMohsin Kazmi      u8 l4_proto = 0;
154022e9cfd7SMohsin Kazmi
154172e7312aSMohsin Kazmi      ethernet_header_t *eh =
154272e7312aSMohsin Kazmi	(ethernet_header_t *) vlib_buffer_get_current (b0);
154322e9cfd7SMohsin Kazmi      u16 ethertype = clib_net_to_host_u16 (eh->type);
154422e9cfd7SMohsin Kazmi      u16 l2hdr_sz = sizeof (ethernet_header_t);
154522e9cfd7SMohsin Kazmi
154614bea1bbSMohsin Kazmi      if (ethernet_frame_is_tagged (ethertype))
154714bea1bbSMohsin Kazmi	{
154814bea1bbSMohsin Kazmi	  ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1);
154914bea1bbSMohsin Kazmi
155014bea1bbSMohsin Kazmi	  ethertype = clib_net_to_host_u16 (vlan->type);
155114bea1bbSMohsin Kazmi	  l2hdr_sz += sizeof (*vlan);
155214bea1bbSMohsin Kazmi	  if (ethertype == ETHERNET_TYPE_VLAN)
155314bea1bbSMohsin Kazmi	    {
155414bea1bbSMohsin Kazmi	      vlan++;
155514bea1bbSMohsin Kazmi	      ethertype = clib_net_to_host_u16 (vlan->type);
155614bea1bbSMohsin Kazmi	      l2hdr_sz += sizeof (*vlan);
155714bea1bbSMohsin Kazmi	    }
155814bea1bbSMohsin Kazmi	}
155914bea1bbSMohsin Kazmi
1560157a4ab4SMohsin Kazmi      vnet_buffer (b0)->l2_hdr_offset = 0;
1561157a4ab4SMohsin Kazmi      vnet_buffer (b0)->l3_hdr_offset = l2hdr_sz;
1562157a4ab4SMohsin Kazmi
156322e9cfd7SMohsin Kazmi      if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP4))
156422e9cfd7SMohsin Kazmi	{
156572e7312aSMohsin Kazmi	  ip4_header_t *ip4 =
156672e7312aSMohsin Kazmi	    (ip4_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz);
1567157a4ab4SMohsin Kazmi	  vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4);
156822e9cfd7SMohsin Kazmi	  l4_proto = ip4->protocol;
156922e9cfd7SMohsin Kazmi	  b0->flags |=
157072e7312aSMohsin Kazmi	    (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
1571157a4ab4SMohsin Kazmi	  b0->flags |= (VNET_BUFFER_F_L2_HDR_OFFSET_VALID
1572157a4ab4SMohsin Kazmi			| VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
1573157a4ab4SMohsin Kazmi			VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
157422e9cfd7SMohsin Kazmi	}
157522e9cfd7SMohsin Kazmi      else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
157622e9cfd7SMohsin Kazmi	{
157772e7312aSMohsin Kazmi	  ip6_header_t *ip6 =
157872e7312aSMohsin Kazmi	    (ip6_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz);
1579157a4ab4SMohsin Kazmi	  vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + sizeof (ip6_header_t);
158022e9cfd7SMohsin Kazmi	  /* FIXME IPv6 EH traversal */
158122e9cfd7SMohsin Kazmi	  l4_proto = ip6->protocol;
1582157a4ab4SMohsin Kazmi	  b0->flags |=
1583157a4ab4SMohsin Kazmi	    (VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
1584157a4ab4SMohsin Kazmi	     VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
1585157a4ab4SMohsin Kazmi	     VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
158622e9cfd7SMohsin Kazmi	}
1587698eb87aSVladimir Isaev
158822e9cfd7SMohsin Kazmi      if (l4_proto == IP_PROTOCOL_TCP)
158922e9cfd7SMohsin Kazmi	{
1590698eb87aSVladimir Isaev	  b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
15910cf52823SMohsin Kazmi
15920cf52823SMohsin Kazmi	  /* only set GSO flag for chained buffers */
15930cf52823SMohsin Kazmi	  if (gso_enabled && (b0->flags & VLIB_BUFFER_NEXT_PRESENT))
1594698eb87aSVladimir Isaev	    {
1595698eb87aSVladimir Isaev	      b0->flags |= VNET_BUFFER_F_GSO;
15960cf52823SMohsin Kazmi	      tcp_header_t *tcp =
15970cf52823SMohsin Kazmi		(tcp_header_t *) (vlib_buffer_get_current (b0) +
15980cf52823SMohsin Kazmi				  vnet_buffer (b0)->l4_hdr_offset);
15990cf52823SMohsin Kazmi	      vnet_buffer2 (b0)->gso_l4_hdr_sz = tcp_header_bytes (tcp);
1600698eb87aSVladimir Isaev	      vnet_buffer2 (b0)->gso_size = gso_size;
1601698eb87aSVladimir Isaev	    }
160222e9cfd7SMohsin Kazmi	}
160322e9cfd7SMohsin Kazmi      else if (l4_proto == IP_PROTOCOL_UDP)
160422e9cfd7SMohsin Kazmi	{
160522e9cfd7SMohsin Kazmi	  b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
160622e9cfd7SMohsin Kazmi	}
160722e9cfd7SMohsin Kazmi    }
160822e9cfd7SMohsin Kazmi}
160922e9cfd7SMohsin Kazmi
1610cb9cadadSEd Warnickestatic uword
1611cb9cadadSEd Warnickepg_generate_packets (vlib_node_runtime_t * node,
1612cb9cadadSEd Warnicke		     pg_main_t * pg,
161371e97c63SCalvin		     pg_stream_t * s, uword n_packets_to_generate)
1614cb9cadadSEd Warnicke{
161564034367SDamjan Marion  vlib_main_t *vm = vlib_get_main ();
161671e97c63SCalvin  u32 *to_next, n_this_frame, n_left, n_trace, n_packets_in_fifo;
1617cb9cadadSEd Warnicke  uword n_packets_generated;
161871e97c63SCalvin  pg_buffer_index_t *bi, *bi0;
1619d2017f6dSDamjan Marion  u32 next_index = s->next_index;
1620d2017f6dSDamjan Marion  vnet_feature_main_t *fm = &feature_main;
1621d2017f6dSDamjan Marion  vnet_feature_config_main_t *cm;
1622d2017f6dSDamjan Marion  u8 feature_arc_index = fm->device_input_feature_arc_index;
1623d2017f6dSDamjan Marion  cm = &fm->feature_config_mains[feature_arc_index];
1624d2017f6dSDamjan Marion  u32 current_config_index = ~(u32) 0;
162522e9cfd7SMohsin Kazmi  pg_interface_t *pi = pool_elt_at_index (pg->interfaces, s->pg_if_index);
1626d2017f6dSDamjan Marion  int i;
1627cb9cadadSEd Warnicke
1628cb9cadadSEd Warnicke  bi0 = s->buffer_indices;
1629cb9cadadSEd Warnicke
1630cb9cadadSEd Warnicke  n_packets_in_fifo = pg_stream_fill (pg, s, n_packets_to_generate);
1631cb9cadadSEd Warnicke  n_packets_to_generate = clib_min (n_packets_in_fifo, n_packets_to_generate);
1632cb9cadadSEd Warnicke  n_packets_generated = 0;
1633cb9cadadSEd Warnicke
1634d2017f6dSDamjan Marion  if (PREDICT_FALSE
1635d2017f6dSDamjan Marion      (vnet_have_features (feature_arc_index, s->sw_if_index[VLIB_RX])))
1636d2017f6dSDamjan Marion    {
1637d2017f6dSDamjan Marion      current_config_index =
1638d2017f6dSDamjan Marion	vec_elt (cm->config_index_by_sw_if_index, s->sw_if_index[VLIB_RX]);
1639d2017f6dSDamjan Marion      vnet_get_config_data (&cm->config_main, &current_config_index,
1640d2017f6dSDamjan Marion			    &next_index, 0);
1641d2017f6dSDamjan Marion    }
1642d2017f6dSDamjan Marion
1643cb9cadadSEd Warnicke  while (n_packets_to_generate > 0)
1644cb9cadadSEd Warnicke    {
164571e97c63SCalvin      u32 *head, *start, *end;
1646cb9cadadSEd Warnicke
1647650223c0SDamjan Marion      if (PREDICT_TRUE (next_index == VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT))
1648650223c0SDamjan Marion	{
1649650223c0SDamjan Marion	  vlib_next_frame_t *nf;
1650650223c0SDamjan Marion	  vlib_frame_t *f;
1651650223c0SDamjan Marion	  ethernet_input_frame_t *ef;
1652650223c0SDamjan Marion	  vlib_get_new_next_frame (vm, node, next_index, to_next, n_left);
1653650223c0SDamjan Marion	  nf = vlib_node_runtime_get_next_frame (vm, node, next_index);
165458b2eb1aSAndreas Schultz	  f = vlib_get_frame (vm, nf->frame);
1655650223c0SDamjan Marion	  f->flags = ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX;
1656650223c0SDamjan Marion
1657650223c0SDamjan Marion	  ef = vlib_frame_scalar_args (f);
1658650223c0SDamjan Marion	  ef->sw_if_index = pi->sw_if_index;
1659650223c0SDamjan Marion	  ef->hw_if_index = pi->hw_if_index;
1660296988d3SDamjan Marion	  vlib_frame_no_append (f);
1661650223c0SDamjan Marion	}
1662650223c0SDamjan Marion      else
1663650223c0SDamjan Marion	vlib_get_next_frame (vm, node, next_index, to_next, n_left);
1664cb9cadadSEd Warnicke
1665cb9cadadSEd Warnicke      n_this_frame = n_packets_to_generate;
1666cb9cadadSEd Warnicke      if (n_this_frame > n_left)
1667cb9cadadSEd Warnicke	n_this_frame = n_left;
1668cb9cadadSEd Warnicke
1669cb9cadadSEd Warnicke      start = bi0->buffer_fifo;
1670cb9cadadSEd Warnicke      end = clib_fifo_end (bi0->buffer_fifo);
1671cb9cadadSEd Warnicke      head = clib_fifo_head (bi0->buffer_fifo);
1672cb9cadadSEd Warnicke
1673cb9cadadSEd Warnicke      if (head + n_this_frame <= end)
167464d557cdSDamjan Marion	vlib_buffer_copy_indices (to_next, head, n_this_frame);
1675cb9cadadSEd Warnicke      else
1676cb9cadadSEd Warnicke	{
1677cb9cadadSEd Warnicke	  u32 n = end - head;
167864d557cdSDamjan Marion	  vlib_buffer_copy_indices (to_next + 0, head, n);
167964d557cdSDamjan Marion	  vlib_buffer_copy_indices (to_next + n, start, n_this_frame - n);
1680cb9cadadSEd Warnicke	}
1681cb9cadadSEd Warnicke
16823c8e1468SDave Barach      if (s->replay_packet_templates == 0)
16833c8e1468SDave Barach	{
16843c8e1468SDave Barach	  vec_foreach (bi, s->buffer_indices)
16853c8e1468SDave Barach	    clib_fifo_advance_head (bi->buffer_fifo, n_this_frame);
16863c8e1468SDave Barach	}
16873c8e1468SDave Barach      else
16883c8e1468SDave Barach	{
16893c8e1468SDave Barach	  clib_fifo_advance_head (bi0->buffer_fifo, n_this_frame);
16903c8e1468SDave Barach	}
1691cb9cadadSEd Warnicke
1692d2017f6dSDamjan Marion      if (current_config_index != ~(u32) 0)
1693d2017f6dSDamjan Marion	for (i = 0; i < n_this_frame; i++)
1694d2017f6dSDamjan Marion	  {
1695d2017f6dSDamjan Marion	    vlib_buffer_t *b;
1696d2017f6dSDamjan Marion	    b = vlib_get_buffer (vm, to_next[i]);
1697d2017f6dSDamjan Marion	    b->current_config_index = current_config_index;
1698aa682a39SDamjan Marion	    vnet_buffer (b)->feature_arc_index = feature_arc_index;
1699d2017f6dSDamjan Marion	  }
1700d2017f6dSDamjan Marion
1701698eb87aSVladimir Isaev      if (pi->gso_enabled ||
1702698eb87aSVladimir Isaev	  (s->buffer_flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
1703698eb87aSVladimir Isaev			      VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
1704698eb87aSVladimir Isaev			      VNET_BUFFER_F_OFFLOAD_IP_CKSUM)))
1705698eb87aSVladimir Isaev	{
1706698eb87aSVladimir Isaev	  fill_buffer_offload_flags (vm, to_next, n_this_frame,
1707698eb87aSVladimir Isaev				     pi->gso_enabled, pi->gso_size);
1708698eb87aSVladimir Isaev	}
170922e9cfd7SMohsin Kazmi
1710cb9cadadSEd Warnicke      n_trace = vlib_get_trace_count (vm, node);
1711cb9cadadSEd Warnicke      if (n_trace > 0)
1712cb9cadadSEd Warnicke	{
1713cb9cadadSEd Warnicke	  u32 n = clib_min (n_trace, n_this_frame);
171465cbcfe8SDamjan Marion	  pg_input_trace (pg, node, s - pg->streams, next_index, to_next, n);
1715cb9cadadSEd Warnicke	  vlib_set_trace_count (vm, node, n_trace - n);
1716cb9cadadSEd Warnicke	}
1717cb9cadadSEd Warnicke      n_packets_to_generate -= n_this_frame;
1718cb9cadadSEd Warnicke      n_packets_generated += n_this_frame;
1719cb9cadadSEd Warnicke      n_left -= n_this_frame;
17203c8e1468SDave Barach      if (CLIB_DEBUG > 0)
17213c8e1468SDave Barach	{
17223c8e1468SDave Barach	  int i;
17233c8e1468SDave Barach	  vlib_buffer_t *b;
17243c8e1468SDave Barach
17252768cdc0SDamjan Marion	  for (i = 0; i < n_this_frame; i++)
17263c8e1468SDave Barach	    {
17273c8e1468SDave Barach	      b = vlib_get_buffer (vm, to_next[i]);