1/*
2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15/*
16 * pg_stream.c: packet generator streams
17 *
18 * Copyright (c) 2008 Eliot Dresselhaus
19 *
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
27 *
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
30 *
31 *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 */
39
40#include <vnet/vnet.h>
41#include <vnet/pg/pg.h>
42#include <vnet/ethernet/ethernet.h>
43#include <vnet/ip/ip.h>
44#include <vnet/mpls/mpls.h>
45#include <vnet/devices/devices.h>
46
47/* Mark stream active or inactive. */
48void
49pg_stream_enable_disable (pg_main_t * pg, pg_stream_t * s, int want_enabled)
50{
51  vlib_main_t *vm;
52  vnet_main_t *vnm = vnet_get_main ();
53  pg_interface_t *pi = pool_elt_at_index (pg->interfaces, s->pg_if_index);
54
55  want_enabled = want_enabled != 0;
56
57  if (pg_stream_is_enabled (s) == want_enabled)
58    /* No change necessary. */
59    return;
60
61  if (want_enabled)
62    s->n_packets_generated = 0;
63
64  /* Toggle enabled flag. */
65  s->flags ^= PG_STREAM_FLAGS_IS_ENABLED;
66
67  ASSERT (!pool_is_free (pg->streams, s));
68
69  vec_validate (pg->enabled_streams, s->worker_index);
70  pg->enabled_streams[s->worker_index] =
71    clib_bitmap_set (pg->enabled_streams[s->worker_index], s - pg->streams,
72		     want_enabled);
73
74  if (want_enabled)
75    {
76      vnet_hw_interface_set_flags (vnm, pi->hw_if_index,
77				   VNET_HW_INTERFACE_FLAG_LINK_UP);
78
79      vnet_sw_interface_set_flags (vnm, pi->sw_if_index,
80				   VNET_SW_INTERFACE_FLAG_ADMIN_UP);
81    }
82
83  if (vlib_num_workers ())
84    vm = vlib_get_worker_vlib_main (s->worker_index);
85  else
86    vm = vlib_get_main ();
87
88  vlib_node_set_state (vm, pg_input_node.index,
89		       (clib_bitmap_is_zero
90			(pg->enabled_streams[s->worker_index]) ?
91			VLIB_NODE_STATE_DISABLED : VLIB_NODE_STATE_POLLING));
92
93  s->packet_accumulator = 0;
94  s->time_last_generate = 0;
95}
96
97static u8 *
98format_pg_output_trace (u8 * s, va_list * va)
99{
100  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
101  CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
102  pg_output_trace_t *t = va_arg (*va, pg_output_trace_t *);
103  u32 indent = format_get_indent (s);
104
105  s = format (s, "%Ubuffer 0x%x: %U",
106	      format_white_space, indent,
107	      t->buffer_index, format_vnet_buffer, &t->buffer);
108
109  s = format (s, "\n%U%U", format_white_space, indent,
110	      format_ethernet_header_with_length, t->buffer.pre_data,
111	      sizeof (t->buffer.pre_data));
112
113  return s;
114}
115
116static u8 *
117format_pg_interface_name (u8 * s, va_list * args)
118{
119  pg_main_t *pg = &pg_main;
120  u32 if_index = va_arg (*args, u32);
121  pg_interface_t *pi;
122
123  pi = pool_elt_at_index (pg->interfaces, if_index);
124  s = format (s, "pg%d", pi->id);
125
126  return s;
127}
128
129static clib_error_t *
130pg_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
131{
132  u32 hw_flags = 0;
133
134  if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP)
135    hw_flags = VNET_HW_INTERFACE_FLAG_LINK_UP;
136
137  vnet_hw_interface_set_flags (vnm, hw_if_index, hw_flags);
138
139  return 0;
140}
141
142/* *INDENT-OFF* */
143VNET_DEVICE_CLASS (pg_dev_class) = {
144  .name = "pg",
145  .tx_function = pg_output,
146  .format_device_name = format_pg_interface_name,
147  .format_tx_trace = format_pg_output_trace,
148  .admin_up_down_function = pg_interface_admin_up_down,
149};
150/* *INDENT-ON* */
151
152static u8 *
153pg_build_rewrite (vnet_main_t * vnm,
154		  u32 sw_if_index,
155		  vnet_link_t link_type, const void *dst_address)
156{
157  u8 *rewrite = NULL;
158  u16 *h;
159
160  vec_validate (rewrite, sizeof (*h) - 1);
161  h = (u16 *) rewrite;
162  h[0] = clib_host_to_net_u16 (vnet_link_to_l3_proto (link_type));
163
164  return (rewrite);
165}
166
167/* *INDENT-OFF* */
168VNET_HW_INTERFACE_CLASS (pg_interface_class,static) = {
169  .name = "Packet generator",
170  .build_rewrite = pg_build_rewrite,
171};
172/* *INDENT-ON* */
173
174static u32
175pg_eth_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 flags)
176{
177  /* nothing for now */
178  return 0;
179}
180
181u32
182pg_interface_add_or_get (pg_main_t * pg, uword if_id, u8 gso_enabled,
183			 u32 gso_size)
184{
185  vnet_main_t *vnm = vnet_get_main ();
186  vlib_main_t *vm = vlib_get_main ();
187  pg_interface_t *pi;
188  vnet_hw_interface_t *hi;
189  uword *p;
190  u32 i;
191
192  p = hash_get (pg->if_index_by_if_id, if_id);
193
194  if (p)
195    {
196      return p[0];
197    }
198  else
199    {
200      u8 hw_addr[6];
201      f64 now = vlib_time_now (vm);
202      u32 rnd;
203
204      pool_get (pg->interfaces, pi);
205      i = pi - pg->interfaces;
206
207      rnd = (u32) (now * 1e6);
208      rnd = random_u32 (&rnd);
209      clib_memcpy_fast (hw_addr + 2, &rnd, sizeof (rnd));
210      hw_addr[0] = 2;
211      hw_addr[1] = 0xfe;
212
213      pi->id = if_id;
214      ethernet_register_interface (vnm, pg_dev_class.index, i, hw_addr,
215				   &pi->hw_if_index, pg_eth_flag_change);
216      hi = vnet_get_hw_interface (vnm, pi->hw_if_index);
217      if (gso_enabled)
218	{
219	  hi->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO;
220	  pi->gso_enabled = 1;
221	  pi->gso_size = gso_size;
222	}
223      pi->sw_if_index = hi->sw_if_index;
224
225      hash_set (pg->if_index_by_if_id, if_id, i);
226
227      if (vlib_num_workers ())
228	{
229	  pi->lockp = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
230					      CLIB_CACHE_LINE_BYTES);
231	  *pi->lockp = 0;
232	}
233    }
234
235  return i;
236}
237
238static void
239do_edit (pg_stream_t * stream,
240	 pg_edit_group_t * g, pg_edit_t * e, uword want_commit)
241{
242  u32 i, i0, i1, mask, n_bits_left;
243  u8 *v, *s, *m;
244
245  i0 = e->lsb_bit_offset / BITS (u8);
246
247  /* Make space for edit in value and mask. */
248  vec_validate (g->fixed_packet_data, i0);
249  vec_validate (g->fixed_packet_data_mask, i0);
250
251  if (e->type != PG_EDIT_FIXED)
252    {
253      switch (e->type)
254	{
255	case PG_EDIT_RANDOM:
256	case PG_EDIT_INCREMENT:
257	  e->last_increment_value = pg_edit_get_value (e, PG_EDIT_LO);
258	  break;
259
260	default:
261	  break;
262	}
263
264      if (want_commit)
265	{
266	  ASSERT (e->type != PG_EDIT_INVALID_TYPE);
267	  vec_add1 (g->non_fixed_edits, e[0]);
268	}
269      return;
270    }
271
272  s = g->fixed_packet_data;
273  m = g->fixed_packet_data_mask;
274
275  n_bits_left = e->n_bits;
276  i0 = e->lsb_bit_offset / BITS (u8);
277  i1 = e->lsb_bit_offset % BITS (u8);
278
279  v = e->values[PG_EDIT_LO];
280  i = pg_edit_n_alloc_bytes (e) - 1;
281
282  /* Odd low order bits?. */
283  if (i1 != 0 && n_bits_left > 0)
284    {
285      u32 n = clib_min (n_bits_left, BITS (u8) - i1);
286
287      mask = pow2_mask (n) << i1;
288
289      ASSERT (i0 < vec_len (s));
290      ASSERT (i < vec_len (v));
291      ASSERT ((v[i] & ~mask) == 0);
292
293      s[i0] |= v[i] & mask;
294      m[i0] |= mask;
295
296      i0--;
297      i--;
298      n_bits_left -= n;
299    }
300
301  /* Even bytes. */
302  while (n_bits_left >= 8)
303    {
304      ASSERT (i0 < vec_len (s));
305      ASSERT (i < vec_len (v));
306
307      s[i0] = v[i];
308      m[i0] = ~0;
309
310      i0--;
311      i--;
312      n_bits_left -= 8;
313    }
314
315  /* Odd high order bits. */
316  if (n_bits_left > 0)
317    {
318      mask = pow2_mask (n_bits_left);
319
320      ASSERT (i0 < vec_len (s));
321      ASSERT (i < vec_len (v));
322      ASSERT ((v[i] & ~mask) == 0);
323
324      s[i0] |= v[i] & mask;
325      m[i0] |= mask;
326    }
327
328  if (want_commit)
329    pg_edit_free (e);
330}
331
332void
333pg_edit_group_get_fixed_packet_data (pg_stream_t * s,
334				     u32 group_index,
335				     void *packet_data,
336				     void *packet_data_mask)
337{
338  pg_edit_group_t *g = pg_stream_get_group (s, group_index);
339  pg_edit_t *e;
340
341  vec_foreach (e, g->edits) do_edit (s, g, e, /* want_commit */ 0);
342
343  clib_memcpy_fast (packet_data, g->fixed_packet_data,
344		    vec_len (g->fixed_packet_data));
345  clib_memcpy_fast (packet_data_mask, g->fixed_packet_data_mask,
346		    vec_len (g->fixed_packet_data_mask));
347}
348
349static void
350perform_fixed_edits (pg_stream_t * s)
351{
352  pg_edit_group_t *g;
353  pg_edit_t *e;
354  word i;
355
356  for (i = vec_len (s->edit_groups) - 1; i >= 0; i--)
357    {
358      g = vec_elt_at_index (s->edit_groups, i);
359      vec_foreach (e, g->edits) do_edit (s, g, e, /* want_commit */ 1);
360
361      /* All edits have either been performed or added to
362         g->non_fixed_edits.  So, we can delete the vector. */
363      vec_free (g->edits);
364    }
365
366  vec_free (s->fixed_packet_data_mask);
367  vec_free (s->fixed_packet_data);
368  vec_foreach (g, s->edit_groups)
369  {
370    int i;
371    g->start_byte_offset = vec_len (s->fixed_packet_data);
372
373    /* Relocate and copy non-fixed edits from group to stream. */
374    vec_foreach (e, g->non_fixed_edits)
375      e->lsb_bit_offset += g->start_byte_offset * BITS (u8);
376
377    for (i = 0; i < vec_len (g->non_fixed_edits); i++)
378      ASSERT (g->non_fixed_edits[i].type != PG_EDIT_INVALID_TYPE);
379
380    vec_add (s->non_fixed_edits,
381	     g->non_fixed_edits, vec_len (g->non_fixed_edits));
382    vec_free (g->non_fixed_edits);
383
384    vec_add (s->fixed_packet_data,
385	     g->fixed_packet_data, vec_len (g->fixed_packet_data));
386    vec_add (s->fixed_packet_data_mask,
387	     g->fixed_packet_data_mask, vec_len (g->fixed_packet_data_mask));
388  }
389}
390
391void
392pg_stream_add (pg_main_t * pg, pg_stream_t * s_init)
393{
394  vlib_main_t *vm = vlib_get_main ();
395  pg_stream_t *s;
396  uword *p;
397
398  if (!pg->stream_index_by_name)
399    pg->stream_index_by_name
400      = hash_create_vec (0, sizeof (s->name[0]), sizeof (uword));
401
402  /* Delete any old stream with the same name. */
403  if (s_init->name
404      && (p = hash_get_mem (pg->stream_index_by_name, s_init->name)))
405    {
406      pg_stream_del (pg, p[0]);
407    }
408
409  pool_get (pg->streams, s);
410  s[0] = s_init[0];
411
412  /* Give it a name. */
413  if (!s->name)
414    s->name = format (0, "stream%d", s - pg->streams);
415  else
416    s->name = vec_dup (s->name);
417
418  hash_set_mem (pg->stream_index_by_name, s->name, s - pg->streams);
419
420  /* Get fixed part of buffer data. */
421  if (s->edit_groups)
422    perform_fixed_edits (s);
423
424  /* Determine packet size. */
425  switch (s->packet_size_edit_type)
426    {
427    case PG_EDIT_INCREMENT:
428    case PG_EDIT_RANDOM:
429      if (s->min_packet_bytes == s->max_packet_bytes)
430	s->packet_size_edit_type = PG_EDIT_FIXED;
431      break;
432
433    default:
434      /* Get packet size from fixed edits. */
435      s->packet_size_edit_type = PG_EDIT_FIXED;
436      if (!s->replay_packet_templates)
437	s->min_packet_bytes = s->max_packet_bytes =
438	  vec_len (s->fixed_packet_data);
439      break;
440    }
441
442  s->last_increment_packet_size = s->min_packet_bytes;
443
444  {
445    int n;
446
447    s->buffer_bytes = vlib_buffer_get_default_data_size (vm);
448    n = s->max_packet_bytes / s->buffer_bytes;
449    n += (s->max_packet_bytes % s->buffer_bytes) != 0;
450
451    vec_resize (s->buffer_indices, n);
452  }
453
454  /* Find an interface to use. */
455  s->pg_if_index =
456    pg_interface_add_or_get (pg, s->if_id, 0 /* gso_enabled */ ,
457			     0 /* gso_size */ );
458
459  if (s->sw_if_index[VLIB_RX] == ~0)
460    {
461      pg_interface_t *pi = pool_elt_at_index (pg->interfaces, s->pg_if_index);
462      /*
463       * Default the RX interface if unset. It's a bad mistake to
464       * set [VLIB_TX] prior to ip lookup, since the ip lookup code
465       * interprets [VLIB_TX] as a fib index...
466       */
467      s->sw_if_index[VLIB_RX] = pi->sw_if_index;
468    }
469
470  /* Connect the graph. */
471  s->next_index = vlib_node_add_next (vm, device_input_node.index,
472				      s->node_index);
473}
474
475void
476pg_stream_del (pg_main_t * pg, uword index)
477{
478  pg_stream_t *s;
479  pg_buffer_index_t *bi;
480
481  s = pool_elt_at_index (pg->streams, index);
482
483  pg_stream_enable_disable (pg, s, /* want_enabled */ 0);
484  hash_unset_mem (pg->stream_index_by_name, s->name);
485
486  vec_foreach (bi, s->buffer_indices)
487  {
488    clib_fifo_free (bi->buffer_fifo);
489  }
490
491  pg_stream_free (s);
492  pool_put (pg->streams, s);
493}
494
495void
496pg_stream_change (pg_main_t * pg, pg_stream_t * s)
497{
498  /* Determine packet size. */
499  switch (s->packet_size_edit_type)
500    {
501    case PG_EDIT_INCREMENT:
502    case PG_EDIT_RANDOM:
503      if (s->min_packet_bytes == s->max_packet_bytes)
504	s->packet_size_edit_type = PG_EDIT_FIXED;
505    case PG_EDIT_FIXED:
506      break;
507
508    default:
509      /* Get packet size from fixed edits. */
510      s->packet_size_edit_type = PG_EDIT_FIXED;
511      if (!s->replay_packet_templates)
512	s->min_packet_bytes = s->max_packet_bytes =
513	  vec_len (s->fixed_packet_data);
514      break;
515    }
516
517  s->last_increment_packet_size = s->min_packet_bytes;
518}
519
520
521/*
522 * fd.io coding-style-patch-verification: ON
523 *
524 * Local Variables:
525 * eval: (c-set-style "gnu")
526 * End:
527 */
528