1/*
2 * Copyright (c) 2017 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16/**
17 * @file
18 * @brief IPv4 Shallow Virtual Reassembly.
19 *
20 * This file contains the source code for IPv4 Shallow Virtual reassembly.
21 */
22
23#include <vppinfra/vec.h>
24#include <vnet/vnet.h>
25#include <vnet/ip/ip.h>
26#include <vnet/ip/ip4_to_ip6.h>
27#include <vppinfra/fifo.h>
28#include <vppinfra/bihash_16_8.h>
29#include <vnet/ip/reass/ip4_sv_reass.h>
30
31#define MSEC_PER_SEC 1000
32#define IP4_SV_REASS_TIMEOUT_DEFAULT_MS 100
33#define IP4_SV_REASS_EXPIRE_WALK_INTERVAL_DEFAULT_MS 10000	// 10 seconds default
34#define IP4_SV_REASS_MAX_REASSEMBLIES_DEFAULT 1024
35#define IP4_SV_REASS_MAX_REASSEMBLY_LENGTH_DEFAULT 3
36#define IP4_SV_REASS_HT_LOAD_FACTOR (0.75)
37
38typedef enum
39{
40  IP4_SV_REASS_RC_OK,
41  IP4_SV_REASS_RC_TOO_MANY_FRAGMENTS,
42  IP4_SV_REASS_RC_UNSUPP_IP_PROTO,
43} ip4_sv_reass_rc_t;
44
45typedef struct
46{
47  union
48  {
49    struct
50    {
51      u32 xx_id;
52      ip4_address_t src;
53      ip4_address_t dst;
54      u16 frag_id;
55      u8 proto;
56      u8 unused;
57    };
58    u64 as_u64[2];
59  };
60} ip4_sv_reass_key_t;
61
62typedef union
63{
64  struct
65  {
66    u32 reass_index;
67    u32 thread_index;
68  };
69  u64 as_u64;
70} ip4_sv_reass_val_t;
71
72typedef union
73{
74  struct
75  {
76    ip4_sv_reass_key_t k;
77    ip4_sv_reass_val_t v;
78  };
79  clib_bihash_kv_16_8_t kv;
80} ip4_sv_reass_kv_t;
81
82typedef struct
83{
84  // hash table key
85  ip4_sv_reass_key_t key;
86  // time when last packet was received
87  f64 last_heard;
88  // internal id of this reassembly
89  u64 id;
90  // trace operation counter
91  u32 trace_op_counter;
92  // minimum fragment length for this reassembly - used to estimate MTU
93  u16 min_fragment_length;
94  // buffer indexes of buffers in this reassembly in chronological order -
95  // including overlaps and duplicate fragments
96  u32 *cached_buffers;
97  // set to true when this reassembly is completed
98  bool is_complete;
99  // ip protocol
100  u8 ip_proto;
101  u8 icmp_type_or_tcp_flags;
102  u32 tcp_ack_number;
103  u32 tcp_seq_number;
104  // l4 src port
105  u16 l4_src_port;
106  // l4 dst port
107  u16 l4_dst_port;
108  u32 next_index;
109  // lru indexes
110  u32 lru_prev;
111  u32 lru_next;
112} ip4_sv_reass_t;
113
114typedef struct
115{
116  ip4_sv_reass_t *pool;
117  u32 reass_n;
118  u32 id_counter;
119  clib_spinlock_t lock;
120  // lru indexes
121  u32 lru_first;
122  u32 lru_last;
123
124} ip4_sv_reass_per_thread_t;
125
126typedef struct
127{
128  // IPv4 config
129  u32 timeout_ms;
130  f64 timeout;
131  u32 expire_walk_interval_ms;
132  // maximum number of fragments in one reassembly
133  u32 max_reass_len;
134  // maximum number of reassemblies
135  u32 max_reass_n;
136
137  // IPv4 runtime
138  clib_bihash_16_8_t hash;
139  // per-thread data
140  ip4_sv_reass_per_thread_t *per_thread_data;
141
142  // convenience
143  vlib_main_t *vlib_main;
144  vnet_main_t *vnet_main;
145
146  // node index of ip4-drop node
147  u32 ip4_drop_idx;
148  u32 ip4_sv_reass_expire_node_idx;
149
150  /** Worker handoff */
151  u32 fq_index;
152  u32 fq_feature_index;
153
154  // reference count for enabling/disabling feature - per interface
155  u32 *feature_use_refcount_per_intf;
156
157  // reference count for enabling/disabling feature - per interface
158  u32 *output_feature_use_refcount_per_intf;
159
160} ip4_sv_reass_main_t;
161
162extern ip4_sv_reass_main_t ip4_sv_reass_main;
163
164#ifndef CLIB_MARCH_VARIANT
165ip4_sv_reass_main_t ip4_sv_reass_main;
166#endif /* CLIB_MARCH_VARIANT */
167
168typedef enum
169{
170  IP4_SV_REASSEMBLY_NEXT_INPUT,
171  IP4_SV_REASSEMBLY_NEXT_DROP,
172  IP4_SV_REASSEMBLY_NEXT_HANDOFF,
173  IP4_SV_REASSEMBLY_N_NEXT,
174} ip4_sv_reass_next_t;
175
176typedef enum
177{
178  REASS_FRAGMENT_CACHE,
179  REASS_FINISH,
180  REASS_FRAGMENT_FORWARD,
181  REASS_PASSTHROUGH,
182} ip4_sv_reass_trace_operation_e;
183
184typedef struct
185{
186  ip4_sv_reass_trace_operation_e action;
187  u32 reass_id;
188  u32 op_id;
189  u8 ip_proto;
190  u16 l4_src_port;
191  u16 l4_dst_port;
192} ip4_sv_reass_trace_t;
193
194extern vlib_node_registration_t ip4_sv_reass_node;
195extern vlib_node_registration_t ip4_sv_reass_node_feature;
196
197static u8 *
198format_ip4_sv_reass_trace (u8 * s, va_list * args)
199{
200  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
201  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
202  ip4_sv_reass_trace_t *t = va_arg (*args, ip4_sv_reass_trace_t *);
203  if (REASS_PASSTHROUGH != t->action)
204    {
205      s = format (s, "reass id: %u, op id: %u ", t->reass_id, t->op_id);
206    }
207  switch (t->action)
208    {
209    case REASS_FRAGMENT_CACHE:
210      s = format (s, "[cached]");
211      break;
212    case REASS_FINISH:
213      s =
214	format (s, "[finish, ip proto=%u, src_port=%u, dst_port=%u]",
215		t->ip_proto, clib_net_to_host_u16 (t->l4_src_port),
216		clib_net_to_host_u16 (t->l4_dst_port));
217      break;
218    case REASS_FRAGMENT_FORWARD:
219      s =
220	format (s, "[forward, ip proto=%u, src_port=%u, dst_port=%u]",
221		t->ip_proto, clib_net_to_host_u16 (t->l4_src_port),
222		clib_net_to_host_u16 (t->l4_dst_port));
223      break;
224    case REASS_PASSTHROUGH:
225      s = format (s, "[not-fragmented]");
226      break;
227    }
228  return s;
229}
230
231static void
232ip4_sv_reass_add_trace (vlib_main_t * vm, vlib_node_runtime_t * node,
233			ip4_sv_reass_main_t * rm, ip4_sv_reass_t * reass,
234			u32 bi, ip4_sv_reass_trace_operation_e action,
235			u32 ip_proto, u16 l4_src_port, u16 l4_dst_port)
236{
237  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
238  ip4_sv_reass_trace_t *t = vlib_add_trace (vm, node, b, sizeof (t[0]));
239  if (reass)
240    {
241      t->reass_id = reass->id;
242      t->op_id = reass->trace_op_counter;
243      ++reass->trace_op_counter;
244    }
245  t->action = action;
246  t->ip_proto = ip_proto;
247  t->l4_src_port = l4_src_port;
248  t->l4_dst_port = l4_dst_port;
249#if 0
250  static u8 *s = NULL;
251  s = format (s, "%U", format_ip4_sv_reass_trace, NULL, NULL, t);
252  printf ("%.*s\n", vec_len (s), s);
253  fflush (stdout);
254  vec_reset_length (s);
255#endif
256}
257
258
259always_inline void
260ip4_sv_reass_free (vlib_main_t * vm, ip4_sv_reass_main_t * rm,
261		   ip4_sv_reass_per_thread_t * rt, ip4_sv_reass_t * reass)
262{
263  clib_bihash_kv_16_8_t kv;
264  kv.key[0] = reass->key.as_u64[0];
265  kv.key[1] = reass->key.as_u64[1];
266  clib_bihash_add_del_16_8 (&rm->hash, &kv, 0);
267  vlib_buffer_free (vm, reass->cached_buffers,
268		    vec_len (reass->cached_buffers));
269  vec_free (reass->cached_buffers);
270  reass->cached_buffers = NULL;
271  if (~0 != reass->lru_prev)
272    {
273      ip4_sv_reass_t *lru_prev =
274	pool_elt_at_index (rt->pool, reass->lru_prev);
275      lru_prev->lru_next = reass->lru_next;
276    }
277  if (~0 != reass->lru_next)
278    {
279      ip4_sv_reass_t *lru_next =
280	pool_elt_at_index (rt->pool, reass->lru_next);
281      lru_next->lru_prev = reass->lru_prev;
282    }
283  if (rt->lru_first == reass - rt->pool)
284    {
285      rt->lru_first = reass->lru_next;
286    }
287  if (rt->lru_last == reass - rt->pool)
288    {
289      rt->lru_last = reass->lru_prev;
290    }
291  pool_put (rt->pool, reass);
292  --rt->reass_n;
293}
294
295always_inline void
296ip4_sv_reass_init (ip4_sv_reass_t * reass)
297{
298  reass->cached_buffers = NULL;
299  reass->is_complete = false;
300}
301
302always_inline ip4_sv_reass_t *
303ip4_sv_reass_find_or_create (vlib_main_t * vm, ip4_sv_reass_main_t * rm,
304			     ip4_sv_reass_per_thread_t * rt,
305			     ip4_sv_reass_kv_t * kv, u8 * do_handoff)
306{
307  ip4_sv_reass_t *reass = NULL;
308  f64 now = vlib_time_now (vm);
309
310  if (!clib_bihash_search_16_8 (&rm->hash, &kv->kv, &kv->kv))
311    {
312      if (vm->thread_index != kv->v.thread_index)
313	{
314	  *do_handoff = 1;
315	  return NULL;
316	}
317      reass = pool_elt_at_index (rt->pool, kv->v.reass_index);
318
319      if (now > reass->last_heard + rm->timeout)
320	{
321	  ip4_sv_reass_free (vm, rm, rt, reass);
322	  reass = NULL;
323	}
324    }
325
326  if (reass)
327    {
328      reass->last_heard = now;
329      return reass;
330    }
331
332  if (rt->reass_n >= rm->max_reass_n && rm->max_reass_n)
333    {
334      reass = pool_elt_at_index (rt->pool, rt->lru_last);
335      ip4_sv_reass_free (vm, rm, rt, reass);
336    }
337
338  pool_get (rt->pool, reass);
339  clib_memset (reass, 0, sizeof (*reass));
340  reass->id = ((u64) vm->thread_index * 1000000000) + rt->id_counter;
341  ++rt->id_counter;
342  ip4_sv_reass_init (reass);
343  ++rt->reass_n;
344  reass->lru_prev = reass->lru_next = ~0;
345
346  if (~0 != rt->lru_last)
347    {
348      ip4_sv_reass_t *lru_last = pool_elt_at_index (rt->pool, rt->lru_last);
349      reass->lru_prev = rt->lru_last;
350      lru_last->lru_next = rt->lru_last = reass - rt->pool;
351    }
352
353  if (~0 == rt->lru_first)
354    {
355      rt->lru_first = rt->lru_last = reass - rt->pool;
356    }
357
358  reass->key.as_u64[0] = kv->kv.key[0];
359  reass->key.as_u64[1] = kv->kv.key[1];
360  kv->v.reass_index = (reass - rt->pool);
361  kv->v.thread_index = vm->thread_index;
362  reass->last_heard = now;
363
364  if (clib_bihash_add_del_16_8 (&rm->hash, &kv->kv, 1))
365    {
366      ip4_sv_reass_free (vm, rm, rt, reass);
367      reass = NULL;
368    }
369
370  return reass;
371}
372
373always_inline ip4_sv_reass_rc_t
374ip4_sv_reass_update (vlib_main_t * vm, vlib_node_runtime_t * node,
375		     ip4_sv_reass_main_t * rm, ip4_sv_reass_per_thread_t * rt,
376		     ip4_header_t * ip0, ip4_sv_reass_t * reass, u32 bi0)
377{
378  vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
379  ip4_sv_reass_rc_t rc = IP4_SV_REASS_RC_OK;
380  const u32 fragment_first = ip4_get_fragment_offset_bytes (ip0);
381  if (0 == fragment_first)
382    {
383      reass->ip_proto = ip0->protocol;
384      reass->l4_src_port = ip4_get_port (ip0, 1);
385      reass->l4_dst_port = ip4_get_port (ip0, 0);
386      if (!reass->l4_src_port || !reass->l4_dst_port)
387	return IP4_SV_REASS_RC_UNSUPP_IP_PROTO;
388      if (IP_PROTOCOL_TCP == reass->ip_proto)
389	{
390	  reass->icmp_type_or_tcp_flags = ((tcp_header_t *) (ip0 + 1))->flags;
391	  reass->tcp_ack_number = ((tcp_header_t *) (ip0 + 1))->ack_number;
392	  reass->tcp_seq_number = ((tcp_header_t *) (ip0 + 1))->seq_number;
393	}
394      else if (IP_PROTOCOL_ICMP == reass->ip_proto)
395	{
396	  reass->icmp_type_or_tcp_flags =
397	    ((icmp46_header_t *) (ip0 + 1))->type;
398	}
399      reass->is_complete = true;
400      vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
401      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
402	{
403	  ip4_sv_reass_add_trace (vm, node, rm, reass, bi0, REASS_FINISH,
404				  reass->ip_proto, reass->l4_src_port,
405				  reass->l4_dst_port);
406	}
407    }
408  vec_add1 (reass->cached_buffers, bi0);
409  if (!reass->is_complete)
410    {
411      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
412	{
413	  ip4_sv_reass_add_trace (vm, node, rm, reass, bi0,
414				  REASS_FRAGMENT_CACHE, ~0, ~0, ~0);
415	}
416      if (vec_len (reass->cached_buffers) > rm->max_reass_len)
417	{
418	  rc = IP4_SV_REASS_RC_TOO_MANY_FRAGMENTS;
419	}
420    }
421  return rc;
422}
423
424always_inline uword
425ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
426		     vlib_frame_t * frame, bool is_feature,
427		     bool is_output_feature, bool is_custom)
428{
429  u32 *from = vlib_frame_vector_args (frame);
430  u32 n_left_from, n_left_to_next, *to_next, next_index;
431  ip4_sv_reass_main_t *rm = &ip4_sv_reass_main;
432  ip4_sv_reass_per_thread_t *rt = &rm->per_thread_data[vm->thread_index];
433  clib_spinlock_lock (&rt->lock);
434
435  n_left_from = frame->n_vectors;
436  next_index = node->cached_next_index;
437
438  while (n_left_from > 0)
439    {
440      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
441
442      while (n_left_from > 0 && n_left_to_next > 0)
443	{
444	  u32 bi0;
445	  vlib_buffer_t *b0;
446	  u32 next0;
447	  u32 error0 = IP4_ERROR_NONE;
448
449	  bi0 = from[0];
450	  b0 = vlib_get_buffer (vm, bi0);
451
452	  ip4_header_t *ip0 =
453	    (ip4_header_t *) u8_ptr_add (vlib_buffer_get_current (b0),
454					 (is_output_feature ? 1 : 0) *
455					 vnet_buffer (b0)->
456					 ip.save_rewrite_length);
457	  if (!ip4_get_fragment_more (ip0) && !ip4_get_fragment_offset (ip0))
458	    {
459	      // this is a regular packet - no fragmentation
460	      if (is_custom)
461		{
462		  next0 = vnet_buffer (b0)->ip.reass.next_index;
463		}
464	      else
465		{
466		  next0 = IP4_SV_REASSEMBLY_NEXT_INPUT;
467		}
468	      vnet_buffer (b0)->ip.reass.is_non_first_fragment = 0;
469	      vnet_buffer (b0)->ip.reass.ip_proto = ip0->protocol;
470	      if (IP_PROTOCOL_TCP == ip0->protocol)
471		{
472		  vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags =
473		    ((tcp_header_t *) (ip0 + 1))->flags;
474		  vnet_buffer (b0)->ip.reass.tcp_ack_number =
475		    ((tcp_header_t *) (ip0 + 1))->ack_number;
476		  vnet_buffer (b0)->ip.reass.tcp_seq_number =
477		    ((tcp_header_t *) (ip0 + 1))->seq_number;
478		}
479	      else if (IP_PROTOCOL_ICMP == ip0->protocol)
480		{
481		  vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags =
482		    ((icmp46_header_t *) (ip0 + 1))->type;
483		}
484	      vnet_buffer (b0)->ip.reass.l4_src_port = ip4_get_port (ip0, 1);
485	      vnet_buffer (b0)->ip.reass.l4_dst_port = ip4_get_port (ip0, 0);
486	      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
487		{
488		  ip4_sv_reass_add_trace (vm, node, rm, NULL, bi0,
489					  REASS_PASSTHROUGH,
490					  vnet_buffer (b0)->ip.reass.ip_proto,
491					  vnet_buffer (b0)->ip.
492					  reass.l4_src_port,
493					  vnet_buffer (b0)->ip.
494					  reass.l4_dst_port);
495		}
496	      goto packet_enqueue;
497	    }
498	  const u32 fragment_first = ip4_get_fragment_offset_bytes (ip0);
499	  const u32 fragment_length =
500	    clib_net_to_host_u16 (ip0->length) - ip4_header_bytes (ip0);
501	  const u32 fragment_last = fragment_first + fragment_length - 1;
502	  if (fragment_first > fragment_last || fragment_first + fragment_length > UINT16_MAX - 20 || (fragment_length < 8 && ip4_get_fragment_more (ip0)))	// 8 is minimum frag length per RFC 791
503	    {
504	      next0 = IP4_SV_REASSEMBLY_NEXT_DROP;
505	      error0 = IP4_ERROR_REASS_MALFORMED_PACKET;
506	      b0->error = node->errors[error0];
507	      goto packet_enqueue;
508	    }
509	  ip4_sv_reass_kv_t kv;
510	  u8 do_handoff = 0;
511
512	  kv.k.as_u64[0] =
513	    (u64) vec_elt (ip4_main.fib_index_by_sw_if_index,
514			   vnet_buffer (b0)->sw_if_index[VLIB_RX]) |
515	    (u64) ip0->src_address.as_u32 << 32;
516	  kv.k.as_u64[1] =
517	    (u64) ip0->dst_address.
518	    as_u32 | (u64) ip0->fragment_id << 32 | (u64) ip0->protocol << 48;
519
520	  ip4_sv_reass_t *reass =
521	    ip4_sv_reass_find_or_create (vm, rm, rt, &kv, &do_handoff);
522
523	  if (PREDICT_FALSE (do_handoff))
524	    {
525	      next0 = IP4_SV_REASSEMBLY_NEXT_HANDOFF;
526	      vnet_buffer (b0)->ip.reass.owner_thread_index =
527		kv.v.thread_index;
528	      goto packet_enqueue;
529	    }
530
531	  if (!reass)
532	    {
533	      next0 = IP4_SV_REASSEMBLY_NEXT_DROP;
534	      error0 = IP4_ERROR_REASS_LIMIT_REACHED;
535	      b0->error = node->errors[error0];
536	      goto packet_enqueue;
537	    }
538
539	  if (reass->is_complete)
540	    {
541	      if (is_custom)
542		{
543		  next0 = vnet_buffer (b0)->ip.reass.next_index;
544		}
545	      else
546		{
547		  next0 = IP4_SV_REASSEMBLY_NEXT_INPUT;
548		}
549	      vnet_buffer (b0)->ip.reass.is_non_first_fragment =
550		! !fragment_first;
551	      vnet_buffer (b0)->ip.reass.ip_proto = reass->ip_proto;
552	      vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags =
553		reass->icmp_type_or_tcp_flags;
554	      vnet_buffer (b0)->ip.reass.tcp_ack_number =
555		reass->tcp_ack_number;
556	      vnet_buffer (b0)->ip.reass.tcp_seq_number =
557		reass->tcp_seq_number;
558	      vnet_buffer (b0)->ip.reass.l4_src_port = reass->l4_src_port;
559	      vnet_buffer (b0)->ip.reass.l4_dst_port = reass->l4_dst_port;
560	      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
561		{
562		  ip4_sv_reass_add_trace (vm, node, rm, reass, bi0,
563					  REASS_FRAGMENT_FORWARD,
564					  reass->ip_proto,
565					  reass->l4_src_port,
566					  reass->l4_dst_port);
567		}
568	      goto packet_enqueue;
569	    }
570
571	  ip4_sv_reass_rc_t rc =
572	    ip4_sv_reass_update (vm, node, rm, rt, ip0, reass, bi0);
573	  switch (rc)
574	    {
575	    case IP4_SV_REASS_RC_OK:
576	      /* nothing to do here */
577	      break;
578	    case IP4_SV_REASS_RC_TOO_MANY_FRAGMENTS:
579	      vlib_node_increment_counter (vm, node->node_index,
580					   IP4_ERROR_REASS_FRAGMENT_CHAIN_TOO_LONG,
581					   1);
582	      ip4_sv_reass_free (vm, rm, rt, reass);
583	      goto next_packet;
584	      break;
585	    case IP4_SV_REASS_RC_UNSUPP_IP_PROTO:
586	      vlib_node_increment_counter (vm, node->node_index,
587					   IP4_ERROR_REASS_FRAGMENT_CHAIN_TOO_LONG,
588					   1);
589	      ip4_sv_reass_free (vm, rm, rt, reass);
590	      goto next_packet;
591	      break;
592	    }
593	  if (reass->is_complete)
594	    {
595	      u32 idx;
596	      vec_foreach_index (idx, reass->cached_buffers)
597	      {
598		u32 bi0 = vec_elt (reass->cached_buffers, idx);
599		vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
600		ip0 =
601		  (ip4_header_t *) u8_ptr_add (vlib_buffer_get_current (b0),
602					       (is_output_feature ? 1 : 0) *
603					       vnet_buffer (b0)->
604					       ip.save_rewrite_length);
605		u32 next0 = IP4_SV_REASSEMBLY_NEXT_INPUT;
606		if (is_feature)
607		  {
608		    vnet_feature_next (&next0, b0);
609		  }
610		if (is_custom)
611		  {
612		    next0 = vnet_buffer (b0)->ip.reass.next_index;
613		  }
614		if (0 == n_left_to_next)
615		  {
616		    vlib_put_next_frame (vm, node, next_index,
617					 n_left_to_next);
618		    vlib_get_next_frame (vm, node, next_index, to_next,
619					 n_left_to_next);
620		  }
621		to_next[0] = bi0;
622		to_next += 1;
623		n_left_to_next -= 1;
624		vnet_buffer (b0)->ip.reass.is_non_first_fragment =
625		  ! !ip4_get_fragment_offset (ip0);
626		vnet_buffer (b0)->ip.reass.ip_proto = reass->ip_proto;
627		vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags =
628		  reass->icmp_type_or_tcp_flags;
629		vnet_buffer (b0)->ip.reass.tcp_ack_number =
630		  reass->tcp_ack_number;
631		vnet_buffer (b0)->ip.reass.tcp_seq_number =
632		  reass->tcp_seq_number;
633		vnet_buffer (b0)->ip.reass.l4_src_port = reass->l4_src_port;
634		vnet_buffer (b0)->ip.reass.l4_dst_port = reass->l4_dst_port;
635		if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
636		  {
637		    ip4_sv_reass_add_trace (vm, node, rm, reass, bi0,
638					    REASS_FRAGMENT_FORWARD,
639					    reass->ip_proto,
640					    reass->l4_src_port,
641					    reass->l4_dst_port);
642		  }
643		vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
644						 to_next, n_left_to_next, bi0,
645						 next0);
646	      }
647	      _vec_len (reass->cached_buffers) = 0;	// buffers are owned by frame now
648	    }
649	  goto next_packet;
650
651	packet_enqueue:
652	  to_next[0] = bi0;
653	  to_next += 1;
654	  n_left_to_next -= 1;
655	  if (is_feature && IP4_ERROR_NONE == error0)
656	    {
657	      b0 = vlib_get_buffer (vm, bi0);
658	      vnet_feature_next (&next0, b0);
659	    }
660	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
661					   to_next, n_left_to_next,
662					   bi0, next0);
663
664	next_packet:
665	  from += 1;
666	  n_left_from -= 1;
667	}
668
669      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
670    }
671
672  clib_spinlock_unlock (&rt->lock);
673  return frame->n_vectors;
674}
675
676static char *ip4_sv_reass_error_strings[] = {
677#define _(sym, string) string,
678  foreach_ip4_error
679#undef _
680};
681
682VLIB_NODE_FN (ip4_sv_reass_node) (vlib_main_t * vm,
683				  vlib_node_runtime_t * node,
684				  vlib_frame_t * frame)
685{
686  return ip4_sv_reass_inline (vm, node, frame, false /* is_feature */ ,
687			      false /* is_output_feature */ ,
688			      false /* is_custom */ );
689}
690
691/* *INDENT-OFF* */
692VLIB_REGISTER_NODE (ip4_sv_reass_node) = {
693    .name = "ip4-sv-reassembly",
694    .vector_size = sizeof (u32),
695    .format_trace = format_ip4_sv_reass_trace,
696    .n_errors = ARRAY_LEN (ip4_sv_reass_error_strings),
697    .error_strings = ip4_sv_reass_error_strings,
698    .n_next_nodes = IP4_SV_REASSEMBLY_N_NEXT,
699    .next_nodes =
700        {
701                [IP4_SV_REASSEMBLY_NEXT_INPUT] = "ip4-input",
702                [IP4_SV_REASSEMBLY_NEXT_DROP] = "ip4-drop",
703                [IP4_SV_REASSEMBLY_NEXT_HANDOFF] = "ip4-sv-reassembly-handoff",
704
705        },
706};
707/* *INDENT-ON* */
708
709VLIB_NODE_FN (ip4_sv_reass_node_feature) (vlib_main_t * vm,
710					  vlib_node_runtime_t * node,
711					  vlib_frame_t * frame)
712{
713  return ip4_sv_reass_inline (vm, node, frame, true /* is_feature */ ,
714			      false /* is_output_feature */ ,
715			      false /* is_custom */ );
716}
717
718/* *INDENT-OFF* */
719VLIB_REGISTER_NODE (ip4_sv_reass_node_feature) = {
720    .name = "ip4-sv-reassembly-feature",
721    .vector_size = sizeof (u32),
722    .format_trace = format_ip4_sv_reass_trace,
723    .n_errors = ARRAY_LEN (ip4_sv_reass_error_strings),
724    .error_strings = ip4_sv_reass_error_strings,
725    .n_next_nodes = IP4_SV_REASSEMBLY_N_NEXT,
726    .next_nodes =
727        {
728                [IP4_SV_REASSEMBLY_NEXT_INPUT] = "ip4-input",
729                [IP4_SV_REASSEMBLY_NEXT_DROP] = "ip4-drop",
730                [IP4_SV_REASSEMBLY_NEXT_HANDOFF] = "ip4-sv-reass-feature-hoff",
731        },
732};
733/* *INDENT-ON* */
734
735/* *INDENT-OFF* */
736VNET_FEATURE_INIT (ip4_sv_reass_feature) = {
737    .arc_name = "ip4-unicast",
738    .node_name = "ip4-sv-reassembly-feature",
739    .runs_before = VNET_FEATURES ("ip4-lookup"),
740    .runs_after = 0,
741};
742/* *INDENT-ON* */
743
744VLIB_NODE_FN (ip4_sv_reass_node_output_feature) (vlib_main_t * vm,
745						 vlib_node_runtime_t * node,
746						 vlib_frame_t * frame)
747{
748  return ip4_sv_reass_inline (vm, node, frame, true /* is_feature */ ,
749			      true /* is_output_feature */ ,
750			      false /* is_custom */ );
751}
752
753
754/* *INDENT-OFF* */
755VLIB_REGISTER_NODE (ip4_sv_reass_node_output_feature) = {
756    .name = "ip4-sv-reassembly-output-feature",
757    .vector_size = sizeof (u32),
758    .format_trace = format_ip4_sv_reass_trace,
759    .n_errors = ARRAY_LEN (ip4_sv_reass_error_strings),
760    .error_strings = ip4_sv_reass_error_strings,
761    .n_next_nodes = IP4_SV_REASSEMBLY_N_NEXT,
762    .next_nodes =
763        {
764                [IP4_SV_REASSEMBLY_NEXT_INPUT] = "ip4-input",
765                [IP4_SV_REASSEMBLY_NEXT_DROP] = "ip4-drop",
766                [IP4_SV_REASSEMBLY_NEXT_HANDOFF] = "ip4-sv-reass-feature-hoff",
767        },
768};
769/* *INDENT-ON* */
770
771/* *INDENT-OFF* */
772VNET_FEATURE_INIT (ip4_sv_reass_output_feature) = {
773    .arc_name = "ip4-output",
774    .node_name = "ip4-sv-reassembly-output-feature",
775    .runs_before = 0,
776    .runs_after = 0,
777};
778/* *INDENT-ON* */
779
780/* *INDENT-OFF* */
781VLIB_REGISTER_NODE (ip4_sv_reass_custom_node) = {
782    .name = "ip4-sv-reassembly-custom-next",
783    .vector_size = sizeof (u32),
784    .format_trace = format_ip4_sv_reass_trace,
785    .n_errors = ARRAY_LEN (ip4_sv_reass_error_strings),
786    .error_strings = ip4_sv_reass_error_strings,
787    .n_next_nodes = IP4_SV_REASSEMBLY_N_NEXT,
788    .next_nodes =
789        {
790                [IP4_SV_REASSEMBLY_NEXT_INPUT] = "ip4-input",
791                [IP4_SV_REASSEMBLY_NEXT_DROP] = "ip4-drop",
792                [IP4_SV_REASSEMBLY_NEXT_HANDOFF] = "ip4-sv-reassembly-handoff",
793
794        },
795};
796/* *INDENT-ON* */
797
798VLIB_NODE_FN (ip4_sv_reass_custom_node) (vlib_main_t * vm,
799					 vlib_node_runtime_t * node,
800					 vlib_frame_t * frame)
801{
802  return ip4_sv_reass_inline (vm, node, frame, false /* is_feature */ ,
803			      false /* is_output_feature */ ,
804			      true /* is_custom */ );
805}
806
807#ifndef CLIB_MARCH_VARIANT
808always_inline u32
809ip4_sv_reass_get_nbuckets ()
810{
811  ip4_sv_reass_main_t *rm = &ip4_sv_reass_main;
812  u32 nbuckets;
813  u8 i;
814
815  nbuckets = (u32) (rm->max_reass_n / IP4_SV_REASS_HT_LOAD_FACTOR);
816
817  for (i = 0; i < 31; i++)
818    if ((1 << i) >= nbuckets)
819      break;
820  nbuckets = 1 << i;
821
822  return nbuckets;
823}
824#endif /* CLIB_MARCH_VARIANT */
825
826typedef enum
827{
828  IP4_EVENT_CONFIG_CHANGED = 1,
829} ip4_sv_reass_event_t;
830
831typedef struct
832{
833  int failure;
834  clib_bihash_16_8_t *new_hash;
835} ip4_rehash_cb_ctx;
836
837#ifndef CLIB_MARCH_VARIANT
838static int
839ip4_rehash_cb (clib_bihash_kv_16_8_t * kv, void *_ctx)
840{
841  ip4_rehash_cb_ctx *ctx = _ctx;
842  if (clib_bihash_add_del_16_8 (ctx->new_hash, kv, 1))
843    {
844      ctx->failure = 1;
845    }
846  return (BIHASH_WALK_CONTINUE);
847}
848
849static void
850ip4_sv_reass_set_params (u32 timeout_ms, u32 max_reassemblies,
851			 u32 max_reassembly_length,
852			 u32 expire_walk_interval_ms)
853{
854  ip4_sv_reass_main.timeout_ms = timeout_ms;
855  ip4_sv_reass_main.timeout = (f64) timeout_ms / (f64) MSEC_PER_SEC;
856  ip4_sv_reass_main.max_reass_n = max_reassemblies;
857  ip4_sv_reass_main.max_reass_len = max_reassembly_length;
858  ip4_sv_reass_main.expire_walk_interval_ms = expire_walk_interval_ms;
859}
860
861vnet_api_error_t
862ip4_sv_reass_set (u32 timeout_ms, u32 max_reassemblies,
863		  u32 max_reassembly_length, u32 expire_walk_interval_ms)
864{
865  u32 old_nbuckets = ip4_sv_reass_get_nbuckets ();
866  ip4_sv_reass_set_params (timeout_ms, max_reassemblies,
867			   max_reassembly_length, expire_walk_interval_ms);
868  vlib_process_signal_event (ip4_sv_reass_main.vlib_main,
869			     ip4_sv_reass_main.ip4_sv_reass_expire_node_idx,
870			     IP4_EVENT_CONFIG_CHANGED, 0);
871  u32 new_nbuckets = ip4_sv_reass_get_nbuckets ();
872  if (ip4_sv_reass_main.max_reass_n > 0 && new_nbuckets > old_nbuckets)
873    {
874      clib_bihash_16_8_t new_hash;
875      clib_memset (&new_hash, 0, sizeof (new_hash));
876      ip4_rehash_cb_ctx ctx;
877      ctx.failure = 0;
878      ctx.new_hash = &new_hash;
879      clib_bihash_init_16_8 (&new_hash, "ip4-dr", new_nbuckets,
880			     new_nbuckets * 1024);
881      clib_bihash_foreach_key_value_pair_16_8 (&ip4_sv_reass_main.hash,
882					       ip4_rehash_cb, &ctx);
883      if (ctx.failure)
884	{
885	  clib_bihash_free_16_8 (&new_hash);
886	  return -1;
887	}
888      else
889	{
890	  clib_bihash_free_16_8 (&ip4_sv_reass_main.hash);
891	  clib_memcpy_fast (&ip4_sv_reass_main.hash, &new_hash,
892			    sizeof (ip4_sv_reass_main.hash));
893	  clib_bihash_copied (&ip4_sv_reass_main.hash, &new_hash);
894	}
895    }
896  return 0;
897}
898
899vnet_api_error_t
900ip4_sv_reass_get (u32 * timeout_ms, u32 * max_reassemblies,
901		  u32 * max_reassembly_length, u32 * expire_walk_interval_ms)
902{
903  *timeout_ms = ip4_sv_reass_main.timeout_ms;
904  *max_reassemblies = ip4_sv_reass_main.max_reass_n;
905  *max_reassembly_length = ip4_sv_reass_main.max_reass_len;
906  *expire_walk_interval_ms = ip4_sv_reass_main.expire_walk_interval_ms;
907  return 0;
908}
909
910static clib_error_t *
911ip4_sv_reass_init_function (vlib_main_t * vm)
912{
913  ip4_sv_reass_main_t *rm = &ip4_sv_reass_main;
914  clib_error_t *error = 0;
915  u32 nbuckets;
916  vlib_node_t *node;
917
918  rm->vlib_main = vm;
919  rm->vnet_main = vnet_get_main ();
920
921  vec_validate (rm->per_thread_data, vlib_num_workers ());
922  ip4_sv_reass_per_thread_t *rt;
923  vec_foreach (rt, rm->per_thread_data)
924  {
925    clib_spinlock_init (&rt->lock);
926    pool_alloc (rt->pool, rm->max_reass_n);
927    rt->lru_first = rt->lru_last = ~0;
928  }
929
930  node = vlib_get_node_by_name (vm, (u8 *) "ip4-sv-reassembly-expire-walk");
931  ASSERT (node);
932  rm->ip4_sv_reass_expire_node_idx = node->index;
933
934  ip4_sv_reass_set_params (IP4_SV_REASS_TIMEOUT_DEFAULT_MS,
935			   IP4_SV_REASS_MAX_REASSEMBLIES_DEFAULT,
936			   IP4_SV_REASS_MAX_REASSEMBLY_LENGTH_DEFAULT,
937			   IP4_SV_REASS_EXPIRE_WALK_INTERVAL_DEFAULT_MS);
938
939  nbuckets = ip4_sv_reass_get_nbuckets ();
940  clib_bihash_init_16_8 (&rm->hash, "ip4-dr", nbuckets, nbuckets * 1024);
941
942  node = vlib_get_node_by_name (vm, (u8 *) "ip4-drop");
943  ASSERT (node);
944  rm->ip4_drop_idx = node->index;
945
946  rm->fq_index = vlib_frame_queue_main_init (ip4_sv_reass_node.index, 0);
947  rm->fq_feature_index =
948    vlib_frame_queue_main_init (ip4_sv_reass_node_feature.index, 0);
949
950  rm->feature_use_refcount_per_intf = NULL;
951  rm->output_feature_use_refcount_per_intf = NULL;
952
953  return error;
954}
955
956VLIB_INIT_FUNCTION (ip4_sv_reass_init_function);
957#endif /* CLIB_MARCH_VARIANT */
958
959static uword
960ip4_sv_reass_walk_expired (vlib_main_t * vm,
961			   vlib_node_runtime_t * node, vlib_frame_t * f)
962{
963  ip4_sv_reass_main_t *rm = &ip4_sv_reass_main;
964  uword event_type, *event_data = 0;
965
966  while (true)
967    {
968      vlib_process_wait_for_event_or_clock (vm,
969					    (f64)
970					    rm->expire_walk_interval_ms /
971					    (f64) MSEC_PER_SEC);
972      event_type = vlib_process_get_events (vm, &event_data);
973
974      switch (event_type)
975	{
976	case ~0:		/* no events => timeout */
977	  /* nothing to do here */
978	  break;
979	case IP4_EVENT_CONFIG_CHANGED:
980	  break;
981	default:
982	  clib_warning ("BUG: event type 0x%wx", event_type);
983	  break;
984	}
985      f64 now = vlib_time_now (vm);
986
987      ip4_sv_reass_t *reass;
988      int *pool_indexes_to_free = NULL;
989
990      uword thread_index = 0;
991      int index;
992      const uword nthreads = vlib_num_workers () + 1;
993      for (thread_index = 0; thread_index < nthreads; ++thread_index)
994	{
995	  ip4_sv_reass_per_thread_t *rt = &rm->per_thread_data[thread_index];
996	  clib_spinlock_lock (&rt->lock);
997
998	  vec_reset_length (pool_indexes_to_free);
999          /* *INDENT-OFF* */
1000          pool_foreach_index (index, rt->pool, ({
1001                                reass = pool_elt_at_index (rt->pool, index);
1002                                if (now > reass->last_heard + rm->timeout)
1003                                  {
1004                                    vec_add1 (pool_indexes_to_free, index);
1005                                  }
1006                              }));
1007          /* *INDENT-ON* */
1008	  int *i;
1009          /* *INDENT-OFF* */
1010          vec_foreach (i, pool_indexes_to_free)
1011          {
1012            ip4_sv_reass_t *reass = pool_elt_at_index (rt->pool, i[0]);
1013            ip4_sv_reass_free (vm, rm, rt, reass);
1014          }
1015          /* *INDENT-ON* */
1016
1017	  clib_spinlock_unlock (&rt->lock);
1018	}
1019
1020      vec_free (pool_indexes_to_free);
1021      if (event_data)
1022	{
1023	  _vec_len (event_data) = 0;
1024	}
1025    }
1026
1027  return 0;
1028}
1029
1030/* *INDENT-OFF* */
1031VLIB_REGISTER_NODE (ip4_sv_reass_expire_node) = {
1032    .function = ip4_sv_reass_walk_expired,
1033    .type = VLIB_NODE_TYPE_PROCESS,
1034    .name = "ip4-sv-reassembly-expire-walk",
1035    .format_trace = format_ip4_sv_reass_trace,
1036    .n_errors = ARRAY_LEN (ip4_sv_reass_error_strings),
1037    .error_strings = ip4_sv_reass_error_strings,
1038
1039};
1040/* *INDENT-ON* */
1041
1042static u8 *
1043format_ip4_sv_reass_key (u8 * s, va_list * args)
1044{
1045  ip4_sv_reass_key_t *key = va_arg (*args, ip4_sv_reass_key_t *);
1046  s =
1047    format (s,
1048	    "xx_id: %u, src: %U, dst: %U, frag_id: %u, proto: %u",
1049	    key->xx_id, format_ip4_address, &key->src, format_ip4_address,
1050	    &key->dst, clib_net_to_host_u16 (key->frag_id), key->proto);
1051  return s;
1052}
1053
1054static u8 *
1055format_ip4_sv_reass (u8 * s, va_list * args)
1056{
1057  vlib_main_t *vm = va_arg (*args, vlib_main_t *);
1058  ip4_sv_reass_t *reass = va_arg (*args, ip4_sv_reass_t *);
1059
1060  s = format (s, "ID: %lu, key: %U trace_op_counter: %u\n",
1061	      reass->id, format_ip4_sv_reass_key, &reass->key,
1062	      reass->trace_op_counter);
1063
1064  vlib_buffer_t *b;
1065  u32 *bip;
1066  u32 counter = 0;
1067  vec_foreach (bip, reass->cached_buffers)
1068  {
1069    u32 bi = *bip;
1070    do
1071      {
1072	b = vlib_get_buffer (vm, bi);
1073	s = format (s, "  #%03u: bi: %u, ", counter, bi);
1074	++counter;
1075	bi = b->next_buffer;
1076      }
1077    while (b->flags & VLIB_BUFFER_NEXT_PRESENT);
1078  }
1079  return s;
1080}
1081
1082static clib_error_t *
1083show_ip4_reass (vlib_main_t * vm,
1084		unformat_input_t * input,
1085		CLIB_UNUSED (vlib_cli_command_t * lmd))
1086{
1087  ip4_sv_reass_main_t *rm = &ip4_sv_reass_main;
1088
1089  vlib_cli_output (vm, "---------------------");
1090  vlib_cli_output (vm, "IP4 reassembly status");
1091  vlib_cli_output (vm, "---------------------");
1092  bool details = false;
1093  if (unformat (input, "details"))
1094    {
1095      details = true;
1096    }
1097
1098  u32 sum_reass_n = 0;
1099  ip4_sv_reass_t *reass;
1100  uword thread_index;
1101  const uword nthreads = vlib_num_workers () + 1;
1102  for (thread_index = 0; thread_index < nthreads; ++thread_index)
1103    {
1104      ip4_sv_reass_per_thread_t *rt = &rm->per_thread_data[thread_index];
1105      clib_spinlock_lock (&rt->lock);
1106      if (details)
1107	{
1108          /* *INDENT-OFF* */
1109          pool_foreach (reass, rt->pool, {
1110            vlib_cli_output (vm, "%U", format_ip4_sv_reass, vm, reass);
1111          });
1112          /* *INDENT-ON* */
1113	}
1114      sum_reass_n += rt->reass_n;
1115      clib_spinlock_unlock (&rt->lock);
1116    }
1117  vlib_cli_output (vm, "---------------------");
1118  vlib_cli_output (vm, "Current IP4 reassemblies count: %lu\n",
1119		   (long unsigned) sum_reass_n);
1120  vlib_cli_output (vm,
1121		   "Maximum configured concurrent shallow virtual IP4 reassemblies per worker-thread: %lu\n",
1122		   (long unsigned) rm->max_reass_n);
1123  vlib_cli_output (vm,
1124		   "Maximum configured shallow virtual IP4 reassembly timeout: %lums\n",
1125		   (long unsigned) rm->timeout_ms);
1126  vlib_cli_output (vm,
1127		   "Maximum configured shallow virtual IP4 reassembly expire walk interval: %lums\n",
1128		   (long unsigned) rm->expire_walk_interval_ms);
1129  return 0;
1130}
1131
1132/* *INDENT-OFF* */
1133VLIB_CLI_COMMAND (show_ip4_sv_reass_cmd, static) = {
1134    .path = "show ip4-sv-reassembly",
1135    .short_help = "show ip4-sv-reassembly [details]",
1136    .function = show_ip4_reass,
1137};
1138/* *INDENT-ON* */
1139
1140#ifndef CLIB_MARCH_VARIANT
1141vnet_api_error_t
1142ip4_sv_reass_enable_disable (u32 sw_if_index, u8 enable_disable)
1143{
1144  return ip4_sv_reass_enable_disable_with_refcnt (sw_if_index,
1145						  enable_disable);
1146}
1147#endif /* CLIB_MARCH_VARIANT */
1148
1149
1150#define foreach_ip4_sv_reass_handoff_error                       \
1151_(CONGESTION_DROP, "congestion drop")
1152
1153
1154typedef enum
1155{
1156#define _(sym,str) IP4_SV_REASSEMBLY_HANDOFF_ERROR_##sym,
1157  foreach_ip4_sv_reass_handoff_error
1158#undef _
1159    IP4_SV_REASSEMBLY_HANDOFF_N_ERROR,
1160} ip4_sv_reass_handoff_error_t;
1161
1162static char *ip4_sv_reass_handoff_error_strings[] = {
1163#define _(sym,string) string,
1164  foreach_ip4_sv_reass_handoff_error
1165#undef _
1166};
1167
1168typedef struct
1169{
1170  u32 next_worker_index;
1171} ip4_sv_reass_handoff_trace_t;
1172
1173static u8 *
1174format_ip4_sv_reass_handoff_trace (u8 * s, va_list * args)
1175{
1176  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1177  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1178  ip4_sv_reass_handoff_trace_t *t =
1179    va_arg (*args, ip4_sv_reass_handoff_trace_t *);
1180
1181  s =
1182    format (s, "ip4-sv-reassembly-handoff: next-worker %d",
1183	    t->next_worker_index);
1184
1185  return s;
1186}
1187
1188always_inline uword
1189ip4_sv_reass_handoff_node_inline (vlib_main_t * vm,
1190				  vlib_node_runtime_t * node,
1191				  vlib_frame_t * frame, bool is_feature)
1192{
1193  ip4_sv_reass_main_t *rm = &ip4_sv_reass_main;
1194
1195  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
1196  u32 n_enq, n_left_from, *from;
1197  u16 thread_indices[VLIB_FRAME_SIZE], *ti;
1198  u32 fq_index;
1199
1200  from = vlib_frame_vector_args (frame);
1201  n_left_from = frame->n_vectors;
1202  vlib_get_buffers (vm, from, bufs, n_left_from);
1203
1204  b = bufs;
1205  ti = thread_indices;
1206
1207  fq_index = (is_feature) ? rm->fq_feature_index : rm->fq_index;
1208
1209  while (n_left_from > 0)
1210    {
1211      ti[0] = vnet_buffer (b[0])->ip.reass.owner_thread_index;
1212
1213      if (PREDICT_FALSE
1214	  ((node->flags & VLIB_NODE_FLAG_TRACE)
1215	   && (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
1216	{
1217	  ip4_sv_reass_handoff_trace_t *t =
1218	    vlib_add_trace (vm, node, b[0], sizeof (*t));
1219	  t->next_worker_index = ti[0];
1220	}
1221
1222      n_left_from -= 1;
1223      ti += 1;
1224      b += 1;
1225    }
1226  n_enq =
1227    vlib_buffer_enqueue_to_thread (vm, fq_index, from, thread_indices,
1228				   frame->n_vectors, 1);
1229
1230  if (n_enq < frame->n_vectors)
1231    vlib_node_increment_counter (vm, node->node_index,
1232				 IP4_SV_REASSEMBLY_HANDOFF_ERROR_CONGESTION_DROP,
1233				 frame->n_vectors - n_enq);
1234  return frame->n_vectors;
1235}
1236
1237VLIB_NODE_FN (ip4_sv_reass_handoff_node) (vlib_main_t * vm,
1238					  vlib_node_runtime_t * node,
1239					  vlib_frame_t * frame)
1240{
1241  return ip4_sv_reass_handoff_node_inline (vm, node, frame,
1242					   false /* is_feature */ );
1243}
1244
1245
1246/* *INDENT-OFF* */
1247VLIB_REGISTER_NODE (ip4_sv_reass_handoff_node) = {
1248  .name = "ip4-sv-reassembly-handoff",
1249  .vector_size = sizeof (u32),
1250  .n_errors = ARRAY_LEN(ip4_sv_reass_handoff_error_strings),
1251  .error_strings = ip4_sv_reass_handoff_error_strings,
1252  .format_trace = format_ip4_sv_reass_handoff_trace,
1253
1254  .n_next_nodes = 1,
1255
1256  .next_nodes = {
1257    [0] = "error-drop",
1258  },
1259};
1260/* *INDENT-ON* */
1261
1262
1263/* *INDENT-OFF* */
1264VLIB_NODE_FN (ip4_sv_reass_feature_handoff_node) (vlib_main_t * vm,
1265						    vlib_node_runtime_t *
1266						    node,
1267						    vlib_frame_t * frame)
1268{
1269  return ip4_sv_reass_handoff_node_inline (vm, node, frame,
1270					     true /* is_feature */ );
1271}
1272/* *INDENT-ON* */
1273
1274
1275/* *INDENT-OFF* */
1276VLIB_REGISTER_NODE (ip4_sv_reass_feature_handoff_node) = {
1277  .name = "ip4-sv-reass-feature-hoff",
1278  .vector_size = sizeof (u32),
1279  .n_errors = ARRAY_LEN(ip4_sv_reass_handoff_error_strings),
1280  .error_strings = ip4_sv_reass_handoff_error_strings,
1281  .format_trace = format_ip4_sv_reass_handoff_trace,
1282
1283  .n_next_nodes = 1,
1284
1285  .next_nodes = {
1286    [0] = "error-drop",
1287  },
1288};
1289/* *INDENT-ON* */
1290
1291#ifndef CLIB_MARCH_VARIANT
1292int
1293ip4_sv_reass_enable_disable_with_refcnt (u32 sw_if_index, int is_enable)
1294{
1295  ip4_sv_reass_main_t *rm = &ip4_sv_reass_main;
1296  vec_validate (rm->feature_use_refcount_per_intf, sw_if_index);
1297  if (is_enable)
1298    {
1299      if (!rm->feature_use_refcount_per_intf[sw_if_index])
1300	{
1301	  ++rm->feature_use_refcount_per_intf[sw_if_index];
1302	  return vnet_feature_enable_disable ("ip4-unicast",
1303					      "ip4-sv-reassembly-feature",
1304					      sw_if_index, 1, 0, 0);
1305	}
1306      ++rm->feature_use_refcount_per_intf[sw_if_index];
1307    }
1308  else
1309    {
1310      if (rm->feature_use_refcount_per_intf[sw_if_index])
1311	--rm->feature_use_refcount_per_intf[sw_if_index];
1312      if (!rm->feature_use_refcount_per_intf[sw_if_index])
1313	return vnet_feature_enable_disable ("ip4-unicast",
1314					    "ip4-sv-reassembly-feature",
1315					    sw_if_index, 0, 0, 0);
1316    }
1317  return 0;
1318}
1319
1320uword
1321ip4_sv_reass_custom_register_next_node (uword node_index)
1322{
1323  return vlib_node_add_next (vlib_get_main (), ip4_sv_reass_custom_node.index,
1324			     node_index);
1325}
1326
1327int
1328ip4_sv_reass_output_enable_disable_with_refcnt (u32 sw_if_index,
1329						int is_enable)
1330{
1331  ip4_sv_reass_main_t *rm = &ip4_sv_reass_main;
1332  vec_validate (rm->output_feature_use_refcount_per_intf, sw_if_index);
1333  if (is_enable)
1334    {
1335      if (!rm->output_feature_use_refcount_per_intf[sw_if_index])
1336	{
1337	  ++rm->output_feature_use_refcount_per_intf[sw_if_index];
1338	  return vnet_feature_enable_disable ("ip4-output",
1339					      "ip4-sv-reassembly-output-feature",
1340					      sw_if_index, 1, 0, 0);
1341	}
1342      ++rm->output_feature_use_refcount_per_intf[sw_if_index];
1343    }
1344  else
1345    {
1346      if (rm->output_feature_use_refcount_per_intf[sw_if_index])
1347	--rm->output_feature_use_refcount_per_intf[sw_if_index];
1348      if (!rm->output_feature_use_refcount_per_intf[sw_if_index])
1349	return vnet_feature_enable_disable ("ip4-output",
1350					    "ip4-sv-reassembly-output-feature",
1351					    sw_if_index, 0, 0, 0);
1352    }
1353  return 0;
1354}
1355#endif
1356
1357/*
1358 * fd.io coding-style-patch-verification: ON
1359 *
1360 * Local Variables:
1361 * eval: (c-set-style "gnu")
1362 * End:
1363 */
1364