ip4_sv_reass.c revision 8ad070e1
1/*
2 * Copyright (c) 2017 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16/**
17 * @file
18 * @brief IPv4 Shallow Virtual Reassembly.
19 *
20 * This file contains the source code for IPv4 Shallow Virtual reassembly.
21 */
22
23#include <vppinfra/vec.h>
24#include <vnet/vnet.h>
25#include <vnet/ip/ip.h>
26#include <vnet/ip/ip4_to_ip6.h>
27#include <vppinfra/fifo.h>
28#include <vppinfra/bihash_16_8.h>
29#include <vnet/ip/reass/ip4_sv_reass.h>
30
31#define MSEC_PER_SEC 1000
32#define IP4_SV_REASS_TIMEOUT_DEFAULT_MS 100
33#define IP4_SV_REASS_EXPIRE_WALK_INTERVAL_DEFAULT_MS 10000	// 10 seconds default
34#define IP4_SV_REASS_MAX_REASSEMBLIES_DEFAULT 1024
35#define IP4_SV_REASS_MAX_REASSEMBLY_LENGTH_DEFAULT 3
36#define IP4_SV_REASS_HT_LOAD_FACTOR (0.75)
37
38typedef enum
39{
40  IP4_SV_REASS_RC_OK,
41  IP4_SV_REASS_RC_TOO_MANY_FRAGMENTS,
42  IP4_SV_REASS_RC_UNSUPP_IP_PROTO,
43} ip4_sv_reass_rc_t;
44
45typedef struct
46{
47  union
48  {
49    struct
50    {
51      u32 xx_id;
52      ip4_address_t src;
53      ip4_address_t dst;
54      u16 frag_id;
55      u8 proto;
56      u8 unused;
57    };
58    u64 as_u64[2];
59  };
60} ip4_sv_reass_key_t;
61
62typedef union
63{
64  struct
65  {
66    u32 reass_index;
67    u32 thread_index;
68  };
69  u64 as_u64;
70} ip4_sv_reass_val_t;
71
72typedef union
73{
74  struct
75  {
76    ip4_sv_reass_key_t k;
77    ip4_sv_reass_val_t v;
78  };
79  clib_bihash_kv_16_8_t kv;
80} ip4_sv_reass_kv_t;
81
82typedef struct
83{
84  // hash table key
85  ip4_sv_reass_key_t key;
86  // time when last packet was received
87  f64 last_heard;
88  // internal id of this reassembly
89  u64 id;
90  // trace operation counter
91  u32 trace_op_counter;
92  // minimum fragment length for this reassembly - used to estimate MTU
93  u16 min_fragment_length;
94  // buffer indexes of buffers in this reassembly in chronological order -
95  // including overlaps and duplicate fragments
96  u32 *cached_buffers;
97  // set to true when this reassembly is completed
98  bool is_complete;
99  // ip protocol
100  u8 ip_proto;
101  u8 icmp_type_or_tcp_flags;
102  u32 tcp_ack_number;
103  u32 tcp_seq_number;
104  // l4 src port
105  u16 l4_src_port;
106  // l4 dst port
107  u16 l4_dst_port;
108  u32 next_index;
109  // lru indexes
110  u32 lru_prev;
111  u32 lru_next;
112} ip4_sv_reass_t;
113
114typedef struct
115{
116  ip4_sv_reass_t *pool;
117  u32 reass_n;
118  u32 id_counter;
119  clib_spinlock_t lock;
120  // lru indexes
121  u32 lru_first;
122  u32 lru_last;
123
124} ip4_sv_reass_per_thread_t;
125
126typedef struct
127{
128  // IPv4 config
129  u32 timeout_ms;
130  f64 timeout;
131  u32 expire_walk_interval_ms;
132  // maximum number of fragments in one reassembly
133  u32 max_reass_len;
134  // maximum number of reassemblies
135  u32 max_reass_n;
136
137  // IPv4 runtime
138  clib_bihash_16_8_t hash;
139  // per-thread data
140  ip4_sv_reass_per_thread_t *per_thread_data;
141
142  // convenience
143  vlib_main_t *vlib_main;
144  vnet_main_t *vnet_main;
145
146  // node index of ip4-drop node
147  u32 ip4_drop_idx;
148  u32 ip4_sv_reass_expire_node_idx;
149
150  /** Worker handoff */
151  u32 fq_index;
152  u32 fq_feature_index;
153
154  // reference count for enabling/disabling feature - per interface
155  u32 *feature_use_refcount_per_intf;
156
157  // reference count for enabling/disabling feature - per interface
158  u32 *output_feature_use_refcount_per_intf;
159
160} ip4_sv_reass_main_t;
161
162extern ip4_sv_reass_main_t ip4_sv_reass_main;
163
164#ifndef CLIB_MARCH_VARIANT
165ip4_sv_reass_main_t ip4_sv_reass_main;
166#endif /* CLIB_MARCH_VARIANT */
167
168typedef enum
169{
170  IP4_SV_REASSEMBLY_NEXT_INPUT,
171  IP4_SV_REASSEMBLY_NEXT_DROP,
172  IP4_SV_REASSEMBLY_NEXT_HANDOFF,
173  IP4_SV_REASSEMBLY_N_NEXT,
174} ip4_sv_reass_next_t;
175
176typedef enum
177{
178  REASS_FRAGMENT_CACHE,
179  REASS_FINISH,
180  REASS_FRAGMENT_FORWARD,
181  REASS_PASSTHROUGH,
182} ip4_sv_reass_trace_operation_e;
183
184typedef struct
185{
186  ip4_sv_reass_trace_operation_e action;
187  u32 reass_id;
188  u32 op_id;
189  u8 ip_proto;
190  u16 l4_src_port;
191  u16 l4_dst_port;
192} ip4_sv_reass_trace_t;
193
194extern vlib_node_registration_t ip4_sv_reass_node;
195extern vlib_node_registration_t ip4_sv_reass_node_feature;
196
197static u8 *
198format_ip4_sv_reass_trace (u8 * s, va_list * args)
199{
200  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
201  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
202  ip4_sv_reass_trace_t *t = va_arg (*args, ip4_sv_reass_trace_t *);
203  if (REASS_PASSTHROUGH != t->action)
204    {
205      s = format (s, "reass id: %u, op id: %u ", t->reass_id, t->op_id);
206    }
207  switch (t->action)
208    {
209    case REASS_FRAGMENT_CACHE:
210      s = format (s, "[cached]");
211      break;
212    case REASS_FINISH:
213      s =
214	format (s, "[finish, ip proto=%u, src_port=%u, dst_port=%u]",
215		t->ip_proto, clib_net_to_host_u16 (t->l4_src_port),
216		clib_net_to_host_u16 (t->l4_dst_port));
217      break;
218    case REASS_FRAGMENT_FORWARD:
219      s =
220	format (s, "[forward, ip proto=%u, src_port=%u, dst_port=%u]",
221		t->ip_proto, clib_net_to_host_u16 (t->l4_src_port),
222		clib_net_to_host_u16 (t->l4_dst_port));
223      break;
224    case REASS_PASSTHROUGH:
225      s = format (s, "[not-fragmented]");
226      break;
227    }
228  return s;
229}
230
231static void
232ip4_sv_reass_add_trace (vlib_main_t * vm, vlib_node_runtime_t * node,
233			ip4_sv_reass_main_t * rm, ip4_sv_reass_t * reass,
234			u32 bi, ip4_sv_reass_trace_operation_e action,
235			u32 ip_proto, u16 l4_src_port, u16 l4_dst_port)
236{
237  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
238  ip4_sv_reass_trace_t *t = vlib_add_trace (vm, node, b, sizeof (t[0]));
239  if (reass)
240    {
241      t->reass_id = reass->id;
242      t->op_id = reass->trace_op_counter;
243      ++reass->trace_op_counter;
244    }
245  t->action = action;
246  t->ip_proto = ip_proto;
247  t->l4_src_port = l4_src_port;
248  t->l4_dst_port = l4_dst_port;
249#if 0
250  static u8 *s = NULL;
251  s = format (s, "%U", format_ip4_sv_reass_trace, NULL, NULL, t);
252  printf ("%.*s\n", vec_len (s), s);
253  fflush (stdout);
254  vec_reset_length (s);
255#endif
256}
257
258
259always_inline void
260ip4_sv_reass_free (vlib_main_t * vm, ip4_sv_reass_main_t * rm,
261		   ip4_sv_reass_per_thread_t * rt, ip4_sv_reass_t * reass)
262{
263  clib_bihash_kv_16_8_t kv;
264  kv.key[0] = reass->key.as_u64[0];
265  kv.key[1] = reass->key.as_u64[1];
266  clib_bihash_add_del_16_8 (&rm->hash, &kv, 0);
267  vlib_buffer_free (vm, reass->cached_buffers,
268		    vec_len (reass->cached_buffers));
269  vec_free (reass->cached_buffers);
270  reass->cached_buffers = NULL;
271  if (~0 != reass->lru_prev)
272    {
273      ip4_sv_reass_t *lru_prev =
274	pool_elt_at_index (rt->pool, reass->lru_prev);
275      lru_prev->lru_next = reass->lru_next;
276    }
277  if (~0 != reass->lru_next)
278    {
279      ip4_sv_reass_t *lru_next =
280	pool_elt_at_index (rt->pool, reass->lru_next);
281      lru_next->lru_prev = reass->lru_prev;
282    }
283  if (rt->lru_first == reass - rt->pool)
284    {
285      rt->lru_first = reass->lru_next;
286    }
287  if (rt->lru_last == reass - rt->pool)
288    {
289      rt->lru_last = reass->lru_prev;
290    }
291  pool_put (rt->pool, reass);
292  --rt->reass_n;
293}
294
295always_inline void
296ip4_sv_reass_init (ip4_sv_reass_t * reass)
297{
298  reass->cached_buffers = NULL;
299  reass->is_complete = false;
300}
301
302always_inline ip4_sv_reass_t *
303ip4_sv_reass_find_or_create (vlib_main_t * vm, ip4_sv_reass_main_t * rm,
304			     ip4_sv_reass_per_thread_t * rt,
305			     ip4_sv_reass_kv_t * kv, u8 * do_handoff)
306{
307  ip4_sv_reass_t *reass = NULL;
308  f64 now = vlib_time_now (rm->vlib_main);
309
310  if (!clib_bihash_search_16_8
311      (&rm->hash, (clib_bihash_kv_16_8_t *) kv, (clib_bihash_kv_16_8_t *) kv))
312    {
313      if (vm->thread_index != kv->v.thread_index)
314	{
315	  *do_handoff = 1;
316	  return NULL;
317	}
318      reass = pool_elt_at_index (rt->pool, kv->v.reass_index);
319
320      if (now > reass->last_heard + rm->timeout)
321	{
322	  ip4_sv_reass_free (vm, rm, rt, reass);
323	  reass = NULL;
324	}
325    }
326
327  if (reass)
328    {
329      reass->last_heard = now;
330      return reass;
331    }
332
333  if (rt->reass_n >= rm->max_reass_n && rm->max_reass_n)
334    {
335      reass = pool_elt_at_index (rt->pool, rt->lru_last);
336      ip4_sv_reass_free (vm, rm, rt, reass);
337    }
338
339  pool_get (rt->pool, reass);
340  clib_memset (reass, 0, sizeof (*reass));
341  reass->id = ((u64) vm->thread_index * 1000000000) + rt->id_counter;
342  ++rt->id_counter;
343  ip4_sv_reass_init (reass);
344  ++rt->reass_n;
345  reass->lru_prev = reass->lru_next = ~0;
346
347  if (~0 != rt->lru_last)
348    {
349      ip4_sv_reass_t *lru_last = pool_elt_at_index (rt->pool, rt->lru_last);
350      reass->lru_prev = rt->lru_last;
351      lru_last->lru_next = rt->lru_last = reass - rt->pool;
352    }
353
354  if (~0 == rt->lru_first)
355    {
356      rt->lru_first = rt->lru_last = reass - rt->pool;
357    }
358
359  reass->key.as_u64[0] = ((clib_bihash_kv_16_8_t *) kv)->key[0];
360  reass->key.as_u64[1] = ((clib_bihash_kv_16_8_t *) kv)->key[1];
361  kv->v.reass_index = (reass - rt->pool);
362  kv->v.thread_index = vm->thread_index;
363  reass->last_heard = now;
364
365  if (clib_bihash_add_del_16_8 (&rm->hash, (clib_bihash_kv_16_8_t *) kv, 1))
366    {
367      ip4_sv_reass_free (vm, rm, rt, reass);
368      reass = NULL;
369    }
370
371  return reass;
372}
373
374always_inline ip4_sv_reass_rc_t
375ip4_sv_reass_update (vlib_main_t * vm, vlib_node_runtime_t * node,
376		     ip4_sv_reass_main_t * rm, ip4_sv_reass_per_thread_t * rt,
377		     ip4_header_t * ip0, ip4_sv_reass_t * reass, u32 bi0)
378{
379  vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
380  ip4_sv_reass_rc_t rc = IP4_SV_REASS_RC_OK;
381  const u32 fragment_first = ip4_get_fragment_offset_bytes (ip0);
382  if (0 == fragment_first)
383    {
384      reass->ip_proto = ip0->protocol;
385      reass->l4_src_port = ip4_get_port (ip0, 1);
386      reass->l4_dst_port = ip4_get_port (ip0, 0);
387      if (!reass->l4_src_port || !reass->l4_dst_port)
388	return IP4_SV_REASS_RC_UNSUPP_IP_PROTO;
389      if (IP_PROTOCOL_TCP == reass->ip_proto)
390	{
391	  reass->icmp_type_or_tcp_flags = ((tcp_header_t *) (ip0 + 1))->flags;
392	  reass->tcp_ack_number = ((tcp_header_t *) (ip0 + 1))->ack_number;
393	  reass->tcp_seq_number = ((tcp_header_t *) (ip0 + 1))->seq_number;
394	}
395      else if (IP_PROTOCOL_ICMP == reass->ip_proto)
396	{
397	  reass->icmp_type_or_tcp_flags =
398	    ((icmp46_header_t *) (ip0 + 1))->type;
399	}
400      reass->is_complete = true;
401      vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
402      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
403	{
404	  ip4_sv_reass_add_trace (vm, node, rm, reass, bi0, REASS_FINISH,
405				  reass->ip_proto, reass->l4_src_port,
406				  reass->l4_dst_port);
407	}
408    }
409  vec_add1 (reass->cached_buffers, bi0);
410  if (!reass->is_complete)
411    {
412      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
413	{
414	  ip4_sv_reass_add_trace (vm, node, rm, reass, bi0,
415				  REASS_FRAGMENT_CACHE, ~0, ~0, ~0);
416	}
417      if (vec_len (reass->cached_buffers) > rm->max_reass_len)
418	{
419	  rc = IP4_SV_REASS_RC_TOO_MANY_FRAGMENTS;
420	}
421    }
422  return rc;
423}
424
425always_inline uword
426ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
427		     vlib_frame_t * frame, bool is_feature,
428		     bool is_output_feature, bool is_custom)
429{
430  u32 *from = vlib_frame_vector_args (frame);
431  u32 n_left_from, n_left_to_next, *to_next, next_index;
432  ip4_sv_reass_main_t *rm = &ip4_sv_reass_main;
433  ip4_sv_reass_per_thread_t *rt = &rm->per_thread_data[vm->thread_index];
434  clib_spinlock_lock (&rt->lock);
435
436  n_left_from = frame->n_vectors;
437  next_index = node->cached_next_index;
438
439  while (n_left_from > 0)
440    {
441      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
442
443      while (n_left_from > 0 && n_left_to_next > 0)
444	{
445	  u32 bi0;
446	  vlib_buffer_t *b0;
447	  u32 next0;
448	  u32 error0 = IP4_ERROR_NONE;
449
450	  bi0 = from[0];
451	  b0 = vlib_get_buffer (vm, bi0);
452
453	  ip4_header_t *ip0 =
454	    (ip4_header_t *) u8_ptr_add (vlib_buffer_get_current (b0),
455					 is_output_feature *
456					 vnet_buffer (b0)->
457					 ip.save_rewrite_length);
458	  if (!ip4_get_fragment_more (ip0) && !ip4_get_fragment_offset (ip0))
459	    {
460	      // this is a regular packet - no fragmentation
461	      if (is_custom)
462		{
463		  next0 = vnet_buffer (b0)->ip.reass.next_index;
464		}
465	      else
466		{
467		  next0 = IP4_SV_REASSEMBLY_NEXT_INPUT;
468		}
469	      vnet_buffer (b0)->ip.reass.is_non_first_fragment = 0;
470	      vnet_buffer (b0)->ip.reass.ip_proto = ip0->protocol;
471	      if (IP_PROTOCOL_TCP == ip0->protocol)
472		{
473		  vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags =
474		    ((tcp_header_t *) (ip0 + 1))->flags;
475		  vnet_buffer (b0)->ip.reass.tcp_ack_number =
476		    ((tcp_header_t *) (ip0 + 1))->ack_number;
477		  vnet_buffer (b0)->ip.reass.tcp_seq_number =
478		    ((tcp_header_t *) (ip0 + 1))->seq_number;
479		}
480	      else if (IP_PROTOCOL_ICMP == ip0->protocol)
481		{
482		  vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags =
483		    ((icmp46_header_t *) (ip0 + 1))->type;
484		}
485	      vnet_buffer (b0)->ip.reass.l4_src_port = ip4_get_port (ip0, 1);
486	      vnet_buffer (b0)->ip.reass.l4_dst_port = ip4_get_port (ip0, 0);
487	      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
488		{
489		  ip4_sv_reass_add_trace (vm, node, rm, NULL, bi0,
490					  REASS_PASSTHROUGH,
491					  vnet_buffer (b0)->ip.reass.ip_proto,
492					  vnet_buffer (b0)->ip.
493					  reass.l4_src_port,
494					  vnet_buffer (b0)->ip.
495					  reass.l4_dst_port);
496		}
497	      goto packet_enqueue;
498	    }
499	  const u32 fragment_first = ip4_get_fragment_offset_bytes (ip0);
500	  const u32 fragment_length =
501	    clib_net_to_host_u16 (ip0->length) - ip4_header_bytes (ip0);
502	  const u32 fragment_last = fragment_first + fragment_length - 1;
503	  if (fragment_first > fragment_last || fragment_first + fragment_length > UINT16_MAX - 20 || (fragment_length < 8 && ip4_get_fragment_more (ip0)))	// 8 is minimum frag length per RFC 791
504	    {
505	      next0 = IP4_SV_REASSEMBLY_NEXT_DROP;
506	      error0 = IP4_ERROR_REASS_MALFORMED_PACKET;
507	      goto packet_enqueue;
508	    }
509	  ip4_sv_reass_kv_t kv;
510	  u8 do_handoff = 0;
511
512	  kv.k.as_u64[0] =
513	    (u64) vec_elt (ip4_main.fib_index_by_sw_if_index,
514			   vnet_buffer (b0)->sw_if_index[VLIB_RX]) |
515	    (u64) ip0->src_address.as_u32 << 32;
516	  kv.k.as_u64[1] =
517	    (u64) ip0->dst_address.
518	    as_u32 | (u64) ip0->fragment_id << 32 | (u64) ip0->protocol << 48;
519
520	  ip4_sv_reass_t *reass =
521	    ip4_sv_reass_find_or_create (vm, rm, rt, &kv, &do_handoff);
522
523	  if (PREDICT_FALSE (do_handoff))
524	    {
525	      next0 = IP4_SV_REASSEMBLY_NEXT_HANDOFF;
526	      vnet_buffer (b0)->ip.reass.owner_thread_index =
527		kv.v.thread_index;
528	      goto packet_enqueue;
529	    }
530
531	  if (!reass)
532	    {
533	      next0 = IP4_SV_REASSEMBLY_NEXT_DROP;
534	      error0 = IP4_ERROR_REASS_LIMIT_REACHED;
535	      goto packet_enqueue;
536	    }
537
538	  if (reass->is_complete)
539	    {
540	      if (is_custom)
541		{
542		  next0 = vnet_buffer (b0)->ip.reass.next_index;
543		}
544	      else
545		{
546		  next0 = IP4_SV_REASSEMBLY_NEXT_INPUT;
547		}
548	      vnet_buffer (b0)->ip.reass.is_non_first_fragment =
549		! !fragment_first;
550	      vnet_buffer (b0)->ip.reass.ip_proto = reass->ip_proto;
551	      vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags =
552		reass->icmp_type_or_tcp_flags;
553	      vnet_buffer (b0)->ip.reass.tcp_ack_number =
554		reass->tcp_ack_number;
555	      vnet_buffer (b0)->ip.reass.tcp_seq_number =
556		reass->tcp_seq_number;
557	      vnet_buffer (b0)->ip.reass.l4_src_port = reass->l4_src_port;
558	      vnet_buffer (b0)->ip.reass.l4_dst_port = reass->l4_dst_port;
559	      error0 = IP4_ERROR_NONE;
560	      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
561		{
562		  ip4_sv_reass_add_trace (vm, node, rm, reass, bi0,
563					  REASS_FRAGMENT_FORWARD,
564					  reass->ip_proto,
565					  reass->l4_src_port,
566					  reass->l4_dst_port);
567		}
568	      goto packet_enqueue;
569	    }
570
571	  ip4_sv_reass_rc_t rc =
572	    ip4_sv_reass_update (vm, node, rm, rt, ip0, reass, bi0);
573	  switch (rc)
574	    {
575	    case IP4_SV_REASS_RC_OK:
576	      /* nothing to do here */
577	      break;
578	    case IP4_SV_REASS_RC_TOO_MANY_FRAGMENTS:
579	      vlib_node_increment_counter (vm, node->node_index,
580					   IP4_ERROR_REASS_FRAGMENT_CHAIN_TOO_LONG,
581					   1);
582	      ip4_sv_reass_free (vm, rm, rt, reass);
583	      goto next_packet;
584	      break;
585	    case IP4_SV_REASS_RC_UNSUPP_IP_PROTO:
586	      vlib_node_increment_counter (vm, node->node_index,
587					   IP4_ERROR_REASS_FRAGMENT_CHAIN_TOO_LONG,
588					   1);
589	      ip4_sv_reass_free (vm, rm, rt, reass);
590	      goto next_packet;
591	      break;
592	    }
593	  if (reass->is_complete)
594	    {
595	      u32 idx;
596	      vec_foreach_index (idx, reass->cached_buffers)
597	      {
598		u32 bi0 = vec_elt (reass->cached_buffers, idx);
599		vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
600		u32 next0 = IP4_SV_REASSEMBLY_NEXT_INPUT;
601		if (is_feature)
602		  {
603		    vnet_feature_next (&next0, b0);
604		  }
605		if (is_custom)
606		  {
607		    next0 = vnet_buffer (b0)->ip.reass.next_index;
608		  }
609		if (0 == n_left_to_next)
610		  {
611		    vlib_put_next_frame (vm, node, next_index,
612					 n_left_to_next);
613		    vlib_get_next_frame (vm, node, next_index, to_next,
614					 n_left_to_next);
615		  }
616		to_next[0] = bi0;
617		to_next += 1;
618		n_left_to_next -= 1;
619		vnet_buffer (b0)->ip.reass.is_non_first_fragment =
620		  ! !ip4_get_fragment_offset (vlib_buffer_get_current (b0));
621		vnet_buffer (b0)->ip.reass.ip_proto = reass->ip_proto;
622		vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags =
623		  reass->icmp_type_or_tcp_flags;
624		vnet_buffer (b0)->ip.reass.tcp_ack_number =
625		  reass->tcp_ack_number;
626		vnet_buffer (b0)->ip.reass.tcp_seq_number =
627		  reass->tcp_seq_number;
628		vnet_buffer (b0)->ip.reass.l4_src_port = reass->l4_src_port;
629		vnet_buffer (b0)->ip.reass.l4_dst_port = reass->l4_dst_port;
630		if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
631		  {
632		    ip4_sv_reass_add_trace (vm, node, rm, reass, bi0,
633					    REASS_FRAGMENT_FORWARD,
634					    reass->ip_proto,
635					    reass->l4_src_port,
636					    reass->l4_dst_port);
637		  }
638		vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
639						 to_next, n_left_to_next, bi0,
640						 next0);
641	      }
642	      _vec_len (reass->cached_buffers) = 0;	// buffers are owned by frame now
643	    }
644	  goto next_packet;
645
646	packet_enqueue:
647	  b0->error = node->errors[error0];
648
649	  to_next[0] = bi0;
650	  to_next += 1;
651	  n_left_to_next -= 1;
652	  if (is_feature && IP4_ERROR_NONE == error0)
653	    {
654	      b0 = vlib_get_buffer (vm, bi0);
655	      vnet_feature_next (&next0, b0);
656	    }
657	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
658					   to_next, n_left_to_next,
659					   bi0, next0);
660
661	next_packet:
662	  from += 1;
663	  n_left_from -= 1;
664	}
665
666      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
667    }
668
669  clib_spinlock_unlock (&rt->lock);
670  return frame->n_vectors;
671}
672
673static char *ip4_sv_reass_error_strings[] = {
674#define _(sym, string) string,
675  foreach_ip4_error
676#undef _
677};
678
679VLIB_NODE_FN (ip4_sv_reass_node) (vlib_main_t * vm,
680				  vlib_node_runtime_t * node,
681				  vlib_frame_t * frame)
682{
683  return ip4_sv_reass_inline (vm, node, frame, false /* is_feature */ ,
684			      false /* is_output_feature */ ,
685			      false /* is_custom */ );
686}
687
688/* *INDENT-OFF* */
689VLIB_REGISTER_NODE (ip4_sv_reass_node) = {
690    .name = "ip4-sv-reassembly",
691    .vector_size = sizeof (u32),
692    .format_trace = format_ip4_sv_reass_trace,
693    .n_errors = ARRAY_LEN (ip4_sv_reass_error_strings),
694    .error_strings = ip4_sv_reass_error_strings,
695    .n_next_nodes = IP4_SV_REASSEMBLY_N_NEXT,
696    .next_nodes =
697        {
698                [IP4_SV_REASSEMBLY_NEXT_INPUT] = "ip4-input",
699                [IP4_SV_REASSEMBLY_NEXT_DROP] = "ip4-drop",
700                [IP4_SV_REASSEMBLY_NEXT_HANDOFF] = "ip4-sv-reassembly-handoff",
701
702        },
703};
704/* *INDENT-ON* */
705
706VLIB_NODE_FN (ip4_sv_reass_node_feature) (vlib_main_t * vm,
707					  vlib_node_runtime_t * node,
708					  vlib_frame_t * frame)
709{
710  return ip4_sv_reass_inline (vm, node, frame, true /* is_feature */ ,
711			      false /* is_output_feature */ ,
712			      false /* is_custom */ );
713}
714
715/* *INDENT-OFF* */
716VLIB_REGISTER_NODE (ip4_sv_reass_node_feature) = {
717    .name = "ip4-sv-reassembly-feature",
718    .vector_size = sizeof (u32),
719    .format_trace = format_ip4_sv_reass_trace,
720    .n_errors = ARRAY_LEN (ip4_sv_reass_error_strings),
721    .error_strings = ip4_sv_reass_error_strings,
722    .n_next_nodes = IP4_SV_REASSEMBLY_N_NEXT,
723    .next_nodes =
724        {
725                [IP4_SV_REASSEMBLY_NEXT_INPUT] = "ip4-input",
726                [IP4_SV_REASSEMBLY_NEXT_DROP] = "ip4-drop",
727                [IP4_SV_REASSEMBLY_NEXT_HANDOFF] = "ip4-sv-reass-feature-hoff",
728        },
729};
730/* *INDENT-ON* */
731
732/* *INDENT-OFF* */
733VNET_FEATURE_INIT (ip4_sv_reass_feature) = {
734    .arc_name = "ip4-unicast",
735    .node_name = "ip4-sv-reassembly-feature",
736    .runs_before = VNET_FEATURES ("ip4-lookup"),
737    .runs_after = 0,
738};
739/* *INDENT-ON* */
740
741VLIB_NODE_FN (ip4_sv_reass_node_output_feature) (vlib_main_t * vm,
742						 vlib_node_runtime_t * node,
743						 vlib_frame_t * frame)
744{
745  return ip4_sv_reass_inline (vm, node, frame, true /* is_feature */ ,
746			      true /* is_output_feature */ ,
747			      false /* is_custom */ );
748}
749
750
751/* *INDENT-OFF* */
752VLIB_REGISTER_NODE (ip4_sv_reass_node_output_feature) = {
753    .name = "ip4-sv-reassembly-output-feature",
754    .vector_size = sizeof (u32),
755    .format_trace = format_ip4_sv_reass_trace,
756    .n_errors = ARRAY_LEN (ip4_sv_reass_error_strings),
757    .error_strings = ip4_sv_reass_error_strings,
758    .n_next_nodes = IP4_SV_REASSEMBLY_N_NEXT,
759    .next_nodes =
760        {
761                [IP4_SV_REASSEMBLY_NEXT_INPUT] = "ip4-input",
762                [IP4_SV_REASSEMBLY_NEXT_DROP] = "ip4-drop",
763                [IP4_SV_REASSEMBLY_NEXT_HANDOFF] = "ip4-sv-reass-feature-hoff",
764        },
765};
766/* *INDENT-ON* */
767
768/* *INDENT-OFF* */
769VNET_FEATURE_INIT (ip4_sv_reass_output_feature) = {
770    .arc_name = "ip4-output",
771    .node_name = "ip4-sv-reassembly-output-feature",
772    .runs_before = 0,
773    .runs_after = 0,
774};
775/* *INDENT-ON* */
776
777/* *INDENT-OFF* */
778VLIB_REGISTER_NODE (ip4_sv_reass_custom_node) = {
779    .name = "ip4-sv-reassembly-custom-next",
780    .vector_size = sizeof (u32),
781    .format_trace = format_ip4_sv_reass_trace,
782    .n_errors = ARRAY_LEN (ip4_sv_reass_error_strings),
783    .error_strings = ip4_sv_reass_error_strings,
784    .n_next_nodes = IP4_SV_REASSEMBLY_N_NEXT,
785    .next_nodes =
786        {
787                [IP4_SV_REASSEMBLY_NEXT_INPUT] = "ip4-input",
788                [IP4_SV_REASSEMBLY_NEXT_DROP] = "ip4-drop",
789                [IP4_SV_REASSEMBLY_NEXT_HANDOFF] = "ip4-sv-reassembly-handoff",
790
791        },
792};
793/* *INDENT-ON* */
794
795VLIB_NODE_FN (ip4_sv_reass_custom_node) (vlib_main_t * vm,
796					 vlib_node_runtime_t * node,
797					 vlib_frame_t * frame)
798{
799  return ip4_sv_reass_inline (vm, node, frame, false /* is_feature */ ,
800			      false /* is_output_feature */ ,
801			      true /* is_custom */ );
802}
803
804#ifndef CLIB_MARCH_VARIANT
805always_inline u32
806ip4_sv_reass_get_nbuckets ()
807{
808  ip4_sv_reass_main_t *rm = &ip4_sv_reass_main;
809  u32 nbuckets;
810  u8 i;
811
812  nbuckets = (u32) (rm->max_reass_n / IP4_SV_REASS_HT_LOAD_FACTOR);
813
814  for (i = 0; i < 31; i++)
815    if ((1 << i) >= nbuckets)
816      break;
817  nbuckets = 1 << i;
818
819  return nbuckets;
820}
821#endif /* CLIB_MARCH_VARIANT */
822
823typedef enum
824{
825  IP4_EVENT_CONFIG_CHANGED = 1,
826} ip4_sv_reass_event_t;
827
828typedef struct
829{
830  int failure;
831  clib_bihash_16_8_t *new_hash;
832} ip4_rehash_cb_ctx;
833
834#ifndef CLIB_MARCH_VARIANT
835static int
836ip4_rehash_cb (clib_bihash_kv_16_8_t * kv, void *_ctx)
837{
838  ip4_rehash_cb_ctx *ctx = _ctx;
839  if (clib_bihash_add_del_16_8 (ctx->new_hash, kv, 1))
840    {
841      ctx->failure = 1;
842    }
843  return (BIHASH_WALK_CONTINUE);
844}
845
846static void
847ip4_sv_reass_set_params (u32 timeout_ms, u32 max_reassemblies,
848			 u32 max_reassembly_length,
849			 u32 expire_walk_interval_ms)
850{
851  ip4_sv_reass_main.timeout_ms = timeout_ms;
852  ip4_sv_reass_main.timeout = (f64) timeout_ms / (f64) MSEC_PER_SEC;
853  ip4_sv_reass_main.max_reass_n = max_reassemblies;
854  ip4_sv_reass_main.max_reass_len = max_reassembly_length;
855  ip4_sv_reass_main.expire_walk_interval_ms = expire_walk_interval_ms;
856}
857
858vnet_api_error_t
859ip4_sv_reass_set (u32 timeout_ms, u32 max_reassemblies,
860		  u32 max_reassembly_length, u32 expire_walk_interval_ms)
861{
862  u32 old_nbuckets = ip4_sv_reass_get_nbuckets ();
863  ip4_sv_reass_set_params (timeout_ms, max_reassemblies,
864			   max_reassembly_length, expire_walk_interval_ms);
865  vlib_process_signal_event (ip4_sv_reass_main.vlib_main,
866			     ip4_sv_reass_main.ip4_sv_reass_expire_node_idx,
867			     IP4_EVENT_CONFIG_CHANGED, 0);
868  u32 new_nbuckets = ip4_sv_reass_get_nbuckets ();
869  if (ip4_sv_reass_main.max_reass_n > 0 && new_nbuckets > old_nbuckets)
870    {
871      clib_bihash_16_8_t new_hash;
872      clib_memset (&new_hash, 0, sizeof (new_hash));
873      ip4_rehash_cb_ctx ctx;
874      ctx.failure = 0;
875      ctx.new_hash = &new_hash;
876      clib_bihash_init_16_8 (&new_hash, "ip4-dr", new_nbuckets,
877			     new_nbuckets * 1024);
878      clib_bihash_foreach_key_value_pair_16_8 (&ip4_sv_reass_main.hash,
879					       ip4_rehash_cb, &ctx);
880      if (ctx.failure)
881	{
882	  clib_bihash_free_16_8 (&new_hash);
883	  return -1;
884	}
885      else
886	{
887	  clib_bihash_free_16_8 (&ip4_sv_reass_main.hash);
888	  clib_memcpy_fast (&ip4_sv_reass_main.hash, &new_hash,
889			    sizeof (ip4_sv_reass_main.hash));
890	  clib_bihash_copied (&ip4_sv_reass_main.hash, &new_hash);
891	}
892    }
893  return 0;
894}
895
896vnet_api_error_t
897ip4_sv_reass_get (u32 * timeout_ms, u32 * max_reassemblies,
898		  u32 * max_reassembly_length, u32 * expire_walk_interval_ms)
899{
900  *timeout_ms = ip4_sv_reass_main.timeout_ms;
901  *max_reassemblies = ip4_sv_reass_main.max_reass_n;
902  *max_reassembly_length = ip4_sv_reass_main.max_reass_len;
903  *expire_walk_interval_ms = ip4_sv_reass_main.expire_walk_interval_ms;
904  return 0;
905}
906
907static clib_error_t *
908ip4_sv_reass_init_function (vlib_main_t * vm)
909{
910  ip4_sv_reass_main_t *rm = &ip4_sv_reass_main;
911  clib_error_t *error = 0;
912  u32 nbuckets;
913  vlib_node_t *node;
914
915  rm->vlib_main = vm;
916  rm->vnet_main = vnet_get_main ();
917
918  vec_validate (rm->per_thread_data, vlib_num_workers ());
919  ip4_sv_reass_per_thread_t *rt;
920  vec_foreach (rt, rm->per_thread_data)
921  {
922    clib_spinlock_init (&rt->lock);
923    pool_alloc (rt->pool, rm->max_reass_n);
924    rt->lru_first = rt->lru_last = ~0;
925  }
926
927  node = vlib_get_node_by_name (vm, (u8 *) "ip4-sv-reassembly-expire-walk");
928  ASSERT (node);
929  rm->ip4_sv_reass_expire_node_idx = node->index;
930
931  ip4_sv_reass_set_params (IP4_SV_REASS_TIMEOUT_DEFAULT_MS,
932			   IP4_SV_REASS_MAX_REASSEMBLIES_DEFAULT,
933			   IP4_SV_REASS_MAX_REASSEMBLY_LENGTH_DEFAULT,
934			   IP4_SV_REASS_EXPIRE_WALK_INTERVAL_DEFAULT_MS);
935
936  nbuckets = ip4_sv_reass_get_nbuckets ();
937  clib_bihash_init_16_8 (&rm->hash, "ip4-dr", nbuckets, nbuckets * 1024);
938
939  node = vlib_get_node_by_name (vm, (u8 *) "ip4-drop");
940  ASSERT (node);
941  rm->ip4_drop_idx = node->index;
942
943  rm->fq_index = vlib_frame_queue_main_init (ip4_sv_reass_node.index, 0);
944  rm->fq_feature_index =
945    vlib_frame_queue_main_init (ip4_sv_reass_node_feature.index, 0);
946
947  rm->feature_use_refcount_per_intf = NULL;
948  rm->output_feature_use_refcount_per_intf = NULL;
949
950  return error;
951}
952
953VLIB_INIT_FUNCTION (ip4_sv_reass_init_function);
954#endif /* CLIB_MARCH_VARIANT */
955
956static uword
957ip4_sv_reass_walk_expired (vlib_main_t * vm,
958			   vlib_node_runtime_t * node, vlib_frame_t * f)
959{
960  ip4_sv_reass_main_t *rm = &ip4_sv_reass_main;
961  uword event_type, *event_data = 0;
962
963  while (true)
964    {
965      vlib_process_wait_for_event_or_clock (vm,
966					    (f64)
967					    rm->expire_walk_interval_ms /
968					    (f64) MSEC_PER_SEC);
969      event_type = vlib_process_get_events (vm, &event_data);
970
971      switch (event_type)
972	{
973	case ~0:		/* no events => timeout */
974	  /* nothing to do here */
975	  break;
976	case IP4_EVENT_CONFIG_CHANGED:
977	  break;
978	default:
979	  clib_warning ("BUG: event type 0x%wx", event_type);
980	  break;
981	}
982      f64 now = vlib_time_now (vm);
983
984      ip4_sv_reass_t *reass;
985      int *pool_indexes_to_free = NULL;
986
987      uword thread_index = 0;
988      int index;
989      const uword nthreads = vlib_num_workers () + 1;
990      for (thread_index = 0; thread_index < nthreads; ++thread_index)
991	{
992	  ip4_sv_reass_per_thread_t *rt = &rm->per_thread_data[thread_index];
993	  clib_spinlock_lock (&rt->lock);
994
995	  vec_reset_length (pool_indexes_to_free);
996          /* *INDENT-OFF* */
997          pool_foreach_index (index, rt->pool, ({
998                                reass = pool_elt_at_index (rt->pool, index);
999                                if (now > reass->last_heard + rm->timeout)
1000                                  {
1001                                    vec_add1 (pool_indexes_to_free, index);
1002                                  }
1003                              }));
1004          /* *INDENT-ON* */
1005	  int *i;
1006          /* *INDENT-OFF* */
1007          vec_foreach (i, pool_indexes_to_free)
1008          {
1009            ip4_sv_reass_t *reass = pool_elt_at_index (rt->pool, i[0]);
1010            ip4_sv_reass_free (vm, rm, rt, reass);
1011          }
1012          /* *INDENT-ON* */
1013
1014	  clib_spinlock_unlock (&rt->lock);
1015	}
1016
1017      vec_free (pool_indexes_to_free);
1018      if (event_data)
1019	{
1020	  _vec_len (event_data) = 0;
1021	}
1022    }
1023
1024  return 0;
1025}
1026
1027/* *INDENT-OFF* */
1028VLIB_REGISTER_NODE (ip4_sv_reass_expire_node) = {
1029    .function = ip4_sv_reass_walk_expired,
1030    .type = VLIB_NODE_TYPE_PROCESS,
1031    .name = "ip4-sv-reassembly-expire-walk",
1032    .format_trace = format_ip4_sv_reass_trace,
1033    .n_errors = ARRAY_LEN (ip4_sv_reass_error_strings),
1034    .error_strings = ip4_sv_reass_error_strings,
1035
1036};
1037/* *INDENT-ON* */
1038
1039static u8 *
1040format_ip4_sv_reass_key (u8 * s, va_list * args)
1041{
1042  ip4_sv_reass_key_t *key = va_arg (*args, ip4_sv_reass_key_t *);
1043  s =
1044    format (s,
1045	    "xx_id: %u, src: %U, dst: %U, frag_id: %u, proto: %u",
1046	    key->xx_id, format_ip4_address, &key->src, format_ip4_address,
1047	    &key->dst, clib_net_to_host_u16 (key->frag_id), key->proto);
1048  return s;
1049}
1050
1051static u8 *
1052format_ip4_sv_reass (u8 * s, va_list * args)
1053{
1054  vlib_main_t *vm = va_arg (*args, vlib_main_t *);
1055  ip4_sv_reass_t *reass = va_arg (*args, ip4_sv_reass_t *);
1056
1057  s = format (s, "ID: %lu, key: %U trace_op_counter: %u\n",
1058	      reass->id, format_ip4_sv_reass_key, &reass->key,
1059	      reass->trace_op_counter);
1060
1061  vlib_buffer_t *b;
1062  u32 *bip;
1063  u32 counter = 0;
1064  vec_foreach (bip, reass->cached_buffers)
1065  {
1066    u32 bi = *bip;
1067    do
1068      {
1069	b = vlib_get_buffer (vm, bi);
1070	s = format (s, "  #%03u: bi: %u, ", counter, bi);
1071	++counter;
1072	bi = b->next_buffer;
1073      }
1074    while (b->flags & VLIB_BUFFER_NEXT_PRESENT);
1075  }
1076  return s;
1077}
1078
1079static clib_error_t *
1080show_ip4_reass (vlib_main_t * vm,
1081		unformat_input_t * input,
1082		CLIB_UNUSED (vlib_cli_command_t * lmd))
1083{
1084  ip4_sv_reass_main_t *rm = &ip4_sv_reass_main;
1085
1086  vlib_cli_output (vm, "---------------------");
1087  vlib_cli_output (vm, "IP4 reassembly status");
1088  vlib_cli_output (vm, "---------------------");
1089  bool details = false;
1090  if (unformat (input, "details"))
1091    {
1092      details = true;
1093    }
1094
1095  u32 sum_reass_n = 0;
1096  ip4_sv_reass_t *reass;
1097  uword thread_index;
1098  const uword nthreads = vlib_num_workers () + 1;
1099  for (thread_index = 0; thread_index < nthreads; ++thread_index)
1100    {
1101      ip4_sv_reass_per_thread_t *rt = &rm->per_thread_data[thread_index];
1102      clib_spinlock_lock (&rt->lock);
1103      if (details)
1104	{
1105          /* *INDENT-OFF* */
1106          pool_foreach (reass, rt->pool, {
1107            vlib_cli_output (vm, "%U", format_ip4_sv_reass, vm, reass);
1108          });
1109          /* *INDENT-ON* */
1110	}
1111      sum_reass_n += rt->reass_n;
1112      clib_spinlock_unlock (&rt->lock);
1113    }
1114  vlib_cli_output (vm, "---------------------");
1115  vlib_cli_output (vm, "Current IP4 reassemblies count: %lu\n",
1116		   (long unsigned) sum_reass_n);
1117  vlib_cli_output (vm,
1118		   "Maximum configured concurrent IP4 reassemblies per worker-thread: %lu\n",
1119		   (long unsigned) rm->max_reass_n);
1120  return 0;
1121}
1122
1123/* *INDENT-OFF* */
1124VLIB_CLI_COMMAND (show_ip4_sv_reass_cmd, static) = {
1125    .path = "show ip4-sv-reassembly",
1126    .short_help = "show ip4-sv-reassembly [details]",
1127    .function = show_ip4_reass,
1128};
1129/* *INDENT-ON* */
1130
1131#ifndef CLIB_MARCH_VARIANT
1132vnet_api_error_t
1133ip4_sv_reass_enable_disable (u32 sw_if_index, u8 enable_disable)
1134{
1135  return ip4_sv_reass_enable_disable_with_refcnt (sw_if_index,
1136						  enable_disable);
1137}
1138#endif /* CLIB_MARCH_VARIANT */
1139
1140
1141#define foreach_ip4_sv_reass_handoff_error                       \
1142_(CONGESTION_DROP, "congestion drop")
1143
1144
1145typedef enum
1146{
1147#define _(sym,str) IP4_SV_REASSEMBLY_HANDOFF_ERROR_##sym,
1148  foreach_ip4_sv_reass_handoff_error
1149#undef _
1150    IP4_SV_REASSEMBLY_HANDOFF_N_ERROR,
1151} ip4_sv_reass_handoff_error_t;
1152
1153static char *ip4_sv_reass_handoff_error_strings[] = {
1154#define _(sym,string) string,
1155  foreach_ip4_sv_reass_handoff_error
1156#undef _
1157};
1158
1159typedef struct
1160{
1161  u32 next_worker_index;
1162} ip4_sv_reass_handoff_trace_t;
1163
1164static u8 *
1165format_ip4_sv_reass_handoff_trace (u8 * s, va_list * args)
1166{
1167  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1168  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1169  ip4_sv_reass_handoff_trace_t *t =
1170    va_arg (*args, ip4_sv_reass_handoff_trace_t *);
1171
1172  s =
1173    format (s, "ip4-sv-reassembly-handoff: next-worker %d",
1174	    t->next_worker_index);
1175
1176  return s;
1177}
1178
1179always_inline uword
1180ip4_sv_reass_handoff_node_inline (vlib_main_t * vm,
1181				  vlib_node_runtime_t * node,
1182				  vlib_frame_t * frame, bool is_feature)
1183{
1184  ip4_sv_reass_main_t *rm = &ip4_sv_reass_main;
1185
1186  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
1187  u32 n_enq, n_left_from, *from;
1188  u16 thread_indices[VLIB_FRAME_SIZE], *ti;
1189  u32 fq_index;
1190
1191  from = vlib_frame_vector_args (frame);
1192  n_left_from = frame->n_vectors;
1193  vlib_get_buffers (vm, from, bufs, n_left_from);
1194
1195  b = bufs;
1196  ti = thread_indices;
1197
1198  fq_index = (is_feature) ? rm->fq_feature_index : rm->fq_index;
1199
1200  while (n_left_from > 0)
1201    {
1202      ti[0] = vnet_buffer (b[0])->ip.reass.owner_thread_index;
1203
1204      if (PREDICT_FALSE
1205	  ((node->flags & VLIB_NODE_FLAG_TRACE)
1206	   && (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
1207	{
1208	  ip4_sv_reass_handoff_trace_t *t =
1209	    vlib_add_trace (vm, node, b[0], sizeof (*t));
1210	  t->next_worker_index = ti[0];
1211	}
1212
1213      n_left_from -= 1;
1214      ti += 1;
1215      b += 1;
1216    }
1217  n_enq =
1218    vlib_buffer_enqueue_to_thread (vm, fq_index, from, thread_indices,
1219				   frame->n_vectors, 1);
1220
1221  if (n_enq < frame->n_vectors)
1222    vlib_node_increment_counter (vm, node->node_index,
1223				 IP4_SV_REASSEMBLY_HANDOFF_ERROR_CONGESTION_DROP,
1224				 frame->n_vectors - n_enq);
1225  return frame->n_vectors;
1226}
1227
1228VLIB_NODE_FN (ip4_sv_reass_handoff_node) (vlib_main_t * vm,
1229					  vlib_node_runtime_t * node,
1230					  vlib_frame_t * frame)
1231{
1232  return ip4_sv_reass_handoff_node_inline (vm, node, frame,
1233					   false /* is_feature */ );
1234}
1235
1236
1237/* *INDENT-OFF* */
1238VLIB_REGISTER_NODE (ip4_sv_reass_handoff_node) = {
1239  .name = "ip4-sv-reassembly-handoff",
1240  .vector_size = sizeof (u32),
1241  .n_errors = ARRAY_LEN(ip4_sv_reass_handoff_error_strings),
1242  .error_strings = ip4_sv_reass_handoff_error_strings,
1243  .format_trace = format_ip4_sv_reass_handoff_trace,
1244
1245  .n_next_nodes = 1,
1246
1247  .next_nodes = {
1248    [0] = "error-drop",
1249  },
1250};
1251/* *INDENT-ON* */
1252
1253
1254/* *INDENT-OFF* */
1255VLIB_NODE_FN (ip4_sv_reass_feature_handoff_node) (vlib_main_t * vm,
1256						    vlib_node_runtime_t *
1257						    node,
1258						    vlib_frame_t * frame)
1259{
1260  return ip4_sv_reass_handoff_node_inline (vm, node, frame,
1261					     true /* is_feature */ );
1262}
1263/* *INDENT-ON* */
1264
1265
1266/* *INDENT-OFF* */
1267VLIB_REGISTER_NODE (ip4_sv_reass_feature_handoff_node) = {
1268  .name = "ip4-sv-reass-feature-hoff",
1269  .vector_size = sizeof (u32),
1270  .n_errors = ARRAY_LEN(ip4_sv_reass_handoff_error_strings),
1271  .error_strings = ip4_sv_reass_handoff_error_strings,
1272  .format_trace = format_ip4_sv_reass_handoff_trace,
1273
1274  .n_next_nodes = 1,
1275
1276  .next_nodes = {
1277    [0] = "error-drop",
1278  },
1279};
1280/* *INDENT-ON* */
1281
1282#ifndef CLIB_MARCH_VARIANT
1283int
1284ip4_sv_reass_enable_disable_with_refcnt (u32 sw_if_index, int is_enable)
1285{
1286  ip4_sv_reass_main_t *rm = &ip4_sv_reass_main;
1287  vec_validate (rm->feature_use_refcount_per_intf, sw_if_index);
1288  if (is_enable)
1289    {
1290      if (!rm->feature_use_refcount_per_intf[sw_if_index])
1291	{
1292	  ++rm->feature_use_refcount_per_intf[sw_if_index];
1293	  return vnet_feature_enable_disable ("ip4-unicast",
1294					      "ip4-sv-reassembly-feature",
1295					      sw_if_index, 1, 0, 0);
1296	}
1297      ++rm->feature_use_refcount_per_intf[sw_if_index];
1298    }
1299  else
1300    {
1301      if (rm->feature_use_refcount_per_intf[sw_if_index])
1302	--rm->feature_use_refcount_per_intf[sw_if_index];
1303      if (!rm->feature_use_refcount_per_intf[sw_if_index])
1304	return vnet_feature_enable_disable ("ip4-unicast",
1305					    "ip4-sv-reassembly-feature",
1306					    sw_if_index, 0, 0, 0);
1307    }
1308  return 0;
1309}
1310
1311uword
1312ip4_sv_reass_custom_register_next_node (uword node_index)
1313{
1314  return vlib_node_add_next (vlib_get_main (), ip4_sv_reass_custom_node.index,
1315			     node_index);
1316}
1317
1318int
1319ip4_sv_reass_output_enable_disable_with_refcnt (u32 sw_if_index,
1320						int is_enable)
1321{
1322  ip4_sv_reass_main_t *rm = &ip4_sv_reass_main;
1323  vec_validate (rm->output_feature_use_refcount_per_intf, sw_if_index);
1324  if (is_enable)
1325    {
1326      if (!rm->output_feature_use_refcount_per_intf[sw_if_index])
1327	{
1328	  ++rm->output_feature_use_refcount_per_intf[sw_if_index];
1329	  return vnet_feature_enable_disable ("ip4-output",
1330					      "ip4-sv-reassembly-output-feature",
1331					      sw_if_index, 1, 0, 0);
1332	}
1333      ++rm->output_feature_use_refcount_per_intf[sw_if_index];
1334    }
1335  else
1336    {
1337      if (rm->output_feature_use_refcount_per_intf[sw_if_index])
1338	--rm->output_feature_use_refcount_per_intf[sw_if_index];
1339      if (!rm->output_feature_use_refcount_per_intf[sw_if_index])
1340	return vnet_feature_enable_disable ("ip4-output",
1341					    "ip4-sv-reassembly-output-feature",
1342					    sw_if_index, 0, 0, 0);
1343    }
1344  return 0;
1345}
1346#endif
1347
1348/*
1349 * fd.io coding-style-patch-verification: ON
1350 *
1351 * Local Variables:
1352 * eval: (c-set-style "gnu")
1353 * End:
1354 */
1355