out2in.c revision 8f6d7a78
1/*
2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15/**
16 * @file
17 * @brief NAT44 endpoint-dependent outside to inside network translation
18 */
19
20#include <vlib/vlib.h>
21#include <vnet/vnet.h>
22#include <vnet/pg/pg.h>
23
24#include <vnet/ip/ip.h>
25#include <vnet/udp/udp.h>
26#include <vnet/ethernet/ethernet.h>
27#include <vnet/fib/ip4_fib.h>
28#include <nat/nat.h>
29#include <nat/nat_ipfix_logging.h>
30#include <nat/nat_reass.h>
31#include <nat/nat_inlines.h>
32#include <nat/nat_syslog.h>
33#include <nat/nat_ha.h>
34
35#include <vppinfra/hash.h>
36#include <vppinfra/error.h>
37#include <vppinfra/elog.h>
38
39typedef struct
40{
41  u32 sw_if_index;
42  u32 next_index;
43  u32 session_index;
44} snat_out2in_trace_t;
45
46/* packet trace format function */
47static u8 *
48format_snat_out2in_trace (u8 * s, va_list * args)
49{
50  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
51  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
52  snat_out2in_trace_t *t = va_arg (*args, snat_out2in_trace_t *);
53
54  s =
55    format (s,
56	    "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
57	    t->sw_if_index, t->next_index, t->session_index);
58  return s;
59}
60
61static u8 *
62format_snat_out2in_fast_trace (u8 * s, va_list * args)
63{
64  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
65  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
66  snat_out2in_trace_t *t = va_arg (*args, snat_out2in_trace_t *);
67
68  s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
69	      t->sw_if_index, t->next_index);
70  return s;
71}
72
73#define foreach_snat_out2in_error                       \
74_(UNSUPPORTED_PROTOCOL, "unsupported protocol")         \
75_(OUT2IN_PACKETS, "good out2in packets processed")      \
76_(OUT_OF_PORTS, "out of ports")                         \
77_(BAD_ICMP_TYPE, "unsupported ICMP type")               \
78_(NO_TRANSLATION, "no translation")                     \
79_(MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded")   \
80_(DROP_FRAGMENT, "drop fragment")                       \
81_(MAX_REASS, "maximum reassemblies exceeded")           \
82_(MAX_FRAG, "maximum fragments per reassembly exceeded")\
83_(TCP_PACKETS, "TCP packets")                           \
84_(UDP_PACKETS, "UDP packets")                           \
85_(ICMP_PACKETS, "ICMP packets")                         \
86_(OTHER_PACKETS, "other protocol packets")              \
87_(FRAGMENTS, "fragments")                               \
88_(CACHED_FRAGMENTS, "cached fragments")                 \
89_(PROCESSED_FRAGMENTS, "processed fragments")
90
91typedef enum
92{
93#define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
94  foreach_snat_out2in_error
95#undef _
96    SNAT_OUT2IN_N_ERROR,
97} snat_out2in_error_t;
98
99static char *snat_out2in_error_strings[] = {
100#define _(sym,string) string,
101  foreach_snat_out2in_error
102#undef _
103};
104
105typedef enum
106{
107  SNAT_OUT2IN_NEXT_DROP,
108  SNAT_OUT2IN_NEXT_LOOKUP,
109  SNAT_OUT2IN_NEXT_ICMP_ERROR,
110  SNAT_OUT2IN_NEXT_REASS,
111  SNAT_OUT2IN_N_NEXT,
112} snat_out2in_next_t;
113
114#ifndef CLIB_MARCH_VARIANT
115int
116nat44_o2i_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
117{
118  snat_main_t *sm = &snat_main;
119  nat44_is_idle_session_ctx_t *ctx = arg;
120  snat_session_t *s;
121  u64 sess_timeout_time;
122  snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
123						       ctx->thread_index);
124  clib_bihash_kv_8_8_t s_kv;
125
126  s = pool_elt_at_index (tsm->sessions, kv->value);
127  sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
128  if (ctx->now >= sess_timeout_time)
129    {
130      s_kv.key = s->in2out.as_u64;
131      if (clib_bihash_add_del_8_8 (&tsm->in2out, &s_kv, 0))
132	nat_elog_warn ("out2in key del failed");
133
134      snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
135					   s->in2out.addr.as_u32,
136					   s->out2in.addr.as_u32,
137					   s->in2out.protocol,
138					   s->in2out.port,
139					   s->out2in.port,
140					   s->in2out.fib_index);
141
142      nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
143			       &s->in2out.addr, s->in2out.port,
144			       &s->out2in.addr, s->out2in.port,
145			       s->in2out.protocol);
146
147      nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
148		   s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
149		   ctx->thread_index);
150
151      if (!snat_is_session_static (s))
152	snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
153					    &s->out2in);
154
155      nat44_delete_session (sm, s, ctx->thread_index);
156      return 1;
157    }
158
159  return 0;
160}
161#endif
162
163/**
164 * @brief Create session for static mapping.
165 *
166 * Create NAT session initiated by host from external network with static
167 * mapping.
168 *
169 * @param sm     NAT main.
170 * @param b0     Vlib buffer.
171 * @param in2out In2out NAT44 session key.
172 * @param out2in Out2in NAT44 session key.
173 * @param node   Vlib node.
174 *
175 * @returns SNAT session if successfully created otherwise 0.
176 */
177static inline snat_session_t *
178create_session_for_static_mapping (snat_main_t * sm,
179				   vlib_buffer_t * b0,
180				   snat_session_key_t in2out,
181				   snat_session_key_t out2in,
182				   vlib_node_runtime_t * node,
183				   u32 thread_index, f64 now)
184{
185  snat_user_t *u;
186  snat_session_t *s;
187  clib_bihash_kv_8_8_t kv0;
188  ip4_header_t *ip0;
189  udp_header_t *udp0;
190  nat44_is_idle_session_ctx_t ctx0;
191
192  if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
193    {
194      b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
195      nat_elog_notice ("maximum sessions exceeded");
196      return 0;
197    }
198
199  ip0 = vlib_buffer_get_current (b0);
200  udp0 = ip4_next_header (ip0);
201
202  u =
203    nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
204  if (!u)
205    {
206      nat_elog_warn ("create NAT user failed");
207      return 0;
208    }
209
210  s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
211  if (!s)
212    {
213      nat44_delete_user_with_no_session (sm, u, thread_index);
214      nat_elog_warn ("create NAT session failed");
215      return 0;
216    }
217
218  s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
219  s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
220  s->ext_host_port = udp0->src_port;
221  user_session_increment (sm, u, 1 /* static */ );
222  s->in2out = in2out;
223  s->out2in = out2in;
224  s->in2out.protocol = out2in.protocol;
225
226  /* Add to translation hashes */
227  ctx0.now = now;
228  ctx0.thread_index = thread_index;
229  kv0.key = s->in2out.as_u64;
230  kv0.value = s - sm->per_thread_data[thread_index].sessions;
231  if (clib_bihash_add_or_overwrite_stale_8_8
232      (&sm->per_thread_data[thread_index].in2out, &kv0,
233       nat44_i2o_is_idle_session_cb, &ctx0))
234    nat_elog_notice ("in2out key add failed");
235
236  kv0.key = s->out2in.as_u64;
237
238  if (clib_bihash_add_or_overwrite_stale_8_8
239      (&sm->per_thread_data[thread_index].out2in, &kv0,
240       nat44_o2i_is_idle_session_cb, &ctx0))
241    nat_elog_notice ("out2in key add failed");
242
243  /* log NAT event */
244  snat_ipfix_logging_nat44_ses_create (thread_index,
245				       s->in2out.addr.as_u32,
246				       s->out2in.addr.as_u32,
247				       s->in2out.protocol,
248				       s->in2out.port,
249				       s->out2in.port, s->in2out.fib_index);
250
251  nat_syslog_nat44_apmadd (s->user_index, s->in2out.fib_index,
252			   &s->in2out.addr, s->in2out.port, &s->out2in.addr,
253			   s->out2in.port, s->in2out.protocol);
254
255  nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
256	       s->out2in.port, &s->ext_host_addr, s->ext_host_port,
257	       &s->ext_host_nat_addr, s->ext_host_nat_port,
258	       s->in2out.protocol, s->in2out.fib_index, s->flags,
259	       thread_index, 0);
260
261  return s;
262}
263
264#ifndef CLIB_MARCH_VARIANT
265static_always_inline
266  snat_out2in_error_t icmp_get_key (ip4_header_t * ip0,
267				    snat_session_key_t * p_key0)
268{
269  icmp46_header_t *icmp0;
270  snat_session_key_t key0;
271  icmp_echo_header_t *echo0, *inner_echo0 = 0;
272  ip4_header_t *inner_ip0;
273  void *l4_header = 0;
274  icmp46_header_t *inner_icmp0;
275
276  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
277  echo0 = (icmp_echo_header_t *) (icmp0 + 1);
278
279  if (!icmp_is_error_message (icmp0))
280    {
281      key0.protocol = SNAT_PROTOCOL_ICMP;
282      key0.addr = ip0->dst_address;
283      key0.port = echo0->identifier;
284    }
285  else
286    {
287      inner_ip0 = (ip4_header_t *) (echo0 + 1);
288      l4_header = ip4_next_header (inner_ip0);
289      key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
290      key0.addr = inner_ip0->src_address;
291      switch (key0.protocol)
292	{
293	case SNAT_PROTOCOL_ICMP:
294	  inner_icmp0 = (icmp46_header_t *) l4_header;
295	  inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
296	  key0.port = inner_echo0->identifier;
297	  break;
298	case SNAT_PROTOCOL_UDP:
299	case SNAT_PROTOCOL_TCP:
300	  key0.port = ((tcp_udp_header_t *) l4_header)->src_port;
301	  break;
302	default:
303	  return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
304	}
305    }
306  *p_key0 = key0;
307  return -1;			/* success */
308}
309
310/**
311 * Get address and port values to be used for ICMP packet translation
312 * and create session if needed
313 *
314 * @param[in,out] sm             NAT main
315 * @param[in,out] node           NAT node runtime
316 * @param[in] thread_index       thread index
317 * @param[in,out] b0             buffer containing packet to be translated
318 * @param[out] p_proto           protocol used for matching
319 * @param[out] p_value           address and port after NAT translation
320 * @param[out] p_dont_translate  if packet should not be translated
321 * @param d                      optional parameter
322 * @param e                      optional parameter
323 */
324u32
325icmp_match_out2in_slow (snat_main_t * sm, vlib_node_runtime_t * node,
326			u32 thread_index, vlib_buffer_t * b0,
327			ip4_header_t * ip0, u8 * p_proto,
328			snat_session_key_t * p_value,
329			u8 * p_dont_translate, void *d, void *e)
330{
331  icmp46_header_t *icmp0;
332  u32 sw_if_index0;
333  u32 rx_fib_index0;
334  snat_session_key_t key0;
335  snat_session_key_t sm0;
336  snat_session_t *s0 = 0;
337  u8 dont_translate = 0;
338  clib_bihash_kv_8_8_t kv0, value0;
339  u8 is_addr_only;
340  u32 next0 = ~0;
341  int err;
342  u8 identity_nat;
343
344  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
345  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
346  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
347
348  key0.protocol = 0;
349
350  err = icmp_get_key (ip0, &key0);
351  if (err != -1)
352    {
353      b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
354      next0 = SNAT_OUT2IN_NEXT_DROP;
355      goto out;
356    }
357  key0.fib_index = rx_fib_index0;
358
359  kv0.key = key0.as_u64;
360
361  if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
362			      &value0))
363    {
364      /* Try to match static mapping by external address and port,
365         destination address and port in packet */
366      if (snat_static_mapping_match
367	  (sm, key0, &sm0, 1, &is_addr_only, 0, 0, 0, &identity_nat))
368	{
369	  if (!sm->forwarding_enabled)
370	    {
371	      /* Don't NAT packet aimed at the intfc address */
372	      if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index0,
373						    ip0->dst_address.as_u32)))
374		{
375		  dont_translate = 1;
376		  goto out;
377		}
378	      b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
379	      next0 = SNAT_OUT2IN_NEXT_DROP;
380	      goto out;
381	    }
382	  else
383	    {
384	      dont_translate = 1;
385	      goto out;
386	    }
387	}
388
389      if (PREDICT_FALSE (icmp0->type != ICMP4_echo_reply &&
390			 (icmp0->type != ICMP4_echo_request
391			  || !is_addr_only)))
392	{
393	  b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
394	  next0 = SNAT_OUT2IN_NEXT_DROP;
395	  goto out;
396	}
397
398      if (PREDICT_FALSE (identity_nat))
399	{
400	  dont_translate = 1;
401	  goto out;
402	}
403      /* Create session initiated by host from external network */
404      s0 = create_session_for_static_mapping (sm, b0, sm0, key0,
405					      node, thread_index,
406					      vlib_time_now (sm->vlib_main));
407
408      if (!s0)
409	{
410	  next0 = SNAT_OUT2IN_NEXT_DROP;
411	  goto out;
412	}
413    }
414  else
415    {
416      if (PREDICT_FALSE (icmp0->type != ICMP4_echo_reply &&
417			 icmp0->type != ICMP4_echo_request &&
418			 !icmp_is_error_message (icmp0)))
419	{
420	  b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
421	  next0 = SNAT_OUT2IN_NEXT_DROP;
422	  goto out;
423	}
424
425      s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
426			      value0.value);
427    }
428
429out:
430  *p_proto = key0.protocol;
431  if (s0)
432    *p_value = s0->in2out;
433  *p_dont_translate = dont_translate;
434  if (d)
435    *(snat_session_t **) d = s0;
436  return next0;
437}
438#endif
439
440#ifndef CLIB_MARCH_VARIANT
441/**
442 * Get address and port values to be used for ICMP packet translation
443 *
444 * @param[in] sm                 NAT main
445 * @param[in,out] node           NAT node runtime
446 * @param[in] thread_index       thread index
447 * @param[in,out] b0             buffer containing packet to be translated
448 * @param[out] p_proto           protocol used for matching
449 * @param[out] p_value           address and port after NAT translation
450 * @param[out] p_dont_translate  if packet should not be translated
451 * @param d                      optional parameter
452 * @param e                      optional parameter
453 */
454u32
455icmp_match_out2in_fast (snat_main_t * sm, vlib_node_runtime_t * node,
456			u32 thread_index, vlib_buffer_t * b0,
457			ip4_header_t * ip0, u8 * p_proto,
458			snat_session_key_t * p_value,
459			u8 * p_dont_translate, void *d, void *e)
460{
461  icmp46_header_t *icmp0;
462  u32 sw_if_index0;
463  u32 rx_fib_index0;
464  snat_session_key_t key0;
465  snat_session_key_t sm0;
466  u8 dont_translate = 0;
467  u8 is_addr_only;
468  u32 next0 = ~0;
469  int err;
470
471  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
472  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
473  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
474
475  err = icmp_get_key (ip0, &key0);
476  if (err != -1)
477    {
478      b0->error = node->errors[err];
479      next0 = SNAT_OUT2IN_NEXT_DROP;
480      goto out2;
481    }
482  key0.fib_index = rx_fib_index0;
483
484  if (snat_static_mapping_match
485      (sm, key0, &sm0, 1, &is_addr_only, 0, 0, 0, 0))
486    {
487      /* Don't NAT packet aimed at the intfc address */
488      if (is_interface_addr (sm, node, sw_if_index0, ip0->dst_address.as_u32))
489	{
490	  dont_translate = 1;
491	  goto out;
492	}
493      b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
494      next0 = SNAT_OUT2IN_NEXT_DROP;
495      goto out;
496    }
497
498  if (PREDICT_FALSE (icmp0->type != ICMP4_echo_reply &&
499		     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
500		     !icmp_is_error_message (icmp0)))
501    {
502      b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
503      next0 = SNAT_OUT2IN_NEXT_DROP;
504      goto out;
505    }
506
507out:
508  *p_value = sm0;
509out2:
510  *p_proto = key0.protocol;
511  *p_dont_translate = dont_translate;
512  return next0;
513}
514#endif
515
516#ifndef CLIB_MARCH_VARIANT
517u32
518icmp_out2in (snat_main_t * sm,
519	     vlib_buffer_t * b0,
520	     ip4_header_t * ip0,
521	     icmp46_header_t * icmp0,
522	     u32 sw_if_index0,
523	     u32 rx_fib_index0,
524	     vlib_node_runtime_t * node,
525	     u32 next0, u32 thread_index, void *d, void *e)
526{
527  snat_session_key_t sm0;
528  u8 protocol;
529  icmp_echo_header_t *echo0, *inner_echo0 = 0;
530  ip4_header_t *inner_ip0 = 0;
531  void *l4_header = 0;
532  icmp46_header_t *inner_icmp0;
533  u8 dont_translate;
534  u32 new_addr0, old_addr0;
535  u16 old_id0, new_id0;
536  ip_csum_t sum0;
537  u16 checksum0;
538  u32 next0_tmp;
539
540  echo0 = (icmp_echo_header_t *) (icmp0 + 1);
541
542  next0_tmp = sm->icmp_match_out2in_cb (sm, node, thread_index, b0, ip0,
543					&protocol, &sm0, &dont_translate, d,
544					e);
545  if (next0_tmp != ~0)
546    next0 = next0_tmp;
547  if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
548    goto out;
549
550  if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
551    {
552      sum0 = ip_incremental_checksum_buffer (sm->vlib_main, b0, (u8 *) icmp0 -
553					     (u8 *)
554					     vlib_buffer_get_current (b0),
555					     ntohs (ip0->length) -
556					     ip4_header_bytes (ip0), 0);
557      checksum0 = ~ip_csum_fold (sum0);
558      if (checksum0 != 0 && checksum0 != 0xffff)
559	{
560	  next0 = SNAT_OUT2IN_NEXT_DROP;
561	  goto out;
562	}
563    }
564
565  old_addr0 = ip0->dst_address.as_u32;
566  new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
567  vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
568
569  sum0 = ip0->checksum;
570  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
571			 dst_address /* changed member */ );
572  ip0->checksum = ip_csum_fold (sum0);
573
574  if (icmp0->checksum == 0)
575    icmp0->checksum = 0xffff;
576
577  if (!icmp_is_error_message (icmp0))
578    {
579      new_id0 = sm0.port;
580      if (PREDICT_FALSE (new_id0 != echo0->identifier))
581	{
582	  old_id0 = echo0->identifier;
583	  new_id0 = sm0.port;
584	  echo0->identifier = new_id0;
585
586	  sum0 = icmp0->checksum;
587	  sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
588				 identifier /* changed member */ );
589	  icmp0->checksum = ip_csum_fold (sum0);
590	}
591    }
592  else
593    {
594      inner_ip0 = (ip4_header_t *) (echo0 + 1);
595      l4_header = ip4_next_header (inner_ip0);
596
597      if (!ip4_header_checksum_is_valid (inner_ip0))
598	{
599	  next0 = SNAT_OUT2IN_NEXT_DROP;
600	  goto out;
601	}
602
603      old_addr0 = inner_ip0->src_address.as_u32;
604      inner_ip0->src_address = sm0.addr;
605      new_addr0 = inner_ip0->src_address.as_u32;
606
607      sum0 = icmp0->checksum;
608      sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
609			     src_address /* changed member */ );
610      icmp0->checksum = ip_csum_fold (sum0);
611
612      switch (protocol)
613	{
614	case SNAT_PROTOCOL_ICMP:
615	  inner_icmp0 = (icmp46_header_t *) l4_header;
616	  inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
617
618	  old_id0 = inner_echo0->identifier;
619	  new_id0 = sm0.port;
620	  inner_echo0->identifier = new_id0;
621
622	  sum0 = icmp0->checksum;
623	  sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
624				 identifier);
625	  icmp0->checksum = ip_csum_fold (sum0);
626	  break;
627	case SNAT_PROTOCOL_UDP:
628	case SNAT_PROTOCOL_TCP:
629	  old_id0 = ((tcp_udp_header_t *) l4_header)->src_port;
630	  new_id0 = sm0.port;
631	  ((tcp_udp_header_t *) l4_header)->src_port = new_id0;
632
633	  sum0 = icmp0->checksum;
634	  sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
635				 src_port);
636	  icmp0->checksum = ip_csum_fold (sum0);
637	  break;
638	default:
639	  ASSERT (0);
640	}
641    }
642
643out:
644  return next0;
645}
646#endif
647
648static inline u32
649icmp_out2in_slow_path (snat_main_t * sm,
650		       vlib_buffer_t * b0,
651		       ip4_header_t * ip0,
652		       icmp46_header_t * icmp0,
653		       u32 sw_if_index0,
654		       u32 rx_fib_index0,
655		       vlib_node_runtime_t * node,
656		       u32 next0, f64 now,
657		       u32 thread_index, snat_session_t ** p_s0)
658{
659  next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
660		       next0, thread_index, p_s0, 0);
661  snat_session_t *s0 = *p_s0;
662  if (PREDICT_TRUE (next0 != SNAT_OUT2IN_NEXT_DROP && s0))
663    {
664      /* Accounting */
665      nat44_session_update_counters (s0, now,
666				     vlib_buffer_length_in_chain
667				     (sm->vlib_main, b0), thread_index);
668      /* Per-user LRU list maintenance */
669      nat44_session_update_lru (sm, s0, thread_index);
670    }
671  return next0;
672}
673
674static int
675nat_out2in_sm_unknown_proto (snat_main_t * sm,
676			     vlib_buffer_t * b,
677			     ip4_header_t * ip, u32 rx_fib_index)
678{
679  clib_bihash_kv_8_8_t kv, value;
680  snat_static_mapping_t *m;
681  snat_session_key_t m_key;
682  u32 old_addr, new_addr;
683  ip_csum_t sum;
684
685  m_key.addr = ip->dst_address;
686  m_key.port = 0;
687  m_key.protocol = 0;
688  m_key.fib_index = 0;
689  kv.key = m_key.as_u64;
690  if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
691    return 1;
692
693  m = pool_elt_at_index (sm->static_mappings, value.value);
694
695  old_addr = ip->dst_address.as_u32;
696  new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
697  sum = ip->checksum;
698  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
699  ip->checksum = ip_csum_fold (sum);
700
701  vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
702  return 0;
703}
704
705VLIB_NODE_FN (snat_out2in_node) (vlib_main_t * vm,
706				 vlib_node_runtime_t * node,
707				 vlib_frame_t * frame)
708{
709  u32 n_left_from, *from, *to_next;
710  snat_out2in_next_t next_index;
711  u32 pkts_processed = 0;
712  snat_main_t *sm = &snat_main;
713  f64 now = vlib_time_now (vm);
714  u32 thread_index = vm->thread_index;
715  u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
716    0, fragments = 0;
717
718  from = vlib_frame_vector_args (frame);
719  n_left_from = frame->n_vectors;
720  next_index = node->cached_next_index;
721
722  while (n_left_from > 0)
723    {
724      u32 n_left_to_next;
725
726      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
727
728      while (n_left_from >= 4 && n_left_to_next >= 2)
729	{
730	  u32 bi0, bi1;
731	  vlib_buffer_t *b0, *b1;
732	  u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
733	  u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
734	  u32 sw_if_index0, sw_if_index1;
735	  ip4_header_t *ip0, *ip1;
736	  ip_csum_t sum0, sum1;
737	  u32 new_addr0, old_addr0;
738	  u16 new_port0, old_port0;
739	  u32 new_addr1, old_addr1;
740	  u16 new_port1, old_port1;
741	  udp_header_t *udp0, *udp1;
742	  tcp_header_t *tcp0, *tcp1;
743	  icmp46_header_t *icmp0, *icmp1;
744	  snat_session_key_t key0, key1, sm0, sm1;
745	  u32 rx_fib_index0, rx_fib_index1;
746	  u32 proto0, proto1;
747	  snat_session_t *s0 = 0, *s1 = 0;
748	  clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
749	  u8 identity_nat0, identity_nat1;
750
751	  /* Prefetch next iteration. */
752	  {
753	    vlib_buffer_t *p2, *p3;
754
755	    p2 = vlib_get_buffer (vm, from[2]);
756	    p3 = vlib_get_buffer (vm, from[3]);
757
758	    vlib_prefetch_buffer_header (p2, LOAD);
759	    vlib_prefetch_buffer_header (p3, LOAD);
760
761	    CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
762	    CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
763	  }
764
765	  /* speculatively enqueue b0 and b1 to the current next frame */
766	  to_next[0] = bi0 = from[0];
767	  to_next[1] = bi1 = from[1];
768	  from += 2;
769	  to_next += 2;
770	  n_left_from -= 2;
771	  n_left_to_next -= 2;
772
773	  b0 = vlib_get_buffer (vm, bi0);
774	  b1 = vlib_get_buffer (vm, bi1);
775
776	  vnet_buffer (b0)->snat.flags = 0;
777	  vnet_buffer (b1)->snat.flags = 0;
778
779	  ip0 = vlib_buffer_get_current (b0);
780	  udp0 = ip4_next_header (ip0);
781	  tcp0 = (tcp_header_t *) udp0;
782	  icmp0 = (icmp46_header_t *) udp0;
783
784	  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
785	  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
786				   sw_if_index0);
787
788	  if (PREDICT_FALSE (ip0->ttl == 1))
789	    {
790	      vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
791	      icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
792					   ICMP4_time_exceeded_ttl_exceeded_in_transit,
793					   0);
794	      next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
795	      goto trace0;
796	    }
797
798	  proto0 = ip_proto_to_snat_proto (ip0->protocol);
799
800	  if (PREDICT_FALSE (proto0 == ~0))
801	    {
802	      if (nat_out2in_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
803		{
804		  if (!sm->forwarding_enabled)
805		    {
806		      b0->error =
807			node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
808		      next0 = SNAT_OUT2IN_NEXT_DROP;
809		    }
810		}
811	      other_packets++;
812	      goto trace0;
813	    }
814
815	  if (PREDICT_FALSE (ip4_is_fragment (ip0)))
816	    {
817	      next0 = SNAT_OUT2IN_NEXT_REASS;
818	      fragments++;
819	      goto trace0;
820	    }
821
822	  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
823	    {
824	      next0 = icmp_out2in_slow_path
825		(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
826		 next0, now, thread_index, &s0);
827	      icmp_packets++;
828	      goto trace0;
829	    }
830
831	  key0.addr = ip0->dst_address;
832	  key0.port = udp0->dst_port;
833	  key0.protocol = proto0;
834	  key0.fib_index = rx_fib_index0;
835
836	  kv0.key = key0.as_u64;
837
838	  if (clib_bihash_search_8_8
839	      (&sm->per_thread_data[thread_index].out2in, &kv0, &value0))
840	    {
841	      /* Try to match static mapping by external address and port,
842	         destination address and port in packet */
843	      if (snat_static_mapping_match
844		  (sm, key0, &sm0, 1, 0, 0, 0, 0, &identity_nat0))
845		{
846		  /*
847		   * Send DHCP packets to the ipv4 stack, or we won't
848		   * be able to use dhcp client on the outside interface
849		   */
850		  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
851				     && (udp0->dst_port ==
852					 clib_host_to_net_u16
853					 (UDP_DST_PORT_dhcp_to_client))))
854		    {
855		      vnet_feature_next (&next0, b0);
856		      goto trace0;
857		    }
858
859		  if (!sm->forwarding_enabled)
860		    {
861		      b0->error =
862			node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
863		      next0 = SNAT_OUT2IN_NEXT_DROP;
864		    }
865		  goto trace0;
866		}
867
868	      if (PREDICT_FALSE (identity_nat0))
869		goto trace0;
870
871	      /* Create session initiated by host from external network */
872	      s0 = create_session_for_static_mapping (sm, b0, sm0, key0, node,
873						      thread_index, now);
874	      if (!s0)
875		{
876		  next0 = SNAT_OUT2IN_NEXT_DROP;
877		  goto trace0;
878		}
879	    }
880	  else
881	    s0 =
882	      pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
883				 value0.value);
884
885	  old_addr0 = ip0->dst_address.as_u32;
886	  ip0->dst_address = s0->in2out.addr;
887	  new_addr0 = ip0->dst_address.as_u32;
888	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
889
890	  sum0 = ip0->checksum;
891	  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
892				 ip4_header_t,
893				 dst_address /* changed member */ );
894	  ip0->checksum = ip_csum_fold (sum0);
895
896	  if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
897	    {
898	      old_port0 = tcp0->dst_port;
899	      tcp0->dst_port = s0->in2out.port;
900	      new_port0 = tcp0->dst_port;
901
902	      sum0 = tcp0->checksum;
903	      sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
904				     ip4_header_t,
905				     dst_address /* changed member */ );
906
907	      sum0 = ip_csum_update (sum0, old_port0, new_port0,
908				     ip4_header_t /* cheat */ ,
909				     length /* changed member */ );
910	      tcp0->checksum = ip_csum_fold (sum0);
911	      tcp_packets++;
912	    }
913	  else
914	    {
915	      old_port0 = udp0->dst_port;
916	      udp0->dst_port = s0->in2out.port;
917	      udp0->checksum = 0;
918	      udp_packets++;
919	    }
920
921	  /* Accounting */
922	  nat44_session_update_counters (s0, now,
923					 vlib_buffer_length_in_chain (vm, b0),
924					 thread_index);
925	  /* Per-user LRU list maintenance */
926	  nat44_session_update_lru (sm, s0, thread_index);
927	trace0:
928
929	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
930			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
931	    {
932	      snat_out2in_trace_t *t =
933		vlib_add_trace (vm, node, b0, sizeof (*t));
934	      t->sw_if_index = sw_if_index0;
935	      t->next_index = next0;
936	      t->session_index = ~0;
937	      if (s0)
938		t->session_index =
939		  s0 - sm->per_thread_data[thread_index].sessions;
940	    }
941
942	  pkts_processed += next0 == SNAT_OUT2IN_NEXT_LOOKUP;
943
944
945	  ip1 = vlib_buffer_get_current (b1);
946	  udp1 = ip4_next_header (ip1);
947	  tcp1 = (tcp_header_t *) udp1;
948	  icmp1 = (icmp46_header_t *) udp1;
949
950	  sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
951	  rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
952				   sw_if_index1);
953
954	  if (PREDICT_FALSE (ip1->ttl == 1))
955	    {
956	      vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
957	      icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
958					   ICMP4_time_exceeded_ttl_exceeded_in_transit,
959					   0);
960	      next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
961	      goto trace1;
962	    }
963
964	  proto1 = ip_proto_to_snat_proto (ip1->protocol);
965
966	  if (PREDICT_FALSE (proto1 == ~0))
967	    {
968	      if (nat_out2in_sm_unknown_proto (sm, b1, ip1, rx_fib_index1))
969		{
970		  if (!sm->forwarding_enabled)
971		    {
972		      b1->error =
973			node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
974		      next1 = SNAT_OUT2IN_NEXT_DROP;
975		    }
976		}
977	      other_packets++;
978	      goto trace1;
979	    }
980
981	  if (PREDICT_FALSE (ip4_is_fragment (ip1)))
982	    {
983	      next1 = SNAT_OUT2IN_NEXT_REASS;
984	      fragments++;
985	      goto trace1;
986	    }
987
988	  if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
989	    {
990	      next1 = icmp_out2in_slow_path
991		(sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
992		 next1, now, thread_index, &s1);
993	      icmp_packets++;
994	      goto trace1;
995	    }
996
997	  key1.addr = ip1->dst_address;
998	  key1.port = udp1->dst_port;
999	  key1.protocol = proto1;
1000	  key1.fib_index = rx_fib_index1;
1001
1002	  kv1.key = key1.as_u64;
1003
1004	  if (clib_bihash_search_8_8
1005	      (&sm->per_thread_data[thread_index].out2in, &kv1, &value1))
1006	    {
1007	      /* Try to match static mapping by external address and port,
1008	         destination address and port in packet */
1009	      if (snat_static_mapping_match
1010		  (sm, key1, &sm1, 1, 0, 0, 0, 0, &identity_nat1))
1011		{
1012		  /*
1013		   * Send DHCP packets to the ipv4 stack, or we won't
1014		   * be able to use dhcp client on the outside interface
1015		   */
1016		  if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1017				     && (udp1->dst_port ==
1018					 clib_host_to_net_u16
1019					 (UDP_DST_PORT_dhcp_to_client))))
1020		    {
1021		      vnet_feature_next (&next1, b1);
1022		      goto trace1;
1023		    }
1024
1025		  if (!sm->forwarding_enabled)
1026		    {
1027		      b1->error =
1028			node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1029		      next1 = SNAT_OUT2IN_NEXT_DROP;
1030		    }
1031		  goto trace1;
1032		}
1033
1034	      if (PREDICT_FALSE (identity_nat1))
1035		goto trace1;
1036
1037	      /* Create session initiated by host from external network */
1038	      s1 = create_session_for_static_mapping (sm, b1, sm1, key1, node,
1039						      thread_index, now);
1040	      if (!s1)
1041		{
1042		  next1 = SNAT_OUT2IN_NEXT_DROP;
1043		  goto trace1;
1044		}
1045	    }
1046	  else
1047	    s1 =
1048	      pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1049				 value1.value);
1050
1051	  old_addr1 = ip1->dst_address.as_u32;
1052	  ip1->dst_address = s1->in2out.addr;
1053	  new_addr1 = ip1->dst_address.as_u32;
1054	  vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1055
1056	  sum1 = ip1->checksum;
1057	  sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1058				 ip4_header_t,
1059				 dst_address /* changed member */ );
1060	  ip1->checksum = ip_csum_fold (sum1);
1061
1062	  if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1063	    {
1064	      old_port1 = tcp1->dst_port;
1065	      tcp1->dst_port = s1->in2out.port;
1066	      new_port1 = tcp1->dst_port;
1067
1068	      sum1 = tcp1->checksum;
1069	      sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1070				     ip4_header_t,
1071				     dst_address /* changed member */ );
1072
1073	      sum1 = ip_csum_update (sum1, old_port1, new_port1,
1074				     ip4_header_t /* cheat */ ,
1075				     length /* changed member */ );
1076	      tcp1->checksum = ip_csum_fold (sum1);
1077	      tcp_packets++;
1078	    }
1079	  else
1080	    {
1081	      old_port1 = udp1->dst_port;
1082	      udp1->dst_port = s1->in2out.port;
1083	      udp1->checksum = 0;
1084	      udp_packets++;
1085	    }
1086
1087	  /* Accounting */
1088	  nat44_session_update_counters (s1, now,
1089					 vlib_buffer_length_in_chain (vm, b1),
1090					 thread_index);
1091	  /* Per-user LRU list maintenance */
1092	  nat44_session_update_lru (sm, s1, thread_index);
1093	trace1:
1094
1095	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1096			     && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1097	    {
1098	      snat_out2in_trace_t *t =
1099		vlib_add_trace (vm, node, b1, sizeof (*t));
1100	      t->sw_if_index = sw_if_index1;
1101	      t->next_index = next1;
1102	      t->session_index = ~0;
1103	      if (s1)
1104		t->session_index =
1105		  s1 - sm->per_thread_data[thread_index].sessions;
1106	    }
1107
1108	  pkts_processed += next1 == SNAT_OUT2IN_NEXT_LOOKUP;
1109
1110	  /* verify speculative enqueues, maybe switch current next frame */
1111	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1112					   to_next, n_left_to_next,
1113					   bi0, bi1, next0, next1);
1114	}
1115
1116      while (n_left_from > 0 && n_left_to_next > 0)
1117	{
1118	  u32 bi0;
1119	  vlib_buffer_t *b0;
1120	  u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1121	  u32 sw_if_index0;
1122	  ip4_header_t *ip0;
1123	  ip_csum_t sum0;
1124	  u32 new_addr0, old_addr0;
1125	  u16 new_port0, old_port0;
1126	  udp_header_t *udp0;
1127	  tcp_header_t *tcp0;
1128	  icmp46_header_t *icmp0;
1129	  snat_session_key_t key0, sm0;
1130	  u32 rx_fib_index0;
1131	  u32 proto0;
1132	  snat_session_t *s0 = 0;
1133	  clib_bihash_kv_8_8_t kv0, value0;
1134	  u8 identity_nat0;
1135
1136	  /* speculatively enqueue b0 to the current next frame */
1137	  bi0 = from[0];
1138	  to_next[0] = bi0;
1139	  from += 1;
1140	  to_next += 1;
1141	  n_left_from -= 1;
1142	  n_left_to_next -= 1;
1143
1144	  b0 = vlib_get_buffer (vm, bi0);
1145
1146	  vnet_buffer (b0)->snat.flags = 0;
1147
1148	  ip0 = vlib_buffer_get_current (b0);
1149	  udp0 = ip4_next_header (ip0);
1150	  tcp0 = (tcp_header_t *) udp0;
1151	  icmp0 = (icmp46_header_t *) udp0;
1152
1153	  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1154	  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1155				   sw_if_index0);
1156
1157	  proto0 = ip_proto_to_snat_proto (ip0->protocol);
1158
1159	  if (PREDICT_FALSE (proto0 == ~0))
1160	    {
1161	      if (nat_out2in_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1162		{
1163		  if (!sm->forwarding_enabled)
1164		    {
1165		      b0->error =
1166			node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1167		      next0 = SNAT_OUT2IN_NEXT_DROP;
1168		    }
1169		}
1170	      other_packets++;
1171	      goto trace00;
1172	    }
1173
1174	  if (PREDICT_FALSE (ip0->ttl == 1))
1175	    {
1176	      vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1177	      icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1178					   ICMP4_time_exceeded_ttl_exceeded_in_transit,
1179					   0);
1180	      next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1181	      goto trace00;
1182	    }
1183
1184	  if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1185	    {
1186	      next0 = SNAT_OUT2IN_NEXT_REASS;
1187	      fragments++;
1188	      goto trace00;
1189	    }
1190
1191	  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1192	    {
1193	      next0 = icmp_out2in_slow_path
1194		(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1195		 next0, now, thread_index, &s0);
1196	      icmp_packets++;
1197	      goto trace00;
1198	    }
1199
1200	  key0.addr = ip0->dst_address;
1201	  key0.port = udp0->dst_port;
1202	  key0.protocol = proto0;
1203	  key0.fib_index = rx_fib_index0;
1204
1205	  kv0.key = key0.as_u64;
1206
1207	  if (clib_bihash_search_8_8
1208	      (&sm->per_thread_data[thread_index].out2in, &kv0, &value0))
1209	    {
1210	      /* Try to match static mapping by external address and port,
1211	         destination address and port in packet */
1212	      if (snat_static_mapping_match
1213		  (sm, key0, &sm0, 1, 0, 0, 0, 0, &identity_nat0))
1214		{
1215		  /*
1216		   * Send DHCP packets to the ipv4 stack, or we won't
1217		   * be able to use dhcp client on the outside interface
1218		   */
1219		  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1220				     && (udp0->dst_port ==
1221					 clib_host_to_net_u16
1222					 (UDP_DST_PORT_dhcp_to_client))))
1223		    {
1224		      vnet_feature_next (&next0, b0);
1225		      goto trace00;
1226		    }
1227
1228		  if (!sm->forwarding_enabled)
1229		    {
1230		      b0->error =
1231			node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1232		      next0 = SNAT_OUT2IN_NEXT_DROP;
1233		    }
1234		  goto trace00;
1235		}
1236
1237	      if (PREDICT_FALSE (identity_nat0))
1238		goto trace00;
1239
1240	      /* Create session initiated by host from external network */
1241	      s0 = create_session_for_static_mapping (sm, b0, sm0, key0, node,
1242						      thread_index, now);
1243	      if (!s0)
1244		{
1245		  next0 = SNAT_OUT2IN_NEXT_DROP;
1246		  goto trace00;
1247		}
1248	    }
1249	  else
1250	    s0 =
1251	      pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1252				 value0.value);
1253
1254	  old_addr0 = ip0->dst_address.as_u32;
1255	  ip0->dst_address = s0->in2out.addr;
1256	  new_addr0 = ip0->dst_address.as_u32;
1257	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1258
1259	  sum0 = ip0->checksum;
1260	  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1261				 ip4_header_t,
1262				 dst_address /* changed member */ );
1263	  ip0->checksum = ip_csum_fold (sum0);
1264
1265	  if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1266	    {
1267	      old_port0 = tcp0->dst_port;
1268	      tcp0->dst_port = s0->in2out.port;
1269	      new_port0 = tcp0->dst_port;
1270
1271	      sum0 = tcp0->checksum;
1272	      sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1273				     ip4_header_t,
1274				     dst_address /* changed member */ );
1275
1276	      sum0 = ip_csum_update (sum0, old_port0, new_port0,
1277				     ip4_header_t /* cheat */ ,
1278				     length /* changed member */ );
1279	      tcp0->checksum = ip_csum_fold (sum0);
1280	      tcp_packets++;
1281	    }
1282	  else
1283	    {
1284	      old_port0 = udp0->dst_port;
1285	      udp0->dst_port = s0->in2out.port;
1286	      udp0->checksum = 0;
1287	      udp_packets++;
1288	    }
1289
1290	  /* Accounting */
1291	  nat44_session_update_counters (s0, now,
1292					 vlib_buffer_length_in_chain (vm, b0),
1293					 thread_index);
1294	  /* Per-user LRU list maintenance */
1295	  nat44_session_update_lru (sm, s0, thread_index);
1296	trace00:
1297
1298	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1299			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1300	    {
1301	      snat_out2in_trace_t *t =
1302		vlib_add_trace (vm, node, b0, sizeof (*t));
1303	      t->sw_if_index = sw_if_index0;
1304	      t->next_index = next0;
1305	      t->session_index = ~0;
1306	      if (s0)
1307		t->session_index =
1308		  s0 - sm->per_thread_data[thread_index].sessions;
1309	    }
1310
1311	  pkts_processed += next0 == SNAT_OUT2IN_NEXT_LOOKUP;
1312
1313	  /* verify speculative enqueue, maybe switch current next frame */
1314	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1315					   to_next, n_left_to_next,
1316					   bi0, next0);
1317	}
1318
1319      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1320    }
1321
1322  vlib_node_increment_counter (vm, sm->out2in_node_index,
1323			       SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1324			       pkts_processed);
1325  vlib_node_increment_counter (vm, sm->out2in_node_index,
1326			       SNAT_OUT2IN_ERROR_TCP_PACKETS, tcp_packets);
1327  vlib_node_increment_counter (vm, sm->out2in_node_index,
1328			       SNAT_OUT2IN_ERROR_UDP_PACKETS, udp_packets);
1329  vlib_node_increment_counter (vm, sm->out2in_node_index,
1330			       SNAT_OUT2IN_ERROR_ICMP_PACKETS, icmp_packets);
1331  vlib_node_increment_counter (vm, sm->out2in_node_index,
1332			       SNAT_OUT2IN_ERROR_OTHER_PACKETS,
1333			       other_packets);
1334  vlib_node_increment_counter (vm, sm->out2in_node_index,
1335			       SNAT_OUT2IN_ERROR_FRAGMENTS, fragments);
1336
1337  return frame->n_vectors;
1338}
1339
1340/* *INDENT-OFF* */
1341VLIB_REGISTER_NODE (snat_out2in_node) = {
1342  .name = "nat44-out2in",
1343  .vector_size = sizeof (u32),
1344  .format_trace = format_snat_out2in_trace,
1345  .type = VLIB_NODE_TYPE_INTERNAL,
1346
1347  .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1348  .error_strings = snat_out2in_error_strings,
1349
1350  .runtime_data_bytes = sizeof (snat_runtime_t),
1351
1352  .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1353
1354  /* edit / add dispositions here */
1355  .next_nodes = {
1356    [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1357    [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1358    [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1359    [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1360  },
1361};
1362/* *INDENT-ON* */
1363
1364VLIB_NODE_FN (nat44_out2in_reass_node) (vlib_main_t * vm,
1365					vlib_node_runtime_t * node,
1366					vlib_frame_t * frame)
1367{
1368  u32 n_left_from, *from, *to_next;
1369  snat_out2in_next_t next_index;
1370  u32 pkts_processed = 0, cached_fragments = 0;
1371  snat_main_t *sm = &snat_main;
1372  f64 now = vlib_time_now (vm);
1373  u32 thread_index = vm->thread_index;
1374  snat_main_per_thread_data_t *per_thread_data =
1375    &sm->per_thread_data[thread_index];
1376  u32 *fragments_to_drop = 0;
1377  u32 *fragments_to_loopback = 0;
1378
1379  from = vlib_frame_vector_args (frame);
1380  n_left_from = frame->n_vectors;
1381  next_index = node->cached_next_index;
1382
1383  while (n_left_from > 0)
1384    {
1385      u32 n_left_to_next;
1386
1387      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1388
1389      while (n_left_from > 0 && n_left_to_next > 0)
1390	{
1391	  u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1392	  vlib_buffer_t *b0;
1393	  u32 next0;
1394	  u8 cached0 = 0;
1395	  ip4_header_t *ip0;
1396	  nat_reass_ip4_t *reass0;
1397	  udp_header_t *udp0;
1398	  tcp_header_t *tcp0;
1399	  icmp46_header_t *icmp0;
1400	  snat_session_key_t key0, sm0;
1401	  clib_bihash_kv_8_8_t kv0, value0;
1402	  snat_session_t *s0 = 0;
1403	  u16 old_port0, new_port0;
1404	  ip_csum_t sum0;
1405	  u8 identity_nat0;
1406
1407	  /* speculatively enqueue b0 to the current next frame */
1408	  bi0 = from[0];
1409	  to_next[0] = bi0;
1410	  from += 1;
1411	  to_next += 1;
1412	  n_left_from -= 1;
1413	  n_left_to_next -= 1;
1414
1415	  b0 = vlib_get_buffer (vm, bi0);
1416	  next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1417
1418	  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1419	  rx_fib_index0 =
1420	    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1421						 sw_if_index0);
1422
1423	  if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1424	    {
1425	      next0 = SNAT_OUT2IN_NEXT_DROP;
1426	      b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1427	      goto trace0;
1428	    }
1429
1430	  ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1431	  udp0 = ip4_next_header (ip0);
1432	  tcp0 = (tcp_header_t *) udp0;
1433	  icmp0 = (icmp46_header_t *) udp0;
1434	  proto0 = ip_proto_to_snat_proto (ip0->protocol);
1435
1436	  reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1437						 ip0->dst_address,
1438						 ip0->fragment_id,
1439						 ip0->protocol,
1440						 1, &fragments_to_drop);
1441
1442	  if (PREDICT_FALSE (!reass0))
1443	    {
1444	      next0 = SNAT_OUT2IN_NEXT_DROP;
1445	      b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1446	      nat_elog_notice ("maximum reassemblies exceeded");
1447	      goto trace0;
1448	    }
1449
1450	  if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1451	    {
1452	      if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1453		{
1454		  next0 = icmp_out2in_slow_path
1455		    (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1456		     next0, now, thread_index, &s0);
1457
1458		  if (PREDICT_TRUE (next0 != SNAT_OUT2IN_NEXT_DROP))
1459		    {
1460		      if (s0)
1461			reass0->sess_index = s0 - per_thread_data->sessions;
1462		      else
1463			reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1464		      reass0->thread_index = thread_index;
1465		      nat_ip4_reass_get_frags (reass0,
1466					       &fragments_to_loopback);
1467		    }
1468
1469		  goto trace0;
1470		}
1471
1472	      key0.addr = ip0->dst_address;
1473	      key0.port = udp0->dst_port;
1474	      key0.protocol = proto0;
1475	      key0.fib_index = rx_fib_index0;
1476	      kv0.key = key0.as_u64;
1477
1478	      if (clib_bihash_search_8_8
1479		  (&per_thread_data->out2in, &kv0, &value0))
1480		{
1481		  /* Try to match static mapping by external address and port,
1482		     destination address and port in packet */
1483		  if (snat_static_mapping_match
1484		      (sm, key0, &sm0, 1, 0, 0, 0, 0, &identity_nat0))
1485		    {
1486		      /*
1487		       * Send DHCP packets to the ipv4 stack, or we won't
1488		       * be able to use dhcp client on the outside interface
1489		       */
1490		      if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1491					 && (udp0->dst_port
1492					     ==
1493					     clib_host_to_net_u16
1494					     (UDP_DST_PORT_dhcp_to_client))))
1495			{
1496			  vnet_feature_next (&next0, b0);
1497			  goto trace0;
1498			}
1499
1500		      if (!sm->forwarding_enabled)
1501			{
1502			  b0->error =
1503			    node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1504			  next0 = SNAT_OUT2IN_NEXT_DROP;
1505			}
1506		      else
1507			{
1508			  reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1509			  nat_ip4_reass_get_frags (reass0,
1510						   &fragments_to_loopback);
1511			}
1512		      goto trace0;
1513		    }
1514
1515		  if (PREDICT_FALSE (identity_nat0))
1516		    goto trace0;
1517
1518		  /* Create session initiated by host from external network */
1519		  s0 =
1520		    create_session_for_static_mapping (sm, b0, sm0, key0,
1521						       node, thread_index,
1522						       now);
1523		  if (!s0)
1524		    {
1525		      b0->error =
1526			node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1527		      next0 = SNAT_OUT2IN_NEXT_DROP;
1528		      goto trace0;
1529		    }
1530		  reass0->sess_index = s0 - per_thread_data->sessions;
1531		  reass0->thread_index = thread_index;
1532		}
1533	      else
1534		{
1535		  s0 = pool_elt_at_index (per_thread_data->sessions,
1536					  value0.value);
1537		  reass0->sess_index = value0.value;
1538		}
1539	      nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1540	    }
1541	  else
1542	    {
1543	      if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
1544		goto trace0;
1545	      if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1546		{
1547		  if (nat_ip4_reass_add_fragment
1548		      (thread_index, reass0, bi0, &fragments_to_drop))
1549		    {
1550		      b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1551		      nat_elog_notice
1552			("maximum fragments per reassembly exceeded");
1553		      next0 = SNAT_OUT2IN_NEXT_DROP;
1554		      goto trace0;
1555		    }
1556		  cached0 = 1;
1557		  goto trace0;
1558		}
1559	      s0 = pool_elt_at_index (per_thread_data->sessions,
1560				      reass0->sess_index);
1561	    }
1562
1563	  old_addr0 = ip0->dst_address.as_u32;
1564	  ip0->dst_address = s0->in2out.addr;
1565	  new_addr0 = ip0->dst_address.as_u32;
1566	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1567
1568	  sum0 = ip0->checksum;
1569	  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1570				 ip4_header_t,
1571				 dst_address /* changed member */ );
1572	  ip0->checksum = ip_csum_fold (sum0);
1573
1574	  if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1575	    {
1576	      if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1577		{
1578		  old_port0 = tcp0->dst_port;
1579		  tcp0->dst_port = s0->in2out.port;
1580		  new_port0 = tcp0->dst_port;
1581
1582		  sum0 = tcp0->checksum;
1583		  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1584					 ip4_header_t,
1585					 dst_address /* changed member */ );
1586
1587		  sum0 = ip_csum_update (sum0, old_port0, new_port0,
1588					 ip4_header_t /* cheat */ ,
1589					 length /* changed member */ );
1590		  tcp0->checksum = ip_csum_fold (sum0);
1591		}
1592	      else
1593		{
1594		  old_port0 = udp0->dst_port;
1595		  udp0->dst_port = s0->in2out.port;
1596		  udp0->checksum = 0;
1597		}
1598	    }
1599
1600	  /* Accounting */
1601	  nat44_session_update_counters (s0, now,
1602					 vlib_buffer_length_in_chain (vm, b0),
1603					 thread_index);
1604	  /* Per-user LRU list maintenance */
1605	  nat44_session_update_lru (sm, s0, thread_index);
1606
1607	trace0:
1608	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1609			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1610	    {
1611	      nat44_reass_trace_t *t =
1612		vlib_add_trace (vm, node, b0, sizeof (*t));
1613	      t->cached = cached0;
1614	      t->sw_if_index = sw_if_index0;
1615	      t->next_index = next0;
1616	    }
1617
1618	  if (cached0)
1619	    {
1620	      n_left_to_next++;
1621	      to_next--;
1622	      cached_fragments++;
1623	    }
1624	  else
1625	    {
1626	      pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1627
1628	      /* verify speculative enqueue, maybe switch current next frame */
1629	      vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1630					       to_next, n_left_to_next,
1631					       bi0, next0);
1632	    }
1633
1634	  if (n_left_from == 0 && vec_len (fragments_to_loopback))
1635	    {
1636	      from = vlib_frame_vector_args (frame);
1637	      u32 len = vec_len (fragments_to_loopback);
1638	      if (len <= VLIB_FRAME_SIZE)
1639		{
1640		  clib_memcpy_fast (from, fragments_to_loopback,
1641				    sizeof (u32) * len);
1642		  n_left_from = len;
1643		  vec_reset_length (fragments_to_loopback);
1644		}
1645	      else
1646		{
1647		  clib_memcpy_fast (from, fragments_to_loopback +
1648				    (len - VLIB_FRAME_SIZE),
1649				    sizeof (u32) * VLIB_FRAME_SIZE);
1650		  n_left_from = VLIB_FRAME_SIZE;
1651		  _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1652		}
1653	    }
1654	}
1655
1656      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1657    }
1658
1659  vlib_node_increment_counter (vm, sm->out2in_reass_node_index,
1660			       SNAT_OUT2IN_ERROR_PROCESSED_FRAGMENTS,
1661			       pkts_processed);
1662  vlib_node_increment_counter (vm, sm->out2in_reass_node_index,
1663			       SNAT_OUT2IN_ERROR_CACHED_FRAGMENTS,
1664			       cached_fragments);
1665
1666  nat_send_all_to_node (vm, fragments_to_drop, node,
1667			&node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
1668			SNAT_OUT2IN_NEXT_DROP);
1669
1670  vec_free (fragments_to_drop);
1671  vec_free (fragments_to_loopback);
1672  return frame->n_vectors;
1673}
1674
1675/* *INDENT-OFF* */
1676VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
1677  .name = "nat44-out2in-reass",
1678  .vector_size = sizeof (u32),
1679  .format_trace = format_nat44_reass_trace,
1680  .type = VLIB_NODE_TYPE_INTERNAL,
1681
1682  .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1683  .error_strings = snat_out2in_error_strings,
1684
1685  .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1686
1687  /* edit / add dispositions here */
1688  .next_nodes = {
1689    [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1690    [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1691    [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1692    [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1693  },
1694};
1695/* *INDENT-ON* */
1696
1697VLIB_NODE_FN (snat_out2in_fast_node) (vlib_main_t * vm,
1698				      vlib_node_runtime_t * node,
1699				      vlib_frame_t * frame)
1700{
1701  u32 n_left_from, *from, *to_next;
1702  snat_out2in_next_t next_index;
1703  u32 pkts_processed = 0;
1704  snat_main_t *sm = &snat_main;
1705
1706  from = vlib_frame_vector_args (frame);
1707  n_left_from = frame->n_vectors;
1708  next_index = node->cached_next_index;
1709
1710  while (n_left_from > 0)
1711    {
1712      u32 n_left_to_next;
1713
1714      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1715
1716      while (n_left_from > 0 && n_left_to_next > 0)
1717	{
1718	  u32 bi0;
1719	  vlib_buffer_t *b0;
1720	  u32 next0 = SNAT_OUT2IN_NEXT_DROP;
1721	  u32 sw_if_index0;
1722	  ip4_header_t *ip0;
1723	  ip_csum_t sum0;
1724	  u32 new_addr0, old_addr0;
1725	  u16 new_port0, old_port0;
1726	  udp_header_t *udp0;
1727	  tcp_header_t *tcp0;
1728	  icmp46_header_t *icmp0;
1729	  snat_session_key_t key0, sm0;
1730	  u32 proto0;
1731	  u32 rx_fib_index0;
1732
1733	  /* speculatively enqueue b0 to the current next frame */
1734	  bi0 = from[0];
1735	  to_next[0] = bi0;
1736	  from += 1;
1737	  to_next += 1;
1738	  n_left_from -= 1;
1739	  n_left_to_next -= 1;
1740
1741	  b0 = vlib_get_buffer (vm, bi0);
1742
1743	  ip0 = vlib_buffer_get_current (b0);
1744	  udp0 = ip4_next_header (ip0);
1745	  tcp0 = (tcp_header_t *) udp0;
1746	  icmp0 = (icmp46_header_t *) udp0;
1747
1748	  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1749	  rx_fib_index0 =
1750	    ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
1751
1752	  vnet_feature_next (&next0, b0);
1753
1754	  if (PREDICT_FALSE (ip0->ttl == 1))
1755	    {
1756	      vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1757	      icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1758					   ICMP4_time_exceeded_ttl_exceeded_in_transit,
1759					   0);
1760	      next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1761	      goto trace00;
1762	    }
1763
1764	  proto0 = ip_proto_to_snat_proto (ip0->protocol);
1765
1766	  if (PREDICT_FALSE (proto0 == ~0))
1767	    goto trace00;
1768
1769	  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1770	    {
1771	      next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0,
1772				   rx_fib_index0, node, next0, ~0, 0, 0);
1773	      goto trace00;
1774	    }
1775
1776	  key0.addr = ip0->dst_address;
1777	  key0.port = udp0->dst_port;
1778	  key0.fib_index = rx_fib_index0;
1779
1780	  if (snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0, 0))
1781	    {
1782	      b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1783	      goto trace00;
1784	    }
1785
1786	  new_addr0 = sm0.addr.as_u32;
1787	  new_port0 = sm0.port;
1788	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1789	  old_addr0 = ip0->dst_address.as_u32;
1790	  ip0->dst_address.as_u32 = new_addr0;
1791
1792	  sum0 = ip0->checksum;
1793	  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1794				 ip4_header_t,
1795				 dst_address /* changed member */ );
1796	  ip0->checksum = ip_csum_fold (sum0);
1797
1798	  if (PREDICT_FALSE (new_port0 != udp0->dst_port))
1799	    {
1800	      if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1801		{
1802		  old_port0 = tcp0->dst_port;
1803		  tcp0->dst_port = new_port0;
1804
1805		  sum0 = tcp0->checksum;
1806		  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1807					 ip4_header_t,
1808					 dst_address /* changed member */ );
1809
1810		  sum0 = ip_csum_update (sum0, old_port0, new_port0,
1811					 ip4_header_t /* cheat */ ,
1812					 length /* changed member */ );
1813		  tcp0->checksum = ip_csum_fold (sum0);
1814		}
1815	      else
1816		{
1817		  old_port0 = udp0->dst_port;
1818		  udp0->dst_port = new_port0;
1819		  udp0->checksum = 0;
1820		}
1821	    }
1822	  else
1823	    {
1824	      if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1825		{
1826		  sum0 = tcp0->checksum;
1827		  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1828					 ip4_header_t,
1829					 dst_address /* changed member */ );
1830
1831		  tcp0->checksum = ip_csum_fold (sum0);
1832		}
1833	    }
1834
1835	trace00:
1836
1837	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1838			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1839	    {
1840	      snat_out2in_trace_t *t =
1841		vlib_add_trace (vm, node, b0, sizeof (*t));
1842	      t->sw_if_index = sw_if_index0;
1843	      t->next_index = next0;
1844	    }
1845
1846	  pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1847
1848	  /* verify speculative enqueue, maybe switch current next frame */
1849	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1850					   to_next, n_left_to_next,
1851					   bi0, next0);
1852	}
1853
1854      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1855    }
1856
1857  vlib_node_increment_counter (vm, sm->out2in_fast_node_index,
1858			       SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1859			       pkts_processed);
1860  return frame->n_vectors;
1861}
1862
1863/* *INDENT-OFF* */
1864VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
1865  .name = "nat44-out2in-fast",
1866  .vector_size = sizeof (u32),
1867  .format_trace = format_snat_out2in_fast_trace,
1868  .type = VLIB_NODE_TYPE_INTERNAL,
1869
1870  .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1871  .error_strings = snat_out2in_error_strings,
1872
1873  .runtime_data_bytes = sizeof (snat_runtime_t),
1874
1875  .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1876
1877  /* edit / add dispositions here */
1878  .next_nodes = {
1879    [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1880    [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1881    [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1882    [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1883  },
1884};
1885/* *INDENT-ON* */
1886
1887/*
1888 * fd.io coding-style-patch-verification: ON
1889 *
1890 * Local Variables:
1891 * eval: (c-set-style "gnu")
1892 * End:
1893 */
1894