out2in_ed.c revision 22bb417e
1/*
2 * Copyright (c) 2018 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15/**
16 * @file
17 * @brief NAT44 endpoint-dependent outside to inside network translation
18 */
19
20#include <vlib/vlib.h>
21#include <vnet/vnet.h>
22#include <vnet/pg/pg.h>
23#include <vnet/ip/ip.h>
24#include <vnet/ethernet/ethernet.h>
25#include <vnet/fib/ip4_fib.h>
26#include <vnet/udp/udp.h>
27#include <vppinfra/error.h>
28#include <nat/nat.h>
29#include <nat/nat_ipfix_logging.h>
30#include <nat/nat_reass.h>
31#include <nat/nat_inlines.h>
32#include <nat/nat_syslog.h>
33#include <nat/nat_ha.h>
34
35static char *nat_out2in_ed_error_strings[] = {
36#define _(sym,string) string,
37  foreach_nat_out2in_ed_error
38#undef _
39};
40
41typedef enum
42{
43  NAT44_ED_OUT2IN_NEXT_DROP,
44  NAT44_ED_OUT2IN_NEXT_LOOKUP,
45  NAT44_ED_OUT2IN_NEXT_ICMP_ERROR,
46  NAT44_ED_OUT2IN_NEXT_IN2OUT,
47  NAT44_ED_OUT2IN_NEXT_SLOW_PATH,
48  NAT44_ED_OUT2IN_NEXT_REASS,
49  NAT44_ED_OUT2IN_N_NEXT,
50} nat44_ed_out2in_next_t;
51
52typedef struct
53{
54  u32 sw_if_index;
55  u32 next_index;
56  u32 session_index;
57  u32 is_slow_path;
58} nat44_ed_out2in_trace_t;
59
60static u8 *
61format_nat44_ed_out2in_trace (u8 * s, va_list * args)
62{
63  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
64  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
65  nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
66  char *tag;
67
68  tag =
69    t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
70    "NAT44_OUT2IN_ED_FAST_PATH";
71
72  s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
73	      t->sw_if_index, t->next_index, t->session_index);
74
75  return s;
76}
77
78static inline u32
79icmp_out2in_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
80			  ip4_header_t * ip0, icmp46_header_t * icmp0,
81			  u32 sw_if_index0, u32 rx_fib_index0,
82			  vlib_node_runtime_t * node, u32 next0, f64 now,
83			  u32 thread_index, snat_session_t ** p_s0)
84{
85  next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
86		       next0, thread_index, p_s0, 0);
87  snat_session_t *s0 = *p_s0;
88  if (PREDICT_TRUE (next0 != NAT44_ED_OUT2IN_NEXT_DROP && s0))
89    {
90      /* Accounting */
91      nat44_session_update_counters (s0, now,
92				     vlib_buffer_length_in_chain
93				     (sm->vlib_main, b0), thread_index);
94      /* Per-user LRU list maintenance */
95      nat44_session_update_lru (sm, s0, thread_index);
96    }
97  return next0;
98}
99
100#ifndef CLIB_MARCH_VARIANT
101int
102nat44_o2i_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
103{
104  snat_main_t *sm = &snat_main;
105  nat44_is_idle_session_ctx_t *ctx = arg;
106  snat_session_t *s;
107  u64 sess_timeout_time;
108  nat_ed_ses_key_t ed_key;
109  clib_bihash_kv_16_8_t ed_kv;
110  int i;
111  snat_address_t *a;
112  snat_session_key_t key;
113  snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
114						       ctx->thread_index);
115
116  s = pool_elt_at_index (tsm->sessions, kv->value);
117  sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
118  if (ctx->now >= sess_timeout_time)
119    {
120      ed_key.l_addr = s->in2out.addr;
121      ed_key.r_addr = s->ext_host_addr;
122      ed_key.fib_index = s->in2out.fib_index;
123      if (snat_is_unk_proto_session (s))
124	{
125	  ed_key.proto = s->in2out.port;
126	  ed_key.r_port = 0;
127	  ed_key.l_port = 0;
128	}
129      else
130	{
131	  ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
132	  ed_key.l_port = s->in2out.port;
133	  ed_key.r_port = s->ext_host_port;
134	}
135      if (is_twice_nat_session (s))
136	{
137	  ed_key.r_addr = s->ext_host_nat_addr;
138	  ed_key.r_port = s->ext_host_nat_port;
139	}
140      ed_kv.key[0] = ed_key.as_u64[0];
141      ed_kv.key[1] = ed_key.as_u64[1];
142      if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
143	nat_elog_warn ("in2out_ed key del failed");
144
145      if (snat_is_unk_proto_session (s))
146	goto delete;
147
148      snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
149					   s->in2out.addr.as_u32,
150					   s->out2in.addr.as_u32,
151					   s->in2out.protocol,
152					   s->in2out.port,
153					   s->out2in.port,
154					   s->in2out.fib_index);
155
156      nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
157			     &s->in2out.addr, s->in2out.port,
158			     &s->ext_host_nat_addr, s->ext_host_nat_port,
159			     &s->out2in.addr, s->out2in.port,
160			     &s->ext_host_addr, s->ext_host_port,
161			     s->in2out.protocol, is_twice_nat_session (s));
162
163      nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
164		   s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
165		   ctx->thread_index);
166
167      if (is_twice_nat_session (s))
168	{
169	  for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
170	    {
171	      key.protocol = s->in2out.protocol;
172	      key.port = s->ext_host_nat_port;
173	      a = sm->twice_nat_addresses + i;
174	      if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
175		{
176		  snat_free_outside_address_and_port (sm->twice_nat_addresses,
177						      ctx->thread_index,
178						      &key);
179		  break;
180		}
181	    }
182	}
183
184      if (snat_is_session_static (s))
185	goto delete;
186
187      snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
188					  &s->out2in);
189    delete:
190      nat44_delete_session (sm, s, ctx->thread_index);
191      return 1;
192    }
193
194  return 0;
195}
196#endif
197
198static snat_session_t *
199create_session_for_static_mapping_ed (snat_main_t * sm,
200				      vlib_buffer_t * b,
201				      snat_session_key_t l_key,
202				      snat_session_key_t e_key,
203				      vlib_node_runtime_t * node,
204				      u32 thread_index,
205				      twice_nat_type_t twice_nat,
206				      lb_nat_type_t lb_nat, f64 now)
207{
208  snat_session_t *s;
209  snat_user_t *u;
210  ip4_header_t *ip;
211  udp_header_t *udp;
212  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
213  clib_bihash_kv_16_8_t kv;
214  snat_session_key_t eh_key;
215  nat44_is_idle_session_ctx_t ctx;
216
217  if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
218    {
219      b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
220      nat_elog_notice ("maximum sessions exceeded");
221      return 0;
222    }
223
224  u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index, thread_index);
225  if (!u)
226    {
227      nat_elog_warn ("create NAT user failed");
228      return 0;
229    }
230
231  s = nat_ed_session_alloc (sm, u, thread_index, now);
232  if (!s)
233    {
234      nat44_delete_user_with_no_session (sm, u, thread_index);
235      nat_elog_warn ("create NAT session failed");
236      return 0;
237    }
238
239  ip = vlib_buffer_get_current (b);
240  udp = ip4_next_header (ip);
241
242  s->ext_host_addr.as_u32 = ip->src_address.as_u32;
243  s->ext_host_port = e_key.protocol == SNAT_PROTOCOL_ICMP ? 0 : udp->src_port;
244  s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
245  if (lb_nat)
246    s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
247  if (lb_nat == AFFINITY_LB_NAT)
248    s->flags |= SNAT_SESSION_FLAG_AFFINITY;
249  s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
250  s->out2in = e_key;
251  s->in2out = l_key;
252  s->in2out.protocol = s->out2in.protocol;
253  user_session_increment (sm, u, 1);
254
255  /* Add to lookup tables */
256  make_ed_kv (&kv, &e_key.addr, &s->ext_host_addr, ip->protocol,
257	      e_key.fib_index, e_key.port, s->ext_host_port);
258  kv.value = s - tsm->sessions;
259  ctx.now = now;
260  ctx.thread_index = thread_index;
261  if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, &kv,
262					       nat44_o2i_ed_is_idle_session_cb,
263					       &ctx))
264    nat_elog_notice ("out2in-ed key add failed");
265
266  if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
267				 ip->src_address.as_u32 == l_key.addr.as_u32))
268    {
269      eh_key.protocol = e_key.protocol;
270      if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
271					       thread_index, &eh_key,
272					       sm->port_per_thread,
273					       tsm->snat_thread_index))
274	{
275	  b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
276	  nat44_delete_session (sm, s, thread_index);
277	  if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 0))
278	    nat_elog_notice ("out2in-ed key del failed");
279	  return 0;
280	}
281      s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
282      s->ext_host_nat_port = eh_key.port;
283      s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
284      make_ed_kv (&kv, &l_key.addr, &s->ext_host_nat_addr, ip->protocol,
285		  l_key.fib_index, l_key.port, s->ext_host_nat_port);
286    }
287  else
288    {
289      make_ed_kv (&kv, &l_key.addr, &s->ext_host_addr, ip->protocol,
290		  l_key.fib_index, l_key.port, s->ext_host_port);
291    }
292  kv.value = s - tsm->sessions;
293  if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &kv,
294					       nat44_i2o_ed_is_idle_session_cb,
295					       &ctx))
296    nat_elog_notice ("in2out-ed key add failed");
297
298  snat_ipfix_logging_nat44_ses_create (thread_index,
299				       s->in2out.addr.as_u32,
300				       s->out2in.addr.as_u32,
301				       s->in2out.protocol,
302				       s->in2out.port,
303				       s->out2in.port, s->in2out.fib_index);
304
305  nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
306			 &s->in2out.addr, s->in2out.port,
307			 &s->ext_host_nat_addr, s->ext_host_nat_port,
308			 &s->out2in.addr, s->out2in.port,
309			 &s->ext_host_addr, s->ext_host_port,
310			 s->in2out.protocol, is_twice_nat_session (s));
311
312  nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
313	       s->out2in.port, &s->ext_host_addr, s->ext_host_port,
314	       &s->ext_host_nat_addr, s->ext_host_nat_port,
315	       s->in2out.protocol, s->in2out.fib_index, s->flags,
316	       thread_index, 0);
317
318  return s;
319}
320
321static int
322next_src_nat (snat_main_t * sm, ip4_header_t * ip, u8 proto, u16 src_port,
323	      u16 dst_port, u32 thread_index, u32 rx_fib_index)
324{
325  clib_bihash_kv_16_8_t kv, value;
326  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
327
328  make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto,
329	      rx_fib_index, src_port, dst_port);
330  if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
331    return 1;
332
333  return 0;
334}
335
336static void
337create_bypass_for_fwd (snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
338		       u32 thread_index)
339{
340  nat_ed_ses_key_t key;
341  clib_bihash_kv_16_8_t kv, value;
342  udp_header_t *udp;
343  snat_user_t *u;
344  snat_session_t *s = 0;
345  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
346  f64 now = vlib_time_now (sm->vlib_main);
347
348  if (ip->protocol == IP_PROTOCOL_ICMP)
349    {
350      if (get_icmp_o2i_ed_key (ip, &key))
351	return;
352    }
353  else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
354    {
355      udp = ip4_next_header (ip);
356      key.r_addr = ip->src_address;
357      key.l_addr = ip->dst_address;
358      key.proto = ip->protocol;
359      key.l_port = udp->dst_port;
360      key.r_port = udp->src_port;
361    }
362  else
363    {
364      key.r_addr = ip->src_address;
365      key.l_addr = ip->dst_address;
366      key.proto = ip->protocol;
367      key.l_port = key.r_port = 0;
368    }
369  key.fib_index = 0;
370  kv.key[0] = key.as_u64[0];
371  kv.key[1] = key.as_u64[1];
372
373  if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
374    {
375      s = pool_elt_at_index (tsm->sessions, value.value);
376    }
377  else
378    {
379      u32 proto;
380
381      if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
382	return;
383
384      u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index,
385				  thread_index);
386      if (!u)
387	{
388	  nat_elog_warn ("create NAT user failed");
389	  return;
390	}
391
392      s = nat_ed_session_alloc (sm, u, thread_index, now);
393      if (!s)
394	{
395	  nat44_delete_user_with_no_session (sm, u, thread_index);
396	  nat_elog_warn ("create NAT session failed");
397	  return;
398	}
399
400      proto = ip_proto_to_snat_proto (key.proto);
401
402      s->ext_host_addr = key.r_addr;
403      s->ext_host_port = key.r_port;
404      s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
405      s->out2in.addr = key.l_addr;
406      s->out2in.port = key.l_port;
407      s->out2in.protocol = proto;
408      if (proto == ~0)
409	{
410	  s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
411	  s->out2in.port = ip->protocol;
412	}
413      s->out2in.fib_index = 0;
414      s->in2out = s->out2in;
415      user_session_increment (sm, u, 0);
416
417      kv.value = s - tsm->sessions;
418      if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
419	nat_elog_notice ("in2out_ed key add failed");
420    }
421
422  if (ip->protocol == IP_PROTOCOL_TCP)
423    {
424      tcp_header_t *tcp = ip4_next_header (ip);
425      if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
426	return;
427    }
428
429  /* Accounting */
430  nat44_session_update_counters (s, now, 0, thread_index);
431  /* Per-user LRU list maintenance */
432  nat44_session_update_lru (sm, s, thread_index);
433}
434
435static inline void
436create_bypass_for_fwd_worker (snat_main_t * sm, ip4_header_t * ip,
437			      u32 rx_fib_index)
438{
439  ip4_header_t ip_wkr = {
440    .src_address = ip->dst_address,
441  };
442  u32 thread_index = sm->worker_in2out_cb (&ip_wkr, rx_fib_index, 0);
443
444  create_bypass_for_fwd (sm, ip, rx_fib_index, thread_index);
445}
446
447#ifndef CLIB_MARCH_VARIANT
448u32
449icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
450		      u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
451		      u8 * p_proto, snat_session_key_t * p_value,
452		      u8 * p_dont_translate, void *d, void *e)
453{
454  u32 next = ~0, sw_if_index, rx_fib_index;
455  icmp46_header_t *icmp;
456  nat_ed_ses_key_t key;
457  clib_bihash_kv_16_8_t kv, value;
458  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
459  snat_session_t *s = 0;
460  u8 dont_translate = 0, is_addr_only, identity_nat;
461  snat_session_key_t e_key, l_key;
462
463  icmp = (icmp46_header_t *) ip4_next_header (ip);
464  sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
465  rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
466
467  if (get_icmp_o2i_ed_key (ip, &key))
468    {
469      b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
470      next = NAT44_ED_OUT2IN_NEXT_DROP;
471      goto out;
472    }
473  key.fib_index = rx_fib_index;
474  kv.key[0] = key.as_u64[0];
475  kv.key[1] = key.as_u64[1];
476
477  if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
478    {
479      /* Try to match static mapping */
480      e_key.addr = ip->dst_address;
481      e_key.port = key.l_port;
482      e_key.protocol = ip_proto_to_snat_proto (key.proto);
483      e_key.fib_index = rx_fib_index;
484      if (snat_static_mapping_match
485	  (sm, e_key, &l_key, 1, &is_addr_only, 0, 0, 0, &identity_nat))
486	{
487	  if (!sm->forwarding_enabled)
488	    {
489	      /* Don't NAT packet aimed at the intfc address */
490	      if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index,
491						    ip->dst_address.as_u32)))
492		{
493		  dont_translate = 1;
494		  goto out;
495		}
496	      b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
497	      next = NAT44_ED_OUT2IN_NEXT_DROP;
498	      goto out;
499	    }
500	  else
501	    {
502	      dont_translate = 1;
503	      if (next_src_nat (sm, ip, key.proto, key.l_port, key.r_port,
504				thread_index, rx_fib_index))
505		{
506		  next = NAT44_ED_OUT2IN_NEXT_IN2OUT;
507		  goto out;
508		}
509	      if (sm->num_workers > 1)
510		create_bypass_for_fwd_worker (sm, ip, rx_fib_index);
511	      else
512		create_bypass_for_fwd (sm, ip, rx_fib_index, thread_index);
513	      goto out;
514	    }
515	}
516
517      if (PREDICT_FALSE (icmp->type != ICMP4_echo_reply &&
518			 (icmp->type != ICMP4_echo_request || !is_addr_only)))
519	{
520	  b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
521	  next = NAT44_ED_OUT2IN_NEXT_DROP;
522	  goto out;
523	}
524
525      if (PREDICT_FALSE (identity_nat))
526	{
527	  dont_translate = 1;
528	  goto out;
529	}
530
531      /* Create session initiated by host from external network */
532      s = create_session_for_static_mapping_ed (sm, b, l_key, e_key, node,
533						thread_index, 0, 0,
534						vlib_time_now
535						(sm->vlib_main));
536
537      if (!s)
538	{
539	  next = NAT44_ED_OUT2IN_NEXT_DROP;
540	  goto out;
541	}
542    }
543  else
544    {
545      if (PREDICT_FALSE (icmp->type != ICMP4_echo_reply &&
546			 icmp->type != ICMP4_echo_request &&
547			 !icmp_is_error_message (icmp)))
548	{
549	  b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
550	  next = NAT44_ED_OUT2IN_NEXT_DROP;
551	  goto out;
552	}
553
554      s = pool_elt_at_index (tsm->sessions, value.value);
555    }
556
557  *p_proto = ip_proto_to_snat_proto (key.proto);
558out:
559  if (s)
560    *p_value = s->in2out;
561  *p_dont_translate = dont_translate;
562  if (d)
563    *(snat_session_t **) d = s;
564  return next;
565}
566#endif
567
568static snat_session_t *
569nat44_ed_out2in_unknown_proto (snat_main_t * sm,
570			       vlib_buffer_t * b,
571			       ip4_header_t * ip,
572			       u32 rx_fib_index,
573			       u32 thread_index,
574			       f64 now,
575			       vlib_main_t * vm, vlib_node_runtime_t * node)
576{
577  clib_bihash_kv_8_8_t kv, value;
578  clib_bihash_kv_16_8_t s_kv, s_value;
579  snat_static_mapping_t *m;
580  u32 old_addr, new_addr;
581  ip_csum_t sum;
582  snat_session_t *s;
583  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
584  snat_user_t *u;
585
586  old_addr = ip->dst_address.as_u32;
587
588  make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
589	      rx_fib_index, 0, 0);
590
591  if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
592    {
593      s = pool_elt_at_index (tsm->sessions, s_value.value);
594      new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
595    }
596  else
597    {
598      if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
599	{
600	  b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
601	  nat_elog_notice ("maximum sessions exceeded");
602	  return 0;
603	}
604
605      make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
606      if (clib_bihash_search_8_8
607	  (&sm->static_mapping_by_external, &kv, &value))
608	{
609	  b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
610	  return 0;
611	}
612
613      m = pool_elt_at_index (sm->static_mappings, value.value);
614
615      new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
616
617      u = nat_user_get_or_create (sm, &m->local_addr, m->fib_index,
618				  thread_index);
619      if (!u)
620	{
621	  nat_elog_warn ("create NAT user failed");
622	  return 0;
623	}
624
625      /* Create a new session */
626      s = nat_ed_session_alloc (sm, u, thread_index, now);
627      if (!s)
628	{
629	  nat44_delete_user_with_no_session (sm, u, thread_index);
630	  nat_elog_warn ("create NAT session failed");
631	  return 0;
632	}
633
634      s->ext_host_addr.as_u32 = ip->src_address.as_u32;
635      s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
636      s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
637      s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
638      s->out2in.addr.as_u32 = old_addr;
639      s->out2in.fib_index = rx_fib_index;
640      s->in2out.addr.as_u32 = new_addr;
641      s->in2out.fib_index = m->fib_index;
642      s->in2out.port = s->out2in.port = ip->protocol;
643      user_session_increment (sm, u, 1);
644
645      /* Add to lookup tables */
646      s_kv.value = s - tsm->sessions;
647      if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
648	nat_elog_notice ("out2in key add failed");
649
650      make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
651		  m->fib_index, 0, 0);
652      s_kv.value = s - tsm->sessions;
653      if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
654	nat_elog_notice ("in2out key add failed");
655    }
656
657  /* Update IP checksum */
658  sum = ip->checksum;
659  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
660  ip->checksum = ip_csum_fold (sum);
661
662  vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
663
664  /* Accounting */
665  nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
666				 thread_index);
667  /* Per-user LRU list maintenance */
668  nat44_session_update_lru (sm, s, thread_index);
669
670  return s;
671}
672
673static inline uword
674nat44_ed_out2in_node_fn_inline (vlib_main_t * vm,
675				vlib_node_runtime_t * node,
676				vlib_frame_t * frame, int is_slow_path)
677{
678  u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
679  nat44_ed_out2in_next_t next_index;
680  snat_main_t *sm = &snat_main;
681  f64 now = vlib_time_now (vm);
682  u32 thread_index = vm->thread_index;
683  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
684  u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
685    0, fragments = 0;
686
687  stats_node_index = is_slow_path ? sm->ed_out2in_slowpath_node_index :
688    sm->ed_out2in_node_index;
689
690  from = vlib_frame_vector_args (frame);
691  n_left_from = frame->n_vectors;
692  next_index = node->cached_next_index;
693
694  while (n_left_from > 0)
695    {
696      u32 n_left_to_next;
697
698      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
699
700      while (n_left_from >= 4 && n_left_to_next >= 2)
701	{
702	  u32 bi0, bi1;
703	  vlib_buffer_t *b0, *b1;
704	  u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
705	    new_addr0;
706	  u32 next1, sw_if_index1, rx_fib_index1, proto1, old_addr1,
707	    new_addr1;
708	  u16 old_port0, new_port0, old_port1, new_port1;
709	  ip4_header_t *ip0, *ip1;
710	  udp_header_t *udp0, *udp1;
711	  tcp_header_t *tcp0, *tcp1;
712	  icmp46_header_t *icmp0, *icmp1;
713	  snat_session_t *s0 = 0, *s1 = 0;
714	  clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
715	  ip_csum_t sum0, sum1;
716	  snat_session_key_t e_key0, l_key0, e_key1, l_key1;
717	  lb_nat_type_t lb_nat0, lb_nat1;
718	  twice_nat_type_t twice_nat0, twice_nat1;
719	  u8 identity_nat0, identity_nat1;
720
721	  /* Prefetch next iteration. */
722	  {
723	    vlib_buffer_t *p2, *p3;
724
725	    p2 = vlib_get_buffer (vm, from[2]);
726	    p3 = vlib_get_buffer (vm, from[3]);
727
728	    vlib_prefetch_buffer_header (p2, LOAD);
729	    vlib_prefetch_buffer_header (p3, LOAD);
730
731	    CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
732	    CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
733	  }
734
735	  /* speculatively enqueue b0 and b1 to the current next frame */
736	  to_next[0] = bi0 = from[0];
737	  to_next[1] = bi1 = from[1];
738	  from += 2;
739	  to_next += 2;
740	  n_left_from -= 2;
741	  n_left_to_next -= 2;
742
743	  b0 = vlib_get_buffer (vm, bi0);
744	  b1 = vlib_get_buffer (vm, bi1);
745
746	  next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
747	  vnet_buffer (b0)->snat.flags = 0;
748	  ip0 = vlib_buffer_get_current (b0);
749
750	  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
751	  rx_fib_index0 =
752	    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
753						 sw_if_index0);
754
755	  if (PREDICT_FALSE (ip0->ttl == 1))
756	    {
757	      vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
758	      icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
759					   ICMP4_time_exceeded_ttl_exceeded_in_transit,
760					   0);
761	      next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
762	      goto trace00;
763	    }
764
765	  udp0 = ip4_next_header (ip0);
766	  tcp0 = (tcp_header_t *) udp0;
767	  icmp0 = (icmp46_header_t *) udp0;
768	  proto0 = ip_proto_to_snat_proto (ip0->protocol);
769
770	  if (is_slow_path)
771	    {
772	      if (PREDICT_FALSE (proto0 == ~0))
773		{
774		  s0 =
775		    nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
776						   thread_index, now, vm,
777						   node);
778		  other_packets++;
779		  if (!sm->forwarding_enabled)
780		    {
781		      if (!s0)
782			next0 = NAT44_ED_OUT2IN_NEXT_DROP;
783		      goto trace00;
784		    }
785		}
786
787	      if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
788		{
789		  next0 = icmp_out2in_ed_slow_path
790		    (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
791		     next0, now, thread_index, &s0);
792		  icmp_packets++;
793		  goto trace00;
794		}
795	    }
796	  else
797	    {
798	      if (PREDICT_FALSE (proto0 == ~0))
799		{
800		  next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
801		  goto trace00;
802		}
803
804	      if (ip4_is_fragment (ip0))
805		{
806		  next0 = NAT44_ED_OUT2IN_NEXT_REASS;
807		  fragments++;
808		  goto trace00;
809		}
810
811	      if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
812		{
813		  next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
814		  goto trace00;
815		}
816	    }
817
818	  make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
819		      ip0->protocol, rx_fib_index0, udp0->dst_port,
820		      udp0->src_port);
821
822	  if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
823	    {
824	      if (is_slow_path)
825		{
826		  /* Try to match static mapping by external address and port,
827		     destination address and port in packet */
828		  e_key0.addr = ip0->dst_address;
829		  e_key0.port = udp0->dst_port;
830		  e_key0.protocol = proto0;
831		  e_key0.fib_index = rx_fib_index0;
832		  if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
833						 &twice_nat0, &lb_nat0,
834						 &ip0->src_address,
835						 &identity_nat0))
836		    {
837		      /*
838		       * Send DHCP packets to the ipv4 stack, or we won't
839		       * be able to use dhcp client on the outside interface
840		       */
841		      if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
842					 && (udp0->dst_port ==
843					     clib_host_to_net_u16
844					     (UDP_DST_PORT_dhcp_to_client))))
845			{
846			  vnet_feature_next (&next0, b0);
847			  goto trace00;
848			}
849
850		      if (!sm->forwarding_enabled)
851			{
852			  b0->error =
853			    node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
854			  next0 = NAT44_ED_OUT2IN_NEXT_DROP;
855			}
856		      else
857			{
858			  if (next_src_nat (sm, ip0, ip0->protocol,
859					    udp0->src_port, udp0->dst_port,
860					    thread_index, rx_fib_index0))
861			    {
862			      next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
863			      goto trace00;
864			    }
865			  if (sm->num_workers > 1)
866			    create_bypass_for_fwd_worker (sm, ip0,
867							  rx_fib_index0);
868			  else
869			    create_bypass_for_fwd (sm, ip0, rx_fib_index0,
870						   thread_index);
871			}
872		      goto trace00;
873		    }
874
875		  if (PREDICT_FALSE (identity_nat0))
876		    goto trace00;
877
878		  if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
879		    {
880		      b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
881		      next0 = NAT44_ED_OUT2IN_NEXT_DROP;
882		      goto trace00;
883		    }
884
885		  /* Create session initiated by host from external network */
886		  s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
887							     e_key0, node,
888							     thread_index,
889							     twice_nat0,
890							     lb_nat0, now);
891
892		  if (!s0)
893		    {
894		      next0 = NAT44_ED_OUT2IN_NEXT_DROP;
895		      goto trace00;
896		    }
897		}
898	      else
899		{
900		  next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
901		  goto trace00;
902		}
903	    }
904	  else
905	    {
906	      s0 = pool_elt_at_index (tsm->sessions, value0.value);
907	    }
908
909	  old_addr0 = ip0->dst_address.as_u32;
910	  new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
911	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
912
913	  sum0 = ip0->checksum;
914	  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
915				 dst_address);
916	  if (PREDICT_FALSE (is_twice_nat_session (s0)))
917	    sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
918				   s0->ext_host_nat_addr.as_u32, ip4_header_t,
919				   src_address);
920	  ip0->checksum = ip_csum_fold (sum0);
921
922	  if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
923	    {
924	      old_port0 = tcp0->dst_port;
925	      new_port0 = tcp0->dst_port = s0->in2out.port;
926
927	      sum0 = tcp0->checksum;
928	      sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
929				     dst_address);
930	      sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
931				     length);
932	      if (is_twice_nat_session (s0))
933		{
934		  sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
935					 s0->ext_host_nat_addr.as_u32,
936					 ip4_header_t, dst_address);
937		  sum0 = ip_csum_update (sum0, tcp0->src_port,
938					 s0->ext_host_nat_port, ip4_header_t,
939					 length);
940		  tcp0->src_port = s0->ext_host_nat_port;
941		  ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
942		}
943	      tcp0->checksum = ip_csum_fold (sum0);
944	      tcp_packets++;
945	      if (nat44_set_tcp_session_state_o2i
946		  (sm, s0, tcp0, thread_index))
947		goto trace00;
948	    }
949	  else
950	    {
951	      udp0->dst_port = s0->in2out.port;
952	      if (is_twice_nat_session (s0))
953		{
954		  udp0->src_port = s0->ext_host_nat_port;
955		  ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
956		}
957	      udp0->checksum = 0;
958	      udp_packets++;
959	    }
960
961	  /* Accounting */
962	  nat44_session_update_counters (s0, now,
963					 vlib_buffer_length_in_chain (vm, b0),
964					 thread_index);
965	  /* Per-user LRU list maintenance */
966	  nat44_session_update_lru (sm, s0, thread_index);
967
968	trace00:
969	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
970			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
971	    {
972	      nat44_ed_out2in_trace_t *t =
973		vlib_add_trace (vm, node, b0, sizeof (*t));
974	      t->is_slow_path = is_slow_path;
975	      t->sw_if_index = sw_if_index0;
976	      t->next_index = next0;
977	      t->session_index = ~0;
978	      if (s0)
979		t->session_index = s0 - tsm->sessions;
980	    }
981
982	  pkts_processed += next0 == NAT44_ED_OUT2IN_NEXT_LOOKUP;
983
984	  next1 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
985	  vnet_buffer (b1)->snat.flags = 0;
986	  ip1 = vlib_buffer_get_current (b1);
987
988	  sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
989	  rx_fib_index1 =
990	    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
991						 sw_if_index1);
992
993	  if (PREDICT_FALSE (ip1->ttl == 1))
994	    {
995	      vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
996	      icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
997					   ICMP4_time_exceeded_ttl_exceeded_in_transit,
998					   0);
999	      next1 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
1000	      goto trace01;
1001	    }
1002
1003	  udp1 = ip4_next_header (ip1);
1004	  tcp1 = (tcp_header_t *) udp1;
1005	  icmp1 = (icmp46_header_t *) udp1;
1006	  proto1 = ip_proto_to_snat_proto (ip1->protocol);
1007
1008	  if (is_slow_path)
1009	    {
1010	      if (PREDICT_FALSE (proto1 == ~0))
1011		{
1012		  s1 =
1013		    nat44_ed_out2in_unknown_proto (sm, b1, ip1, rx_fib_index1,
1014						   thread_index, now, vm,
1015						   node);
1016		  other_packets++;
1017		  if (!sm->forwarding_enabled)
1018		    {
1019		      if (!s1)
1020			next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1021		      goto trace01;
1022		    }
1023		}
1024
1025	      if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1026		{
1027		  next1 = icmp_out2in_ed_slow_path
1028		    (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1029		     next1, now, thread_index, &s1);
1030		  icmp_packets++;
1031		  goto trace01;
1032		}
1033	    }
1034	  else
1035	    {
1036	      if (PREDICT_FALSE (proto1 == ~0))
1037		{
1038		  next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1039		  goto trace01;
1040		}
1041
1042	      if (ip4_is_fragment (ip1))
1043		{
1044		  next1 = NAT44_ED_OUT2IN_NEXT_REASS;
1045		  fragments++;
1046		  goto trace01;
1047		}
1048
1049	      if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1050		{
1051		  next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1052		  goto trace01;
1053		}
1054	    }
1055
1056	  make_ed_kv (&kv1, &ip1->dst_address, &ip1->src_address,
1057		      ip1->protocol, rx_fib_index1, udp1->dst_port,
1058		      udp1->src_port);
1059
1060	  if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv1, &value1))
1061	    {
1062	      if (is_slow_path)
1063		{
1064		  /* Try to match static mapping by external address and port,
1065		     destination address and port in packet */
1066		  e_key1.addr = ip1->dst_address;
1067		  e_key1.port = udp1->dst_port;
1068		  e_key1.protocol = proto1;
1069		  e_key1.fib_index = rx_fib_index1;
1070		  if (snat_static_mapping_match (sm, e_key1, &l_key1, 1, 0,
1071						 &twice_nat1, &lb_nat1,
1072						 &ip1->src_address,
1073						 &identity_nat1))
1074		    {
1075		      /*
1076		       * Send DHCP packets to the ipv4 stack, or we won't
1077		       * be able to use dhcp client on the outside interface
1078		       */
1079		      if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1080					 && (udp1->dst_port ==
1081					     clib_host_to_net_u16
1082					     (UDP_DST_PORT_dhcp_to_client))))
1083			{
1084			  vnet_feature_next (&next1, b1);
1085			  goto trace01;
1086			}
1087
1088		      if (!sm->forwarding_enabled)
1089			{
1090			  b1->error =
1091			    node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1092			  next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1093			}
1094		      else
1095			{
1096			  if (next_src_nat (sm, ip1, ip1->protocol,
1097					    udp1->src_port, udp1->dst_port,
1098					    thread_index, rx_fib_index1))
1099			    {
1100			      next1 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1101			      goto trace01;
1102			    }
1103			  if (sm->num_workers > 1)
1104			    create_bypass_for_fwd_worker (sm, ip1,
1105							  rx_fib_index1);
1106			  else
1107			    create_bypass_for_fwd (sm, ip1, rx_fib_index1,
1108						   thread_index);
1109			}
1110		      goto trace01;
1111		    }
1112
1113		  if (PREDICT_FALSE (identity_nat1))
1114		    goto trace01;
1115
1116		  if ((proto1 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp1))
1117		    {
1118		      b1->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1119		      next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1120		      goto trace01;
1121		    }
1122
1123		  /* Create session initiated by host from external network */
1124		  s1 = create_session_for_static_mapping_ed (sm, b1, l_key1,
1125							     e_key1, node,
1126							     thread_index,
1127							     twice_nat1,
1128							     lb_nat1, now);
1129
1130		  if (!s1)
1131		    {
1132		      next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1133		      goto trace01;
1134		    }
1135		}
1136	      else
1137		{
1138		  next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1139		  goto trace01;
1140		}
1141	    }
1142	  else
1143	    {
1144	      s1 = pool_elt_at_index (tsm->sessions, value1.value);
1145	    }
1146
1147	  old_addr1 = ip1->dst_address.as_u32;
1148	  new_addr1 = ip1->dst_address.as_u32 = s1->in2out.addr.as_u32;
1149	  vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1150
1151	  sum1 = ip1->checksum;
1152	  sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1153				 dst_address);
1154	  if (PREDICT_FALSE (is_twice_nat_session (s1)))
1155	    sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
1156				   s1->ext_host_nat_addr.as_u32, ip4_header_t,
1157				   src_address);
1158	  ip1->checksum = ip_csum_fold (sum1);
1159
1160	  if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1161	    {
1162	      old_port1 = tcp1->dst_port;
1163	      new_port1 = tcp1->dst_port = s1->in2out.port;
1164
1165	      sum1 = tcp1->checksum;
1166	      sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1167				     dst_address);
1168	      sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1169				     length);
1170	      if (is_twice_nat_session (s1))
1171		{
1172		  sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
1173					 s1->ext_host_nat_addr.as_u32,
1174					 ip4_header_t, dst_address);
1175		  sum1 = ip_csum_update (sum1, tcp1->src_port,
1176					 s1->ext_host_nat_port, ip4_header_t,
1177					 length);
1178		  tcp1->src_port = s1->ext_host_nat_port;
1179		  ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
1180		}
1181	      tcp1->checksum = ip_csum_fold (sum1);
1182	      tcp_packets++;
1183	      if (nat44_set_tcp_session_state_o2i
1184		  (sm, s1, tcp1, thread_index))
1185		goto trace01;
1186	    }
1187	  else
1188	    {
1189	      udp1->dst_port = s1->in2out.port;
1190	      if (is_twice_nat_session (s1))
1191		{
1192		  udp1->src_port = s1->ext_host_nat_port;
1193		  ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
1194		}
1195	      udp1->checksum = 0;
1196	      udp_packets++;
1197	    }
1198
1199	  /* Accounting */
1200	  nat44_session_update_counters (s1, now,
1201					 vlib_buffer_length_in_chain (vm, b1),
1202					 thread_index);
1203	  /* Per-user LRU list maintenance */
1204	  nat44_session_update_lru (sm, s1, thread_index);
1205
1206	trace01:
1207	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1208			     && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1209	    {
1210	      nat44_ed_out2in_trace_t *t =
1211		vlib_add_trace (vm, node, b1, sizeof (*t));
1212	      t->is_slow_path = is_slow_path;
1213	      t->sw_if_index = sw_if_index1;
1214	      t->next_index = next1;
1215	      t->session_index = ~0;
1216	      if (s1)
1217		t->session_index = s1 - tsm->sessions;
1218	    }
1219
1220	  pkts_processed += next1 == NAT44_ED_OUT2IN_NEXT_LOOKUP;
1221
1222	  /* verify speculative enqueues, maybe switch current next frame */
1223	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1224					   to_next, n_left_to_next,
1225					   bi0, bi1, next0, next1);
1226	}
1227
1228      while (n_left_from > 0 && n_left_to_next > 0)
1229	{
1230	  u32 bi0;
1231	  vlib_buffer_t *b0;
1232	  u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
1233	    new_addr0;
1234	  u16 old_port0, new_port0;
1235	  ip4_header_t *ip0;
1236	  udp_header_t *udp0;
1237	  tcp_header_t *tcp0;
1238	  icmp46_header_t *icmp0;
1239	  snat_session_t *s0 = 0;
1240	  clib_bihash_kv_16_8_t kv0, value0;
1241	  ip_csum_t sum0;
1242	  snat_session_key_t e_key0, l_key0;
1243	  lb_nat_type_t lb_nat0;
1244	  twice_nat_type_t twice_nat0;
1245	  u8 identity_nat0;
1246
1247	  /* speculatively enqueue b0 to the current next frame */
1248	  bi0 = from[0];
1249	  to_next[0] = bi0;
1250	  from += 1;
1251	  to_next += 1;
1252	  n_left_from -= 1;
1253	  n_left_to_next -= 1;
1254
1255	  b0 = vlib_get_buffer (vm, bi0);
1256	  next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
1257	  vnet_buffer (b0)->snat.flags = 0;
1258	  ip0 = vlib_buffer_get_current (b0);
1259
1260	  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1261	  rx_fib_index0 =
1262	    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1263						 sw_if_index0);
1264
1265	  if (PREDICT_FALSE (ip0->ttl == 1))
1266	    {
1267	      vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1268	      icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1269					   ICMP4_time_exceeded_ttl_exceeded_in_transit,
1270					   0);
1271	      next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
1272	      goto trace0;
1273	    }
1274
1275	  udp0 = ip4_next_header (ip0);
1276	  tcp0 = (tcp_header_t *) udp0;
1277	  icmp0 = (icmp46_header_t *) udp0;
1278	  proto0 = ip_proto_to_snat_proto (ip0->protocol);
1279
1280	  if (is_slow_path)
1281	    {
1282	      if (PREDICT_FALSE (proto0 == ~0))
1283		{
1284		  s0 =
1285		    nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
1286						   thread_index, now, vm,
1287						   node);
1288		  other_packets++;
1289		  if (!sm->forwarding_enabled)
1290		    {
1291		      if (!s0)
1292			next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1293		      goto trace0;
1294		    }
1295		}
1296
1297	      if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1298		{
1299		  next0 = icmp_out2in_ed_slow_path
1300		    (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1301		     next0, now, thread_index, &s0);
1302		  icmp_packets++;
1303		  goto trace0;
1304		}
1305	    }
1306	  else
1307	    {
1308	      if (PREDICT_FALSE (proto0 == ~0))
1309		{
1310		  next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1311		  goto trace0;
1312		}
1313
1314	      if (ip4_is_fragment (ip0))
1315		{
1316		  next0 = NAT44_ED_OUT2IN_NEXT_REASS;
1317		  fragments++;
1318		  goto trace0;
1319		}
1320
1321	      if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1322		{
1323		  next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1324		  goto trace0;
1325		}
1326	    }
1327
1328	  make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
1329		      ip0->protocol, rx_fib_index0, udp0->dst_port,
1330		      udp0->src_port);
1331
1332	  if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
1333	    {
1334	      if (is_slow_path)
1335		{
1336		  /* Try to match static mapping by external address and port,
1337		     destination address and port in packet */
1338		  e_key0.addr = ip0->dst_address;
1339		  e_key0.port = udp0->dst_port;
1340		  e_key0.protocol = proto0;
1341		  e_key0.fib_index = rx_fib_index0;
1342		  if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1343						 &twice_nat0, &lb_nat0,
1344						 &ip0->src_address,
1345						 &identity_nat0))
1346		    {
1347		      /*
1348		       * Send DHCP packets to the ipv4 stack, or we won't
1349		       * be able to use dhcp client on the outside interface
1350		       */
1351		      if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1352					 && (udp0->dst_port ==
1353					     clib_host_to_net_u16
1354					     (UDP_DST_PORT_dhcp_to_client))))
1355			{
1356			  vnet_feature_next (&next0, b0);
1357			  goto trace0;
1358			}
1359
1360		      if (!sm->forwarding_enabled)
1361			{
1362			  b0->error =
1363			    node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1364			  next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1365			}
1366		      else
1367			{
1368			  if (next_src_nat (sm, ip0, ip0->protocol,
1369					    udp0->src_port, udp0->dst_port,
1370					    thread_index, rx_fib_index0))
1371			    {
1372			      next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1373			      goto trace0;
1374			    }
1375			  if (sm->num_workers > 1)
1376			    create_bypass_for_fwd_worker (sm, ip0,
1377							  rx_fib_index0);
1378			  else
1379			    create_bypass_for_fwd (sm, ip0, rx_fib_index0,
1380						   thread_index);
1381			}
1382		      goto trace0;
1383		    }
1384
1385		  if (PREDICT_FALSE (identity_nat0))
1386		    goto trace0;
1387
1388		  if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
1389		    {
1390		      b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1391		      next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1392		      goto trace0;
1393		    }
1394
1395		  /* Create session initiated by host from external network */
1396		  s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1397							     e_key0, node,
1398							     thread_index,
1399							     twice_nat0,
1400							     lb_nat0, now);
1401
1402		  if (!s0)
1403		    {
1404		      next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1405		      goto trace0;
1406		    }
1407		}
1408	      else
1409		{
1410		  next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1411		  goto trace0;
1412		}
1413	    }
1414	  else
1415	    {
1416	      s0 = pool_elt_at_index (tsm->sessions, value0.value);
1417	    }
1418
1419	  old_addr0 = ip0->dst_address.as_u32;
1420	  new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
1421	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1422
1423	  sum0 = ip0->checksum;
1424	  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1425				 dst_address);
1426	  if (PREDICT_FALSE (is_twice_nat_session (s0)))
1427	    sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1428				   s0->ext_host_nat_addr.as_u32, ip4_header_t,
1429				   src_address);
1430	  ip0->checksum = ip_csum_fold (sum0);
1431
1432	  if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1433	    {
1434	      old_port0 = tcp0->dst_port;
1435	      new_port0 = tcp0->dst_port = s0->in2out.port;
1436
1437	      sum0 = tcp0->checksum;
1438	      sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1439				     dst_address);
1440	      sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1441				     length);
1442	      if (is_twice_nat_session (s0))
1443		{
1444		  sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1445					 s0->ext_host_nat_addr.as_u32,
1446					 ip4_header_t, dst_address);
1447		  sum0 = ip_csum_update (sum0, tcp0->src_port,
1448					 s0->ext_host_nat_port, ip4_header_t,
1449					 length);
1450		  tcp0->src_port = s0->ext_host_nat_port;
1451		  ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1452		}
1453	      tcp0->checksum = ip_csum_fold (sum0);
1454	      tcp_packets++;
1455	      if (nat44_set_tcp_session_state_o2i
1456		  (sm, s0, tcp0, thread_index))
1457		goto trace0;
1458	    }
1459	  else
1460	    {
1461	      udp0->dst_port = s0->in2out.port;
1462	      if (is_twice_nat_session (s0))
1463		{
1464		  udp0->src_port = s0->ext_host_nat_port;
1465		  ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1466		}
1467	      udp0->checksum = 0;
1468	      udp_packets++;
1469	    }
1470
1471	  /* Accounting */
1472	  nat44_session_update_counters (s0, now,
1473					 vlib_buffer_length_in_chain (vm, b0),
1474					 thread_index);
1475	  /* Per-user LRU list maintenance */
1476	  nat44_session_update_lru (sm, s0, thread_index);
1477
1478	trace0:
1479	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1480			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1481	    {
1482	      nat44_ed_out2in_trace_t *t =
1483		vlib_add_trace (vm, node, b0, sizeof (*t));
1484	      t->is_slow_path = is_slow_path;
1485	      t->sw_if_index = sw_if_index0;
1486	      t->next_index = next0;
1487	      t->session_index = ~0;
1488	      if (s0)
1489		t->session_index = s0 - tsm->sessions;
1490	    }
1491
1492	  pkts_processed += next0 == NAT44_ED_OUT2IN_NEXT_LOOKUP;
1493	  /* verify speculative enqueue, maybe switch current next frame */
1494	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1495					   to_next, n_left_to_next,
1496					   bi0, next0);
1497	}
1498
1499      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1500    }
1501
1502  vlib_node_increment_counter (vm, stats_node_index,
1503			       NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1504			       pkts_processed);
1505  vlib_node_increment_counter (vm, stats_node_index,
1506			       NAT_OUT2IN_ED_ERROR_TCP_PACKETS, tcp_packets);
1507  vlib_node_increment_counter (vm, stats_node_index,
1508			       NAT_OUT2IN_ED_ERROR_UDP_PACKETS, udp_packets);
1509  vlib_node_increment_counter (vm, stats_node_index,
1510			       NAT_OUT2IN_ED_ERROR_ICMP_PACKETS,
1511			       icmp_packets);
1512  vlib_node_increment_counter (vm, stats_node_index,
1513			       NAT_OUT2IN_ED_ERROR_OTHER_PACKETS,
1514			       other_packets);
1515  vlib_node_increment_counter (vm, stats_node_index,
1516			       NAT_OUT2IN_ED_ERROR_FRAGMENTS, fragments);
1517  return frame->n_vectors;
1518}
1519
1520VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
1521				     vlib_node_runtime_t * node,
1522				     vlib_frame_t * frame)
1523{
1524  return nat44_ed_out2in_node_fn_inline (vm, node, frame, 0);
1525}
1526
1527/* *INDENT-OFF* */
1528VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1529  .name = "nat44-ed-out2in",
1530  .vector_size = sizeof (u32),
1531  .format_trace = format_nat44_ed_out2in_trace,
1532  .type = VLIB_NODE_TYPE_INTERNAL,
1533  .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1534  .error_strings = nat_out2in_ed_error_strings,
1535  .runtime_data_bytes = sizeof (snat_runtime_t),
1536  .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
1537  .next_nodes = {
1538    [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
1539    [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1540    [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
1541    [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1542    [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
1543    [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass",
1544  },
1545};
1546/* *INDENT-ON* */
1547
1548VLIB_NODE_FN (nat44_ed_out2in_slowpath_node) (vlib_main_t * vm,
1549					      vlib_node_runtime_t * node,
1550					      vlib_frame_t * frame)
1551{
1552  return nat44_ed_out2in_node_fn_inline (vm, node, frame, 1);
1553}
1554
1555/* *INDENT-OFF* */
1556VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1557  .name = "nat44-ed-out2in-slowpath",
1558  .vector_size = sizeof (u32),
1559  .format_trace = format_nat44_ed_out2in_trace,
1560  .type = VLIB_NODE_TYPE_INTERNAL,
1561  .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1562  .error_strings = nat_out2in_ed_error_strings,
1563  .runtime_data_bytes = sizeof (snat_runtime_t),
1564  .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
1565  .next_nodes = {
1566    [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
1567    [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1568    [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
1569    [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1570    [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
1571    [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass",
1572  },
1573};
1574/* *INDENT-ON* */
1575
1576VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm,
1577					   vlib_node_runtime_t * node,
1578					   vlib_frame_t * frame)
1579{
1580  u32 n_left_from, *from, *to_next;
1581  nat44_ed_out2in_next_t next_index;
1582  u32 pkts_processed = 0;
1583  snat_main_t *sm = &snat_main;
1584  f64 now = vlib_time_now (vm);
1585  u32 thread_index = vm->thread_index;
1586  snat_main_per_thread_data_t *per_thread_data =
1587    &sm->per_thread_data[thread_index];
1588  u32 *fragments_to_drop = 0;
1589  u32 *fragments_to_loopback = 0;
1590
1591  from = vlib_frame_vector_args (frame);
1592  n_left_from = frame->n_vectors;
1593  next_index = node->cached_next_index;
1594
1595  while (n_left_from > 0)
1596    {
1597      u32 n_left_to_next;
1598
1599      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1600
1601      while (n_left_from > 0 && n_left_to_next > 0)
1602	{
1603	  u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1604	  vlib_buffer_t *b0;
1605	  u32 next0;
1606	  u8 cached0 = 0;
1607	  ip4_header_t *ip0;
1608	  nat_reass_ip4_t *reass0;
1609	  udp_header_t *udp0;
1610	  tcp_header_t *tcp0;
1611	  icmp46_header_t *icmp0;
1612	  clib_bihash_kv_16_8_t kv0, value0;
1613	  snat_session_t *s0 = 0;
1614	  u16 old_port0, new_port0;
1615	  ip_csum_t sum0;
1616	  snat_session_key_t e_key0, l_key0;
1617	  lb_nat_type_t lb0;
1618	  twice_nat_type_t twice_nat0;
1619	  u8 identity_nat0;
1620
1621	  /* speculatively enqueue b0 to the current next frame */
1622	  bi0 = from[0];
1623	  to_next[0] = bi0;
1624	  from += 1;
1625	  to_next += 1;
1626	  n_left_from -= 1;
1627	  n_left_to_next -= 1;
1628
1629	  b0 = vlib_get_buffer (vm, bi0);
1630	  next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
1631
1632	  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1633	  rx_fib_index0 =
1634	    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1635						 sw_if_index0);
1636
1637	  if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1638	    {
1639	      next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1640	      b0->error = node->errors[NAT_OUT2IN_ED_ERROR_DROP_FRAGMENT];
1641	      goto trace0;
1642	    }
1643
1644	  ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1645	  udp0 = ip4_next_header (ip0);
1646	  tcp0 = (tcp_header_t *) udp0;
1647	  icmp0 = (icmp46_header_t *) udp0;
1648	  proto0 = ip_proto_to_snat_proto (ip0->protocol);
1649
1650	  reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1651						 ip0->dst_address,
1652						 ip0->fragment_id,
1653						 ip0->protocol,
1654						 1, &fragments_to_drop);
1655
1656	  if (PREDICT_FALSE (!reass0))
1657	    {
1658	      next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1659	      b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_REASS];
1660	      nat_elog_notice ("maximum reassemblies exceeded");
1661	      goto trace0;
1662	    }
1663
1664	  if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1665	    {
1666	      if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1667		{
1668		  next0 = icmp_out2in_ed_slow_path
1669		    (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1670		     next0, now, thread_index, &s0);
1671
1672		  if (PREDICT_TRUE (next0 != NAT44_ED_OUT2IN_NEXT_DROP))
1673		    {
1674		      if (s0)
1675			reass0->sess_index = s0 - per_thread_data->sessions;
1676		      else
1677			reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1678		      reass0->thread_index = thread_index;
1679		      nat_ip4_reass_get_frags (reass0,
1680					       &fragments_to_loopback);
1681		    }
1682
1683		  goto trace0;
1684		}
1685
1686	      make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
1687			  ip0->protocol, rx_fib_index0, udp0->dst_port,
1688			  udp0->src_port);
1689
1690	      if (clib_bihash_search_16_8
1691		  (&per_thread_data->out2in_ed, &kv0, &value0))
1692		{
1693		  /* Try to match static mapping by external address and port,
1694		     destination address and port in packet */
1695		  e_key0.addr = ip0->dst_address;
1696		  e_key0.port = udp0->dst_port;
1697		  e_key0.protocol = proto0;
1698		  e_key0.fib_index = rx_fib_index0;
1699		  if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1700						 &twice_nat0, &lb0, 0,
1701						 &identity_nat0))
1702		    {
1703		      /*
1704		       * Send DHCP packets to the ipv4 stack, or we won't
1705		       * be able to use dhcp client on the outside interface
1706		       */
1707		      if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1708					 && (udp0->dst_port
1709					     ==
1710					     clib_host_to_net_u16
1711					     (UDP_DST_PORT_dhcp_to_client))))
1712			{
1713			  vnet_feature_next (&next0, b0);
1714			  goto trace0;
1715			}
1716
1717		      if (!sm->forwarding_enabled)
1718			{
1719			  b0->error =
1720			    node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1721			  next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1722			}
1723		      else
1724			{
1725			  if (next_src_nat (sm, ip0, ip0->protocol,
1726					    udp0->src_port, udp0->dst_port,
1727					    thread_index, rx_fib_index0))
1728			    {
1729			      next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1730			      goto trace0;
1731			    }
1732			  if (sm->num_workers > 1)
1733			    create_bypass_for_fwd_worker (sm, ip0,
1734							  rx_fib_index0);
1735			  else
1736			    create_bypass_for_fwd (sm, ip0, rx_fib_index0,
1737						   thread_index);
1738			  reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1739			  nat_ip4_reass_get_frags (reass0,
1740						   &fragments_to_loopback);
1741			}
1742		      goto trace0;
1743		    }
1744
1745		  if (PREDICT_FALSE (identity_nat0))
1746		    {
1747		      reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1748		      goto trace0;
1749		    }
1750
1751		  if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
1752		    {
1753		      b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1754		      next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1755		      goto trace0;
1756		    }
1757
1758		  /* Create session initiated by host from external network */
1759		  s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1760							     e_key0, node,
1761							     thread_index,
1762							     twice_nat0, lb0,
1763							     now);
1764		  if (!s0)
1765		    {
1766		      b0->error =
1767			node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1768		      next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1769		      goto trace0;
1770		    }
1771		  reass0->sess_index = s0 - per_thread_data->sessions;
1772		  reass0->thread_index = thread_index;
1773		}
1774	      else
1775		{
1776		  s0 = pool_elt_at_index (per_thread_data->sessions,
1777					  value0.value);
1778		  reass0->sess_index = value0.value;
1779		}
1780	      nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1781	    }
1782	  else
1783	    {
1784	      if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
1785		goto trace0;
1786	      if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1787		{
1788		  if (nat_ip4_reass_add_fragment
1789		      (thread_index, reass0, bi0, &fragments_to_drop))
1790		    {
1791		      b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_FRAG];
1792		      nat_elog_notice
1793			("maximum fragments per reassembly exceeded");
1794		      next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1795		      goto trace0;
1796		    }
1797		  cached0 = 1;
1798		  goto trace0;
1799		}
1800	      s0 = pool_elt_at_index (per_thread_data->sessions,
1801				      reass0->sess_index);
1802	    }
1803
1804	  old_addr0 = ip0->dst_address.as_u32;
1805	  ip0->dst_address = s0->in2out.addr;
1806	  new_addr0 = ip0->dst_address.as_u32;
1807	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1808
1809	  sum0 = ip0->checksum;
1810	  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1811				 ip4_header_t,
1812				 dst_address /* changed member */ );
1813	  if (PREDICT_FALSE (is_twice_nat_session (s0)))
1814	    sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1815				   s0->ext_host_nat_addr.as_u32, ip4_header_t,
1816				   src_address);
1817	  ip0->checksum = ip_csum_fold (sum0);
1818
1819	  if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1820	    {
1821	      if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1822		{
1823		  old_port0 = tcp0->dst_port;
1824		  tcp0->dst_port = s0->in2out.port;
1825		  new_port0 = tcp0->dst_port;
1826
1827		  sum0 = tcp0->checksum;
1828		  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1829					 ip4_header_t,
1830					 dst_address /* changed member */ );
1831
1832		  sum0 = ip_csum_update (sum0, old_port0, new_port0,
1833					 ip4_header_t /* cheat */ ,
1834					 length /* changed member */ );
1835		  if (is_twice_nat_session (s0))
1836		    {
1837		      sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1838					     s0->ext_host_nat_addr.as_u32,
1839					     ip4_header_t, dst_address);
1840		      sum0 = ip_csum_update (sum0, tcp0->src_port,
1841					     s0->ext_host_nat_port,
1842					     ip4_header_t, length);
1843		      tcp0->src_port = s0->ext_host_nat_port;
1844		      ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1845		    }
1846		  tcp0->checksum = ip_csum_fold (sum0);
1847		}
1848	      else
1849		{
1850		  old_port0 = udp0->dst_port;
1851		  udp0->dst_port = s0->in2out.port;
1852		  if (is_twice_nat_session (s0))
1853		    {
1854		      udp0->src_port = s0->ext_host_nat_port;
1855		      ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1856		    }
1857		  udp0->checksum = 0;
1858		}
1859	    }
1860
1861	  /* Accounting */
1862	  nat44_session_update_counters (s0, now,
1863					 vlib_buffer_length_in_chain (vm, b0),
1864					 thread_index);
1865	  /* Per-user LRU list maintenance */
1866	  nat44_session_update_lru (sm, s0, thread_index);
1867
1868	trace0:
1869	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1870			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1871	    {
1872	      nat44_reass_trace_t *t =
1873		vlib_add_trace (vm, node, b0, sizeof (*t));
1874	      t->cached = cached0;
1875	      t->sw_if_index = sw_if_index0;
1876	      t->next_index = next0;
1877	    }
1878
1879	  if (cached0)
1880	    {
1881	      n_left_to_next++;
1882	      to_next--;
1883	    }
1884	  else
1885	    {
1886	      pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
1887
1888	      /* verify speculative enqueue, maybe switch current next frame */
1889	      vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1890					       to_next, n_left_to_next,
1891					       bi0, next0);
1892	    }
1893
1894	  if (n_left_from == 0 && vec_len (fragments_to_loopback))
1895	    {
1896	      from = vlib_frame_vector_args (frame);
1897	      u32 len = vec_len (fragments_to_loopback);
1898	      if (len <= VLIB_FRAME_SIZE)
1899		{
1900		  clib_memcpy_fast (from, fragments_to_loopback,
1901				    sizeof (u32) * len);
1902		  n_left_from = len;
1903		  vec_reset_length (fragments_to_loopback);
1904		}
1905	      else
1906		{
1907		  clib_memcpy_fast (from, fragments_to_loopback +
1908				    (len - VLIB_FRAME_SIZE),
1909				    sizeof (u32) * VLIB_FRAME_SIZE);
1910		  n_left_from = VLIB_FRAME_SIZE;
1911		  _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1912		}
1913	    }
1914	}
1915
1916      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1917    }
1918
1919  vlib_node_increment_counter (vm, sm->ed_out2in_reass_node_index,
1920			       NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1921			       pkts_processed);
1922
1923  nat_send_all_to_node (vm, fragments_to_drop, node,
1924			&node->errors[NAT_OUT2IN_ED_ERROR_DROP_FRAGMENT],
1925			NAT44_ED_OUT2IN_NEXT_DROP);
1926
1927  vec_free (fragments_to_drop);
1928  vec_free (fragments_to_loopback);
1929  return frame->n_vectors;
1930}
1931
1932/* *INDENT-OFF* */
1933VLIB_REGISTER_NODE (nat44_ed_out2in_reass_node) = {
1934  .name = "nat44-ed-out2in-reass",
1935  .vector_size = sizeof (u32),
1936  .format_trace = format_nat44_reass_trace,
1937  .type = VLIB_NODE_TYPE_INTERNAL,
1938  .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1939  .error_strings = nat_out2in_ed_error_strings,
1940  .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
1941  .next_nodes = {
1942    [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
1943    [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1944    [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
1945    [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1946    [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
1947    [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass",
1948  },
1949};
1950/* *INDENT-ON* */
1951
1952/*
1953 * fd.io coding-style-patch-verification: ON
1954 *
1955 * Local Variables:
1956 * eval: (c-set-style "gnu")
1957 * End:
1958 */
1959