nat44_hairpinning.c revision f126e746
1/*
2 * Copyright (c) 2018 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15/**
16 * @file
17 * @brief NAT44 hairpinning
18 */
19
20#include <vlib/vlib.h>
21#include <vnet/vnet.h>
22#include <vnet/fib/ip4_fib.h>
23#include <nat/nat.h>
24#include <nat/nat_inlines.h>
25
26typedef enum
27{
28  SNAT_HAIRPIN_SRC_NEXT_DROP,
29  SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
30  SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
31  SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
32  SNAT_HAIRPIN_SRC_N_NEXT,
33} snat_hairpin_src_next_t;
34
35typedef enum
36{
37  NAT_HAIRPIN_NEXT_LOOKUP,
38  NAT_HAIRPIN_NEXT_DROP,
39  NAT_HAIRPIN_N_NEXT,
40} nat_hairpin_next_t;
41
42#define foreach_nat44_hairpin_error                       \
43_(PROCESSED, "NAT44 hairpinning packets processed")
44
45typedef enum
46{
47#define _(sym,str) NAT44_HAIRPIN_ERROR_##sym,
48  foreach_nat44_hairpin_error
49#undef _
50    NAT44_HAIRPIN_N_ERROR,
51} nat44_hairpin_error_t;
52
53static char *nat44_hairpin_error_strings[] = {
54#define _(sym,string) string,
55  foreach_nat44_hairpin_error
56#undef _
57};
58
59extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
60
61static_always_inline int
62is_hairpinning (snat_main_t * sm, ip4_address_t * dst_addr)
63{
64  snat_address_t *ap;
65  clib_bihash_kv_8_8_t kv, value;
66  snat_session_key_t m_key;
67
68  /* *INDENT-OFF* */
69  vec_foreach (ap, sm->addresses)
70    {
71      if (ap->addr.as_u32 == dst_addr->as_u32)
72        return 1;
73    }
74  /* *INDENT-ON* */
75
76  m_key.addr.as_u32 = dst_addr->as_u32;
77  m_key.fib_index = 0;
78  m_key.port = 0;
79  m_key.protocol = 0;
80  kv.key = m_key.as_u64;
81  if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
82    return 1;
83
84  return 0;
85}
86
87#ifndef CLIB_MARCH_VARIANT
88int
89snat_hairpinning (snat_main_t * sm,
90		  vlib_buffer_t * b0,
91		  ip4_header_t * ip0,
92		  udp_header_t * udp0,
93		  tcp_header_t * tcp0, u32 proto0, int is_ed)
94{
95  snat_session_key_t key0, sm0;
96  snat_session_t *s0;
97  clib_bihash_kv_8_8_t kv0, value0;
98  ip_csum_t sum0;
99  u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
100  u16 new_dst_port0, old_dst_port0;
101  int rv;
102
103  key0.addr = ip0->dst_address;
104  key0.port = udp0->dst_port;
105  key0.protocol = proto0;
106  key0.fib_index = sm->outside_fib_index;
107  kv0.key = key0.as_u64;
108
109  /* Check if destination is static mappings */
110  if (!snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0, 0))
111    {
112      new_dst_addr0 = sm0.addr.as_u32;
113      new_dst_port0 = sm0.port;
114      vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
115    }
116  /* or active session */
117  else
118    {
119      if (sm->num_workers > 1)
120	ti =
121	  (clib_net_to_host_u16 (udp0->dst_port) -
122	   1024) / sm->port_per_thread;
123      else
124	ti = sm->num_workers;
125
126      if (is_ed)
127	{
128	  clib_bihash_kv_16_8_t ed_kv, ed_value;
129	  make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
130		      ip0->protocol, sm->outside_fib_index, udp0->dst_port,
131		      udp0->src_port);
132	  rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
133					&ed_kv, &ed_value);
134	  si = ed_value.value;
135	}
136      else
137	{
138	  rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
139				       &value0);
140	  si = value0.value;
141	}
142      if (rv)
143	return 0;
144
145      s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
146      new_dst_addr0 = s0->in2out.addr.as_u32;
147      new_dst_port0 = s0->in2out.port;
148      vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
149    }
150
151  /* Destination is behind the same NAT, use internal address and port */
152  if (new_dst_addr0)
153    {
154      old_dst_addr0 = ip0->dst_address.as_u32;
155      ip0->dst_address.as_u32 = new_dst_addr0;
156      sum0 = ip0->checksum;
157      sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
158			     ip4_header_t, dst_address);
159      ip0->checksum = ip_csum_fold (sum0);
160
161      old_dst_port0 = tcp0->dst;
162      if (PREDICT_TRUE (new_dst_port0 != old_dst_port0))
163	{
164	  if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
165	    {
166	      tcp0->dst = new_dst_port0;
167	      sum0 = tcp0->checksum;
168	      sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
169				     ip4_header_t, dst_address);
170	      sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
171				     ip4_header_t /* cheat */ , length);
172	      tcp0->checksum = ip_csum_fold (sum0);
173	    }
174	  else
175	    {
176	      udp0->dst_port = new_dst_port0;
177	      udp0->checksum = 0;
178	    }
179	}
180      else
181	{
182	  if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
183	    {
184	      sum0 = tcp0->checksum;
185	      sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
186				     ip4_header_t, dst_address);
187	      tcp0->checksum = ip_csum_fold (sum0);
188	    }
189	}
190      return 1;
191    }
192  return 0;
193}
194#endif
195
196#ifndef CLIB_MARCH_VARIANT
197u32
198snat_icmp_hairpinning (snat_main_t * sm,
199		       vlib_buffer_t * b0,
200		       ip4_header_t * ip0, icmp46_header_t * icmp0, int is_ed)
201{
202  snat_session_key_t key0;
203  clib_bihash_kv_8_8_t kv0, value0;
204  u32 old_dst_addr0, new_dst_addr0;
205  u32 old_addr0, new_addr0;
206  u16 old_port0, new_port0;
207  u16 old_checksum0, new_checksum0;
208  u32 si, ti = 0;
209  ip_csum_t sum0;
210  snat_session_t *s0;
211  snat_static_mapping_t *m0;
212
213  if (icmp_type_is_error_message
214      (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
215    {
216      ip4_header_t *inner_ip0 = 0;
217      tcp_udp_header_t *l4_header = 0;
218
219      inner_ip0 = (ip4_header_t *) ((icmp_echo_header_t *) (icmp0 + 1) + 1);
220      l4_header = ip4_next_header (inner_ip0);
221      u32 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
222
223      if (protocol != SNAT_PROTOCOL_TCP && protocol != SNAT_PROTOCOL_UDP)
224	return 1;
225
226      if (is_ed)
227	{
228	  clib_bihash_kv_16_8_t ed_kv, ed_value;
229	  make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
230		      inner_ip0->protocol, sm->outside_fib_index,
231		      l4_header->src_port, l4_header->dst_port);
232	  if (clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
233				       &ed_kv, &ed_value))
234	    return 1;
235	  si = ed_value.value;
236	}
237      else
238	{
239	  key0.addr = ip0->dst_address;
240	  key0.port = l4_header->src_port;
241	  key0.protocol = protocol;
242	  key0.fib_index = sm->outside_fib_index;
243	  kv0.key = key0.as_u64;
244	  if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
245				      &value0))
246	    return 1;
247	  si = value0.value;
248	}
249      s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
250      new_dst_addr0 = s0->in2out.addr.as_u32;
251      vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
252
253      /* update inner source IP address */
254      old_addr0 = inner_ip0->src_address.as_u32;
255      inner_ip0->src_address.as_u32 = new_dst_addr0;
256      new_addr0 = inner_ip0->src_address.as_u32;
257      sum0 = icmp0->checksum;
258      sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
259			     src_address);
260      icmp0->checksum = ip_csum_fold (sum0);
261
262      /* update inner IP header checksum */
263      old_checksum0 = inner_ip0->checksum;
264      sum0 = inner_ip0->checksum;
265      sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
266			     src_address);
267      inner_ip0->checksum = ip_csum_fold (sum0);
268      new_checksum0 = inner_ip0->checksum;
269      sum0 = icmp0->checksum;
270      sum0 = ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
271			     checksum);
272      icmp0->checksum = ip_csum_fold (sum0);
273
274      /* update inner source port */
275      old_port0 = l4_header->src_port;
276      l4_header->src_port = s0->in2out.port;
277      new_port0 = l4_header->src_port;
278      sum0 = icmp0->checksum;
279      sum0 = ip_csum_update (sum0, old_port0, new_port0, tcp_udp_header_t,
280			     src_port);
281      icmp0->checksum = ip_csum_fold (sum0);
282    }
283  else
284    {
285      key0.addr = ip0->dst_address;
286      key0.port = 0;
287      key0.protocol = 0;
288      key0.fib_index = sm->outside_fib_index;
289      kv0.key = key0.as_u64;
290
291      if (clib_bihash_search_8_8
292	  (&sm->static_mapping_by_external, &kv0, &value0))
293	{
294	  if (!is_ed)
295	    {
296	      icmp_echo_header_t *echo0 = (icmp_echo_header_t *) (icmp0 + 1);
297	      u16 icmp_id0 = echo0->identifier;
298	      key0.addr = ip0->dst_address;
299	      key0.port = icmp_id0;
300	      key0.protocol = SNAT_PROTOCOL_ICMP;
301	      key0.fib_index = sm->outside_fib_index;
302	      kv0.key = key0.as_u64;
303	      if (sm->num_workers > 1)
304		ti =
305		  (clib_net_to_host_u16 (icmp_id0) -
306		   1024) / sm->port_per_thread;
307	      else
308		ti = sm->num_workers;
309	      int rv =
310		clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
311					&value0);
312	      if (!rv)
313		{
314		  si = value0.value;
315		  s0 =
316		    pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
317		  new_dst_addr0 = s0->in2out.addr.as_u32;
318		  vnet_buffer (b0)->sw_if_index[VLIB_TX] =
319		    s0->in2out.fib_index;
320		  echo0->identifier = s0->in2out.port;
321		  sum0 = icmp0->checksum;
322		  sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
323					 icmp_echo_header_t, identifier);
324		  icmp0->checksum = ip_csum_fold (sum0);
325		  goto change_addr;
326		}
327	    }
328
329	  return 1;
330	}
331
332      m0 = pool_elt_at_index (sm->static_mappings, value0.value);
333
334      new_dst_addr0 = m0->local_addr.as_u32;
335      if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
336	vnet_buffer (b0)->sw_if_index[VLIB_TX] = m0->fib_index;
337    }
338change_addr:
339  /* Destination is behind the same NAT, use internal address and port */
340  if (new_dst_addr0)
341    {
342      old_dst_addr0 = ip0->dst_address.as_u32;
343      ip0->dst_address.as_u32 = new_dst_addr0;
344      sum0 = ip0->checksum;
345      sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
346			     ip4_header_t, dst_address);
347      ip0->checksum = ip_csum_fold (sum0);
348    }
349  return 0;
350}
351#endif
352
353#ifndef CLIB_MARCH_VARIANT
354void
355nat_hairpinning_sm_unknown_proto (snat_main_t * sm,
356				  vlib_buffer_t * b, ip4_header_t * ip)
357{
358  clib_bihash_kv_8_8_t kv, value;
359  snat_static_mapping_t *m;
360  u32 old_addr, new_addr;
361  ip_csum_t sum;
362
363  make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
364  if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
365    return;
366
367  m = pool_elt_at_index (sm->static_mappings, value.value);
368
369  old_addr = ip->dst_address.as_u32;
370  new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
371  sum = ip->checksum;
372  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
373  ip->checksum = ip_csum_fold (sum);
374
375  if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
376    vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
377}
378#endif
379
380#ifndef CLIB_MARCH_VARIANT
381void
382nat44_ed_hairpinning_unknown_proto (snat_main_t * sm,
383				    vlib_buffer_t * b, ip4_header_t * ip)
384{
385  u32 old_addr, new_addr = 0, ti = 0;
386  clib_bihash_kv_8_8_t kv, value;
387  clib_bihash_kv_16_8_t s_kv, s_value;
388  snat_static_mapping_t *m;
389  ip_csum_t sum;
390  snat_session_t *s;
391  snat_main_per_thread_data_t *tsm;
392
393  if (sm->num_workers > 1)
394    ti = sm->worker_out2in_cb (b, ip, sm->outside_fib_index, 0);
395  else
396    ti = sm->num_workers;
397  tsm = &sm->per_thread_data[ti];
398
399  old_addr = ip->dst_address.as_u32;
400  make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
401	      sm->outside_fib_index, 0, 0);
402  if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
403    {
404      make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
405      if (clib_bihash_search_8_8
406	  (&sm->static_mapping_by_external, &kv, &value))
407	return;
408
409      m = pool_elt_at_index (sm->static_mappings, value.value);
410      if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
411	vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
412      new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
413    }
414  else
415    {
416      s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
417      if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
418	vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
419      new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
420    }
421  sum = ip->checksum;
422  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
423  ip->checksum = ip_csum_fold (sum);
424}
425#endif
426
427#ifndef CLIB_MARCH_VARIANT
428void
429nat44_reass_hairpinning (snat_main_t * sm,
430			 vlib_buffer_t * b0,
431			 ip4_header_t * ip0,
432			 u16 sport, u16 dport, u32 proto0, int is_ed)
433{
434  snat_session_key_t key0, sm0;
435  snat_session_t *s0;
436  clib_bihash_kv_8_8_t kv0, value0;
437  ip_csum_t sum0;
438  u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
439  u16 new_dst_port0, old_dst_port0;
440  udp_header_t *udp0;
441  tcp_header_t *tcp0;
442  int rv;
443
444  key0.addr = ip0->dst_address;
445  key0.port = dport;
446  key0.protocol = proto0;
447  key0.fib_index = sm->outside_fib_index;
448  kv0.key = key0.as_u64;
449
450  udp0 = ip4_next_header (ip0);
451
452  /* Check if destination is static mappings */
453  if (!snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0, 0))
454    {
455      new_dst_addr0 = sm0.addr.as_u32;
456      new_dst_port0 = sm0.port;
457      vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
458    }
459  /* or active sessions */
460  else
461    {
462      if (sm->num_workers > 1)
463	ti =
464	  (clib_net_to_host_u16 (udp0->dst_port) -
465	   1024) / sm->port_per_thread;
466      else
467	ti = sm->num_workers;
468
469      if (is_ed)
470	{
471	  clib_bihash_kv_16_8_t ed_kv, ed_value;
472	  make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
473		      ip0->protocol, sm->outside_fib_index, udp0->dst_port,
474		      udp0->src_port);
475	  rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
476					&ed_kv, &ed_value);
477	  si = ed_value.value;
478	}
479      else
480	{
481	  rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
482				       &value0);
483	  si = value0.value;
484	}
485      if (!rv)
486	{
487	  s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
488	  new_dst_addr0 = s0->in2out.addr.as_u32;
489	  new_dst_port0 = s0->in2out.port;
490	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
491	}
492    }
493
494  /* Destination is behind the same NAT, use internal address and port */
495  if (new_dst_addr0)
496    {
497      old_dst_addr0 = ip0->dst_address.as_u32;
498      ip0->dst_address.as_u32 = new_dst_addr0;
499      sum0 = ip0->checksum;
500      sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
501			     ip4_header_t, dst_address);
502      ip0->checksum = ip_csum_fold (sum0);
503
504      old_dst_port0 = dport;
505      if (PREDICT_TRUE (new_dst_port0 != old_dst_port0 &&
506			ip4_is_first_fragment (ip0)))
507	{
508	  if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
509	    {
510	      tcp0 = ip4_next_header (ip0);
511	      tcp0->dst = new_dst_port0;
512	      sum0 = tcp0->checksum;
513	      sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
514				     ip4_header_t, dst_address);
515	      sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
516				     ip4_header_t /* cheat */ , length);
517	      tcp0->checksum = ip_csum_fold (sum0);
518	    }
519	  else
520	    {
521	      udp0->dst_port = new_dst_port0;
522	      udp0->checksum = 0;
523	    }
524	}
525      else
526	{
527	  if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
528	    {
529	      tcp0 = ip4_next_header (ip0);
530	      sum0 = tcp0->checksum;
531	      sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
532				     ip4_header_t, dst_address);
533	      tcp0->checksum = ip_csum_fold (sum0);
534	    }
535	}
536    }
537}
538#endif
539
540static inline uword
541nat44_hairpinning_fn_inline (vlib_main_t * vm,
542			     vlib_node_runtime_t * node,
543			     vlib_frame_t * frame, int is_ed)
544{
545  u32 n_left_from, *from, *to_next, stats_node_index;
546  nat_hairpin_next_t next_index;
547  u32 pkts_processed = 0;
548  snat_main_t *sm = &snat_main;
549  vnet_feature_main_t *fm = &feature_main;
550  u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
551  vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
552
553  stats_node_index = is_ed ? sm->ed_hairpinning_node_index :
554    sm->hairpinning_node_index;
555  from = vlib_frame_vector_args (frame);
556  n_left_from = frame->n_vectors;
557  next_index = node->cached_next_index;
558
559  while (n_left_from > 0)
560    {
561      u32 n_left_to_next;
562
563      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
564
565      while (n_left_from > 0 && n_left_to_next > 0)
566	{
567	  u32 bi0;
568	  vlib_buffer_t *b0;
569	  u32 next0;
570	  ip4_header_t *ip0;
571	  u32 proto0;
572	  udp_header_t *udp0;
573	  tcp_header_t *tcp0;
574
575	  /* speculatively enqueue b0 to the current next frame */
576	  bi0 = from[0];
577	  to_next[0] = bi0;
578	  from += 1;
579	  to_next += 1;
580	  n_left_from -= 1;
581	  n_left_to_next -= 1;
582
583	  b0 = vlib_get_buffer (vm, bi0);
584	  ip0 = vlib_buffer_get_current (b0);
585	  udp0 = ip4_next_header (ip0);
586	  tcp0 = (tcp_header_t *) udp0;
587
588	  proto0 = ip_proto_to_snat_proto (ip0->protocol);
589
590	  vnet_get_config_data (&cm->config_main, &b0->current_config_index,
591				&next0, 0);
592
593	  if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed))
594	    next0 = NAT_HAIRPIN_NEXT_LOOKUP;
595
596	  pkts_processed += next0 != NAT_HAIRPIN_NEXT_DROP;
597
598	  /* verify speculative enqueue, maybe switch current next frame */
599	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
600					   to_next, n_left_to_next,
601					   bi0, next0);
602	}
603
604      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
605    }
606
607  vlib_node_increment_counter (vm, stats_node_index,
608			       NAT44_HAIRPIN_ERROR_PROCESSED, pkts_processed);
609  return frame->n_vectors;
610}
611
612VLIB_NODE_FN (nat44_hairpinning_node) (vlib_main_t * vm,
613				       vlib_node_runtime_t * node,
614				       vlib_frame_t * frame)
615{
616  return nat44_hairpinning_fn_inline (vm, node, frame, 0);
617}
618
619/* *INDENT-OFF* */
620VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
621  .name = "nat44-hairpinning",
622  .vector_size = sizeof (u32),
623  .type = VLIB_NODE_TYPE_INTERNAL,
624  .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
625  .error_strings = nat44_hairpin_error_strings,
626  .n_next_nodes = NAT_HAIRPIN_N_NEXT,
627  .next_nodes = {
628    [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
629    [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
630  },
631};
632/* *INDENT-ON* */
633
634VLIB_NODE_FN (nat44_ed_hairpinning_node) (vlib_main_t * vm,
635					  vlib_node_runtime_t * node,
636					  vlib_frame_t * frame)
637{
638  return nat44_hairpinning_fn_inline (vm, node, frame, 1);
639}
640
641/* *INDENT-OFF* */
642VLIB_REGISTER_NODE (nat44_ed_hairpinning_node) = {
643  .name = "nat44-ed-hairpinning",
644  .vector_size = sizeof (u32),
645  .type = VLIB_NODE_TYPE_INTERNAL,
646  .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
647  .error_strings = nat44_hairpin_error_strings,
648  .n_next_nodes = NAT_HAIRPIN_N_NEXT,
649  .next_nodes = {
650    [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
651    [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
652  },
653};
654/* *INDENT-ON* */
655
656static inline uword
657snat_hairpin_dst_fn_inline (vlib_main_t * vm,
658			    vlib_node_runtime_t * node,
659			    vlib_frame_t * frame, int is_ed)
660{
661  u32 n_left_from, *from, *to_next, stats_node_index;
662  nat_hairpin_next_t next_index;
663  u32 pkts_processed = 0;
664  snat_main_t *sm = &snat_main;
665
666  stats_node_index = is_ed ? sm->ed_hairpin_dst_node_index :
667    sm->hairpin_dst_node_index;
668
669  from = vlib_frame_vector_args (frame);
670  n_left_from = frame->n_vectors;
671  next_index = node->cached_next_index;
672
673  while (n_left_from > 0)
674    {
675      u32 n_left_to_next;
676
677      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
678
679      while (n_left_from > 0 && n_left_to_next > 0)
680	{
681	  u32 bi0;
682	  vlib_buffer_t *b0;
683	  u32 next0;
684	  ip4_header_t *ip0;
685	  u32 proto0;
686
687	  /* speculatively enqueue b0 to the current next frame */
688	  bi0 = from[0];
689	  to_next[0] = bi0;
690	  from += 1;
691	  to_next += 1;
692	  n_left_from -= 1;
693	  n_left_to_next -= 1;
694
695	  b0 = vlib_get_buffer (vm, bi0);
696	  next0 = NAT_HAIRPIN_NEXT_LOOKUP;
697	  ip0 = vlib_buffer_get_current (b0);
698
699	  proto0 = ip_proto_to_snat_proto (ip0->protocol);
700
701	  vnet_buffer (b0)->snat.flags = 0;
702	  if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
703	    {
704	      if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
705		{
706		  udp_header_t *udp0 = ip4_next_header (ip0);
707		  tcp_header_t *tcp0 = (tcp_header_t *) udp0;
708
709		  snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed);
710		}
711	      else if (proto0 == SNAT_PROTOCOL_ICMP)
712		{
713		  icmp46_header_t *icmp0 = ip4_next_header (ip0);
714
715		  snat_icmp_hairpinning (sm, b0, ip0, icmp0, is_ed);
716		}
717	      else
718		{
719		  if (is_ed)
720		    nat44_ed_hairpinning_unknown_proto (sm, b0, ip0);
721		  else
722		    nat_hairpinning_sm_unknown_proto (sm, b0, ip0);
723		}
724
725	      vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
726	    }
727
728	  pkts_processed += next0 != NAT_HAIRPIN_NEXT_DROP;
729
730	  /* verify speculative enqueue, maybe switch current next frame */
731	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
732					   to_next, n_left_to_next,
733					   bi0, next0);
734	}
735
736      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
737    }
738
739  vlib_node_increment_counter (vm, stats_node_index,
740			       NAT44_HAIRPIN_ERROR_PROCESSED, pkts_processed);
741  return frame->n_vectors;
742}
743
744VLIB_NODE_FN (snat_hairpin_dst_node) (vlib_main_t * vm,
745				      vlib_node_runtime_t * node,
746				      vlib_frame_t * frame)
747{
748  return snat_hairpin_dst_fn_inline (vm, node, frame, 0);
749}
750
751/* *INDENT-OFF* */
752VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
753  .name = "nat44-hairpin-dst",
754  .vector_size = sizeof (u32),
755  .type = VLIB_NODE_TYPE_INTERNAL,
756  .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
757  .error_strings = nat44_hairpin_error_strings,
758  .n_next_nodes = NAT_HAIRPIN_N_NEXT,
759  .next_nodes = {
760    [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
761    [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
762  },
763};
764/* *INDENT-ON* */
765
766VLIB_NODE_FN (nat44_ed_hairpin_dst_node) (vlib_main_t * vm,
767					  vlib_node_runtime_t * node,
768					  vlib_frame_t * frame)
769{
770  return snat_hairpin_dst_fn_inline (vm, node, frame, 1);
771}
772
773/* *INDENT-OFF* */
774VLIB_REGISTER_NODE (nat44_ed_hairpin_dst_node) = {
775  .name = "nat44-ed-hairpin-dst",
776  .vector_size = sizeof (u32),
777  .type = VLIB_NODE_TYPE_INTERNAL,
778  .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
779  .error_strings = nat44_hairpin_error_strings,
780  .n_next_nodes = NAT_HAIRPIN_N_NEXT,
781  .next_nodes = {
782    [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
783    [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
784  },
785};
786/* *INDENT-ON* */
787
788static inline uword
789snat_hairpin_src_fn_inline (vlib_main_t * vm,
790			    vlib_node_runtime_t * node,
791			    vlib_frame_t * frame, int is_ed)
792{
793  u32 n_left_from, *from, *to_next, stats_node_index;
794  snat_hairpin_src_next_t next_index;
795  u32 pkts_processed = 0;
796  snat_main_t *sm = &snat_main;
797
798  stats_node_index = is_ed ? sm->ed_hairpin_src_node_index :
799    sm->hairpin_src_node_index;
800
801  from = vlib_frame_vector_args (frame);
802  n_left_from = frame->n_vectors;
803  next_index = node->cached_next_index;
804
805  while (n_left_from > 0)
806    {
807      u32 n_left_to_next;
808
809      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
810
811      while (n_left_from > 0 && n_left_to_next > 0)
812	{
813	  u32 bi0;
814	  vlib_buffer_t *b0;
815	  u32 next0;
816	  snat_interface_t *i;
817	  u32 sw_if_index0;
818
819	  /* speculatively enqueue b0 to the current next frame */
820	  bi0 = from[0];
821	  to_next[0] = bi0;
822	  from += 1;
823	  to_next += 1;
824	  n_left_from -= 1;
825	  n_left_to_next -= 1;
826
827	  b0 = vlib_get_buffer (vm, bi0);
828	  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
829	  vnet_feature_next (&next0, b0);
830
831          /* *INDENT-OFF* */
832          pool_foreach (i, sm->output_feature_interfaces,
833          ({
834            /* Only packets from NAT inside interface */
835            if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
836              {
837                if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
838                                    SNAT_FLAG_HAIRPINNING))
839                  {
840                    if (PREDICT_TRUE (sm->num_workers > 1))
841                      next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
842                    else
843                      next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
844                  }
845                break;
846              }
847          }));
848          /* *INDENT-ON* */
849
850	  pkts_processed += next0 != SNAT_HAIRPIN_SRC_NEXT_DROP;
851
852	  /* verify speculative enqueue, maybe switch current next frame */
853	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
854					   to_next, n_left_to_next,
855					   bi0, next0);
856	}
857
858      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
859    }
860
861  vlib_node_increment_counter (vm, stats_node_index,
862			       NAT44_HAIRPIN_ERROR_PROCESSED, pkts_processed);
863  return frame->n_vectors;
864}
865
866VLIB_NODE_FN (snat_hairpin_src_node) (vlib_main_t * vm,
867				      vlib_node_runtime_t * node,
868				      vlib_frame_t * frame)
869{
870  return snat_hairpin_src_fn_inline (vm, node, frame, 0);
871}
872
873/* *INDENT-OFF* */
874VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
875  .name = "nat44-hairpin-src",
876  .vector_size = sizeof (u32),
877  .type = VLIB_NODE_TYPE_INTERNAL,
878  .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
879  .error_strings = nat44_hairpin_error_strings,
880  .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
881  .next_nodes = {
882     [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
883     [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
884     [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
885     [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
886  },
887};
888/* *INDENT-ON* */
889
890VLIB_NODE_FN (nat44_ed_hairpin_src_node) (vlib_main_t * vm,
891					  vlib_node_runtime_t * node,
892					  vlib_frame_t * frame)
893{
894  return snat_hairpin_src_fn_inline (vm, node, frame, 1);
895}
896
897/* *INDENT-OFF* */
898VLIB_REGISTER_NODE (nat44_ed_hairpin_src_node) = {
899  .name = "nat44-ed-hairpin-src",
900  .vector_size = sizeof (u32),
901  .type = VLIB_NODE_TYPE_INTERNAL,
902  .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
903  .error_strings = nat44_hairpin_error_strings,
904  .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
905  .next_nodes = {
906     [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
907     [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-ed-in2out-output",
908     [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
909     [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
910  },
911};
912/* *INDENT-ON* */
913
914/*
915 * fd.io coding-style-patch-verification: ON
916 *
917 * Local Variables:
918 * eval: (c-set-style "gnu")
919 * End:
920 */
921