vcl_private.h revision d4c70922
1/*
2 * Copyright (c) 2018-2019 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#ifndef SRC_VCL_VCL_PRIVATE_H_
17#define SRC_VCL_VCL_PRIVATE_H_
18
19#include <vnet/session/application_interface.h>
20#include <vcl/vppcom.h>
21#include <vcl/vcl_debug.h>
22
23#if (CLIB_DEBUG > 0)
24/* Set VPPCOM_DEBUG_INIT 2 for connection debug,
25 *                       3 for read/write debug output
26 * or
27 *    export VCL_DEBUG=<#> to set dynamically.
28 */
29#define VPPCOM_DEBUG_INIT 1
30#else
31#define VPPCOM_DEBUG_INIT 0
32#endif
33
34#define VPPCOM_DEBUG vcm->debug
35
36extern __thread uword __vcl_worker_index;
37
38static inline void
39vcl_set_worker_index (uword wrk_index)
40{
41  __vcl_worker_index = wrk_index;
42}
43
44static inline uword
45vcl_get_worker_index (void)
46{
47  return __vcl_worker_index;
48}
49
50/*
51 * VPPCOM Private definitions and functions.
52 */
53typedef enum
54{
55  STATE_APP_START,
56  STATE_APP_CONN_VPP,
57  STATE_APP_ENABLED,
58  STATE_APP_ATTACHED,
59  STATE_APP_ADDING_WORKER,
60  STATE_APP_ADDING_TLS_DATA,
61  STATE_APP_FAILED,
62  STATE_APP_READY
63} app_state_t;
64
65typedef enum
66{
67  STATE_START = 0,
68  STATE_CONNECT = 0x01,
69  STATE_LISTEN = 0x02,
70  STATE_ACCEPT = 0x04,
71  STATE_VPP_CLOSING = 0x08,
72  STATE_DISCONNECT = 0x10,
73  STATE_FAILED = 0x20,
74  STATE_UPDATED = 0x40,
75  STATE_LISTEN_NO_MQ = 0x80,
76} vcl_session_state_t;
77
78#define SERVER_STATE_OPEN  (STATE_ACCEPT|STATE_VPP_CLOSING)
79#define CLIENT_STATE_OPEN  (STATE_CONNECT|STATE_VPP_CLOSING)
80#define STATE_OPEN (SERVER_STATE_OPEN | CLIENT_STATE_OPEN)
81
82typedef struct epoll_event vppcom_epoll_event_t;
83
84typedef struct
85{
86  u32 next_sh;
87  u32 prev_sh;
88  u32 vep_sh;
89  vppcom_epoll_event_t ev;
90#define VEP_DEFAULT_ET_MASK  (EPOLLIN|EPOLLOUT)
91#define VEP_UNSUPPORTED_EVENTS (EPOLLONESHOT|EPOLLEXCLUSIVE)
92  u32 et_mask;
93} vppcom_epoll_t;
94
95/* Select uses the vcl_si_set as if a clib_bitmap. Make sure they are the
96 * same size */
97STATIC_ASSERT (sizeof (clib_bitmap_t) == sizeof (vcl_si_set),
98	       "vppcom bitmap size mismatch");
99
100typedef struct
101{
102  u8 is_ip4;
103  ip46_address_t ip46;
104} vppcom_ip46_t;
105
106#define VCL_ACCEPTED_F_CLOSED 	(1 << 0)
107#define VCL_ACCEPTED_F_RESET 	(1 << 1)
108
109typedef struct vcl_session_msg
110{
111  u32 next;
112  union
113  {
114    session_accepted_msg_t accepted_msg;
115  };
116  u32 flags;
117} vcl_session_msg_t;
118
119typedef enum
120{
121  VCL_SESS_ATTR_SERVER,
122  VCL_SESS_ATTR_CUT_THRU,
123  VCL_SESS_ATTR_VEP,
124  VCL_SESS_ATTR_VEP_SESSION,
125  VCL_SESS_ATTR_LISTEN,		// SOL_SOCKET,SO_ACCEPTCONN
126  VCL_SESS_ATTR_NONBLOCK,	// fcntl,O_NONBLOCK
127  VCL_SESS_ATTR_REUSEADDR,	// SOL_SOCKET,SO_REUSEADDR
128  VCL_SESS_ATTR_REUSEPORT,	// SOL_SOCKET,SO_REUSEPORT
129  VCL_SESS_ATTR_BROADCAST,	// SOL_SOCKET,SO_BROADCAST
130  VCL_SESS_ATTR_V6ONLY,		// SOL_TCP,IPV6_V6ONLY
131  VCL_SESS_ATTR_KEEPALIVE,	// SOL_SOCKET,SO_KEEPALIVE
132  VCL_SESS_ATTR_TCP_NODELAY,	// SOL_TCP,TCP_NODELAY
133  VCL_SESS_ATTR_TCP_KEEPIDLE,	// SOL_TCP,TCP_KEEPIDLE
134  VCL_SESS_ATTR_TCP_KEEPINTVL,	// SOL_TCP,TCP_KEEPINTVL
135  VCL_SESS_ATTR_SHUT_RD,
136  VCL_SESS_ATTR_SHUT_WR,
137  VCL_SESS_ATTR_MAX
138} vppcom_session_attr_t;
139
140#define VCL_SESS_ATTR_SET(ATTR, VAL)            \
141do {                                            \
142  (ATTR) |= 1 << (VAL);                         \
143 } while (0)
144
145#define VCL_SESS_ATTR_CLR(ATTR, VAL)            \
146do {                                            \
147  (ATTR) &= ~(1 << (VAL));                      \
148 } while (0)
149
150#define VCL_SESS_ATTR_TEST(ATTR, VAL)           \
151  ((ATTR) & (1 << (VAL)) ? 1 : 0)
152
153typedef struct
154{
155  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
156#define _(type, name) type name;
157  foreach_app_session_field
158#undef _
159  u32 sndbuf_size;		// VPP-TBD: Hack until support setsockopt(SO_SNDBUF)
160  u32 rcvbuf_size;		// VPP-TBD: Hack until support setsockopt(SO_RCVBUF)
161  u32 user_mss;			// VPP-TBD: Hack until support setsockopt(TCP_MAXSEG)
162  u64 vpp_handle;
163  u32 vpp_thread_index;
164
165  svm_fifo_t *ct_rx_fifo;
166  svm_fifo_t *ct_tx_fifo;
167
168  /* Socket configuration state */
169  u8 is_vep;
170  u8 is_vep_session;
171  /* VCL session index of the listening session (if any) */
172  u32 listener_index;
173  /* Accepted sessions on this listener */
174  int n_accepted_sessions;
175  u8 has_rx_evt;
176  u32 attr;
177  u64 parent_handle;
178  vppcom_epoll_t vep;
179  int libc_epfd;
180  svm_msg_q_t *our_evt_q;
181  vcl_session_msg_t *accept_evts_fifo;
182#if VCL_ELOG
183  elog_track_t elog_track;
184#endif
185} vcl_session_t;
186
187typedef struct vppcom_cfg_t_
188{
189  uword heapsize;
190  u32 max_workers;
191  u32 vpp_api_q_length;
192  uword segment_baseva;
193  u32 segment_size;
194  u32 add_segment_size;
195  u32 preallocated_fifo_pairs;
196  u32 rx_fifo_size;
197  u32 tx_fifo_size;
198  u32 event_queue_size;
199  u32 listen_queue_size;
200  u8 app_proxy_transport_tcp;
201  u8 app_proxy_transport_udp;
202  u8 app_scope_local;
203  u8 app_scope_global;
204  u8 *namespace_id;
205  u64 namespace_secret;
206  u8 use_mq_eventfd;
207  f64 app_timeout;
208  f64 session_timeout;
209  f64 accept_timeout;
210  u32 event_ring_size;
211  char *event_log_path;
212  u8 *vpp_api_filename;
213  u8 *vpp_api_socket_name;
214  u8 *vpp_api_chroot;
215  u32 tls_engine;
216} vppcom_cfg_t;
217
218void vppcom_cfg (vppcom_cfg_t * vcl_cfg);
219
220typedef struct vcl_cut_through_registration_
221{
222  svm_msg_q_t *mq;
223  svm_msg_q_t *peer_mq;
224  u32 sid;
225  u32 epoll_evt_conn_index;	/*< mq evt connection index part of
226				   the mqs evtfd epoll (if used) */
227} vcl_cut_through_registration_t;
228
229typedef struct vcl_mq_evt_conn_
230{
231  svm_msg_q_t *mq;
232  int mq_fd;
233} vcl_mq_evt_conn_t;
234
235typedef struct vcl_worker_
236{
237  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
238
239  /* Session pool */
240  vcl_session_t *sessions;
241
242  /** Worker/thread index in current process */
243  u32 wrk_index;
244
245  /** Worker index in vpp*/
246  u32 vpp_wrk_index;
247
248  /** API client handle */
249  u32 my_client_index;
250
251  /** State of the connection, shared between msg RX thread and main thread */
252  volatile app_state_t wrk_state;
253
254  /** VPP binary api input queue */
255  svm_queue_t *vl_input_queue;
256
257  /** VPP mq to be used for exchanging control messages */
258  svm_msg_q_t *ctrl_mq;
259
260  /** Message queues epoll fd. Initialized only if using mqs with eventfds */
261  int mqs_epfd;
262
263  /** Pool of event message queue event connections */
264  vcl_mq_evt_conn_t *mq_evt_conns;
265
266  /** Per worker buffer for receiving mq epoll events */
267  struct epoll_event *mq_events;
268
269  /** Hash table for disconnect processing */
270  uword *session_index_by_vpp_handles;
271
272  /** Select bitmaps */
273  clib_bitmap_t *rd_bitmap;
274  clib_bitmap_t *wr_bitmap;
275  clib_bitmap_t *ex_bitmap;
276
277  /** Our event message queue */
278  svm_msg_q_t *app_event_queue;
279
280  /** VPP workers event message queues */
281  svm_msg_q_t **vpp_event_queues;
282
283  /** For deadman timers */
284  clib_time_t clib_time;
285
286  /** Vector acting as buffer for mq messages */
287  svm_msg_q_msg_t *mq_msg_vector;
288
289  /** Vector of unhandled events */
290  session_event_t *unhandled_evts_vector;
291
292  u32 *pending_session_wrk_updates;
293
294  /** Used also as a thread stop key buffer */
295  pthread_t thread_id;
296
297  /** Current pid, may be different from main_pid if forked child */
298  pid_t current_pid;
299
300  u32 forked_child;
301
302  socket_client_main_t bapi_sock_ctx;
303  memory_client_main_t bapi_shm_ctx;
304  api_main_t bapi_api_ctx;
305} vcl_worker_t;
306
307typedef struct vppcom_main_t_
308{
309  u8 is_init;
310  u32 debug;
311  pthread_t main_cpu;
312
313  /** Main process pid */
314  pid_t main_pid;
315
316  /** App's index in vpp. It's used by vpp to identify the app */
317  u32 app_index;
318
319  /** State of the connection, shared between msg RX thread and main thread */
320  volatile app_state_t app_state;
321
322  u8 *app_name;
323
324  /** VCL configuration */
325  vppcom_cfg_t cfg;
326
327  volatile u32 forking;
328
329  /** Workers */
330  vcl_worker_t *workers;
331
332  /** Lock to protect worker registrations */
333  clib_spinlock_t workers_lock;
334
335  /** Lock to protect segment hash table */
336  clib_rwlock_t segment_table_lock;
337
338  /** Mapped segments table */
339  uword *segment_table;
340
341  /** Control mq obtained from attach */
342  svm_msg_q_t *ctrl_mq;
343
344  fifo_segment_main_t segment_main;
345
346#ifdef VCL_ELOG
347  /* VPP Event-logger */
348  elog_main_t elog_main;
349  elog_track_t elog_track;
350#endif
351
352  /* VNET_API_ERROR_FOO -> "Foo" hash table */
353  uword *error_string_by_error_number;
354
355} vppcom_main_t;
356
357extern vppcom_main_t *vcm;
358
359#define VCL_INVALID_SESSION_INDEX ((u32)~0)
360#define VCL_INVALID_SESSION_HANDLE ((u64)~0)
361#define VCL_INVALID_SEGMENT_INDEX ((u32)~0)
362#define VCL_INVALID_SEGMENT_HANDLE ((u64)~0)
363
364static inline vcl_session_t *
365vcl_session_alloc (vcl_worker_t * wrk)
366{
367  vcl_session_t *s;
368  pool_get (wrk->sessions, s);
369  memset (s, 0, sizeof (*s));
370  s->session_index = s - wrk->sessions;
371  s->listener_index = VCL_INVALID_SESSION_INDEX;
372  return s;
373}
374
375static inline void
376vcl_session_free (vcl_worker_t * wrk, vcl_session_t * s)
377{
378  pool_put (wrk->sessions, s);
379}
380
381static inline vcl_session_t *
382vcl_session_get (vcl_worker_t * wrk, u32 session_index)
383{
384  if (pool_is_free_index (wrk->sessions, session_index))
385    return 0;
386  return pool_elt_at_index (wrk->sessions, session_index);
387}
388
389static inline vcl_session_handle_t
390vcl_session_handle_from_index (u32 session_index)
391{
392  ASSERT (session_index < 2 << 24);
393  return (vcl_get_worker_index () << 24 | session_index);
394}
395
396static inline vcl_session_handle_t
397vcl_session_handle (vcl_session_t * s)
398{
399  return vcl_session_handle_from_index (s->session_index);
400}
401
402static inline void
403vcl_session_handle_parse (u32 handle, u32 * wrk_index, u32 * session_index)
404{
405  *wrk_index = handle >> 24;
406  *session_index = handle & 0xFFFFFF;
407}
408
409static inline vcl_session_t *
410vcl_session_get_w_handle (vcl_worker_t * wrk, u32 session_handle)
411{
412  u32 session_index, wrk_index;
413  vcl_session_handle_parse (session_handle, &wrk_index, &session_index);
414  ASSERT (wrk_index == wrk->wrk_index);
415  return vcl_session_get (wrk, session_index);
416}
417
418static inline vcl_session_t *
419vcl_session_get_w_vpp_handle (vcl_worker_t * wrk, u64 vpp_handle)
420{
421  uword *p;
422  if ((p = hash_get (wrk->session_index_by_vpp_handles, vpp_handle)))
423    return vcl_session_get (wrk, (u32) p[0]);
424  return 0;
425}
426
427static inline u32
428vcl_session_index_from_vpp_handle (vcl_worker_t * wrk, u64 vpp_handle)
429{
430  uword *p;
431  if ((p = hash_get (wrk->session_index_by_vpp_handles, vpp_handle)))
432    return p[0];
433  return VCL_INVALID_SESSION_INDEX;
434}
435
436static inline void
437vcl_session_table_add_vpp_handle (vcl_worker_t * wrk, u64 handle, u32 value)
438{
439  hash_set (wrk->session_index_by_vpp_handles, handle, value);
440}
441
442static inline void
443vcl_session_table_del_vpp_handle (vcl_worker_t * wrk, u64 vpp_handle)
444{
445  hash_unset (wrk->session_index_by_vpp_handles, vpp_handle);
446}
447
448static inline uword *
449vcl_session_table_lookup_vpp_handle (vcl_worker_t * wrk, u64 handle)
450{
451  return hash_get (wrk->session_index_by_vpp_handles, handle);
452}
453
454static inline void
455vcl_session_table_add_listener (vcl_worker_t * wrk, u64 listener_handle,
456				u32 value)
457{
458  hash_set (wrk->session_index_by_vpp_handles, listener_handle, value);
459}
460
461static inline void
462vcl_session_table_del_listener (vcl_worker_t * wrk, u64 listener_handle)
463{
464  hash_unset (wrk->session_index_by_vpp_handles, listener_handle);
465}
466
467static inline int
468vcl_session_is_connectable_listener (vcl_worker_t * wrk,
469				     vcl_session_t * session)
470{
471  /* Tell if we session_handle is a QUIC session.
472   * We can be in the following cases :
473   * Listen session <- QUIC session <- Stream session
474   * QUIC session <- Stream session
475   */
476  vcl_session_t *ls;
477  if (session->session_type != VPPCOM_PROTO_QUIC)
478    return 0;
479  if (session->listener_index == VCL_INVALID_SESSION_INDEX)
480    return !(session->session_state & STATE_LISTEN);
481  ls = vcl_session_get_w_handle (wrk, session->listener_index);
482  if (!ls)
483    return VPPCOM_EBADFD;
484  return ls->session_state & STATE_LISTEN;
485}
486
487static inline vcl_session_t *
488vcl_session_table_lookup_listener (vcl_worker_t * wrk, u64 handle)
489{
490  uword *p;
491  vcl_session_t *session;
492
493  p = hash_get (wrk->session_index_by_vpp_handles, handle);
494  if (!p)
495    {
496      VDBG (0, "could not find listen session: unknown vpp listener handle"
497	    " %llx", handle);
498      return 0;
499    }
500  session = vcl_session_get (wrk, p[0]);
501  if (!session)
502    {
503      VDBG (1, "invalid listen session index (%u)", p[0]);
504      return 0;
505    }
506
507  ASSERT ((session->session_state & (STATE_LISTEN | STATE_LISTEN_NO_MQ)) ||
508	  vcl_session_is_connectable_listener (wrk, session));
509  return session;
510}
511
512const char *vppcom_session_state_str (vcl_session_state_t state);
513
514static inline u8
515vcl_session_is_ct (vcl_session_t * s)
516{
517  return (s->ct_tx_fifo != 0);
518}
519
520static inline u8
521vcl_session_is_open (vcl_session_t * s)
522{
523  return ((s->session_state & STATE_OPEN)
524	  || (s->session_state == STATE_LISTEN
525	      && s->session_type == VPPCOM_PROTO_UDP));
526}
527
528static inline u8
529vcl_session_is_closing (vcl_session_t * s)
530{
531  return (s->session_state == STATE_VPP_CLOSING
532	  || s->session_state == STATE_DISCONNECT);
533}
534
535static inline int
536vcl_session_closing_error (vcl_session_t * s)
537{
538  return s->session_state == STATE_DISCONNECT ? VPPCOM_ECONNRESET : 0;
539}
540
541static inline int
542vcl_session_closed_error (vcl_session_t * s)
543{
544  return s->session_state == STATE_DISCONNECT
545    ? VPPCOM_ECONNRESET : VPPCOM_ENOTCONN;
546}
547
548/*
549 * Helpers
550 */
551int vcl_wait_for_app_state_change (app_state_t app_state);
552vcl_mq_evt_conn_t *vcl_mq_evt_conn_alloc (vcl_worker_t * wrk);
553u32 vcl_mq_evt_conn_index (vcl_worker_t * wrk, vcl_mq_evt_conn_t * mqc);
554vcl_mq_evt_conn_t *vcl_mq_evt_conn_get (vcl_worker_t * wrk, u32 mq_conn_idx);
555int vcl_mq_epoll_add_evfd (vcl_worker_t * wrk, svm_msg_q_t * mq);
556int vcl_mq_epoll_del_evfd (vcl_worker_t * wrk, u32 mqc_index);
557
558vcl_worker_t *vcl_worker_alloc_and_init (void);
559void vcl_worker_cleanup (vcl_worker_t * wrk, u8 notify_vpp);
560int vcl_worker_register_with_vpp (void);
561int vcl_worker_set_bapi (void);
562svm_msg_q_t *vcl_worker_ctrl_mq (vcl_worker_t * wrk);
563
564void vcl_flush_mq_events (void);
565void vcl_cleanup_bapi (void);
566int vcl_session_cleanup (vcl_worker_t * wrk, vcl_session_t * session,
567			 vcl_session_handle_t sh, u8 do_disconnect);
568
569void vcl_segment_table_add (u64 segment_handle, u32 svm_segment_index);
570u32 vcl_segment_table_lookup (u64 segment_handle);
571void vcl_segment_table_del (u64 segment_handle);
572
573int vcl_session_read_ready (vcl_session_t * session);
574int vcl_session_write_ready (vcl_session_t * session);
575
576static inline vcl_worker_t *
577vcl_worker_get (u32 wrk_index)
578{
579  return pool_elt_at_index (vcm->workers, wrk_index);
580}
581
582static inline vcl_worker_t *
583vcl_worker_get_if_valid (u32 wrk_index)
584{
585  if (pool_is_free_index (vcm->workers, wrk_index))
586    return 0;
587  return pool_elt_at_index (vcm->workers, wrk_index);
588}
589
590static inline vcl_worker_t *
591vcl_worker_get_current (void)
592{
593  return vcl_worker_get (vcl_get_worker_index ());
594}
595
596static inline u8
597vcl_n_workers (void)
598{
599  return pool_elts (vcm->workers);
600}
601
602static inline svm_msg_q_t *
603vcl_session_vpp_evt_q (vcl_worker_t * wrk, vcl_session_t * s)
604{
605  return wrk->vpp_event_queues[s->vpp_thread_index];
606}
607
608void vcl_send_session_worker_update (vcl_worker_t * wrk, vcl_session_t * s,
609				     u32 wrk_index);
610/*
611 * VCL Binary API
612 */
613int vppcom_connect_to_vpp (char *app_name);
614void vppcom_disconnect_from_vpp (void);
615void vppcom_init_error_string_table (void);
616void vppcom_send_session_enable_disable (u8 is_enable);
617void vppcom_app_send_attach (void);
618void vppcom_app_send_detach (void);
619void vcl_send_session_unlisten (vcl_worker_t * wrk, vcl_session_t * s);
620void vppcom_send_disconnect_session (u64 vpp_handle);
621void vppcom_api_hookup (void);
622void vppcom_send_application_tls_cert_add (vcl_session_t * session,
623					   char *cert, u32 cert_len);
624void vppcom_send_application_tls_key_add (vcl_session_t * session, char *key,
625					  u32 key_len);
626void vcl_send_app_worker_add_del (u8 is_add);
627void vcl_send_child_worker_del (vcl_worker_t * wrk);
628
629u32 vcl_max_nsid_len (void);
630
631u8 *format_api_error (u8 * s, va_list * args);
632
633void vls_init ();
634#endif /* SRC_VCL_VCL_PRIVATE_H_ */
635
636/*
637 * fd.io coding-style-patch-verification: ON
638 *
639 * Local Variables:
640 * eval: (c-set-style "gnu")
641 * End:
642 */
643