1/*
2 * Copyright (c) 2017-2019 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include <stdio.h>
17#include <stdlib.h>
18#include <vcl/vppcom.h>
19#include <vcl/vcl_debug.h>
20#include <vcl/vcl_private.h>
21#include <svm/fifo_segment.h>
22
23__thread uword __vcl_worker_index = ~0;
24
25static int
26vcl_segment_is_not_mounted (vcl_worker_t * wrk, u64 segment_handle)
27{
28  u32 segment_index;
29
30  if (segment_handle == VCL_INVALID_SEGMENT_HANDLE)
31    return 0;
32
33  segment_index = vcl_segment_table_lookup (segment_handle);
34  if (segment_index != VCL_INVALID_SEGMENT_INDEX)
35    return 0;
36
37  return 1;
38}
39
40static inline int
41vcl_mq_dequeue_batch (vcl_worker_t * wrk, svm_msg_q_t * mq, u32 n_max_msg)
42{
43  svm_msg_q_msg_t *msg;
44  u32 n_msgs;
45  int i;
46
47  n_msgs = clib_min (svm_msg_q_size (mq), n_max_msg);
48  for (i = 0; i < n_msgs; i++)
49    {
50      vec_add2 (wrk->mq_msg_vector, msg, 1);
51      svm_msg_q_sub_w_lock (mq, msg);
52    }
53  return n_msgs;
54}
55
56const char *
57vppcom_session_state_str (vcl_session_state_t state)
58{
59  char *st;
60
61  switch (state)
62    {
63    case STATE_CLOSED:
64      st = "STATE_CLOSED";
65      break;
66
67    case STATE_CONNECT:
68      st = "STATE_CONNECT";
69      break;
70
71    case STATE_LISTEN:
72      st = "STATE_LISTEN";
73      break;
74
75    case STATE_ACCEPT:
76      st = "STATE_ACCEPT";
77      break;
78
79    case STATE_VPP_CLOSING:
80      st = "STATE_VPP_CLOSING";
81      break;
82
83    case STATE_DISCONNECT:
84      st = "STATE_DISCONNECT";
85      break;
86
87    case STATE_DETACHED:
88      st = "STATE_DETACHED";
89      break;
90
91    case STATE_UPDATED:
92      st = "STATE_UPDATED";
93      break;
94
95    case STATE_LISTEN_NO_MQ:
96      st = "STATE_LISTEN_NO_MQ";
97      break;
98
99    default:
100      st = "UNKNOWN_STATE";
101      break;
102    }
103
104  return st;
105}
106
107u8 *
108format_ip4_address (u8 * s, va_list * args)
109{
110  u8 *a = va_arg (*args, u8 *);
111  return format (s, "%d.%d.%d.%d", a[0], a[1], a[2], a[3]);
112}
113
114u8 *
115format_ip6_address (u8 * s, va_list * args)
116{
117  ip6_address_t *a = va_arg (*args, ip6_address_t *);
118  u32 i, i_max_n_zero, max_n_zeros, i_first_zero, n_zeros, last_double_colon;
119
120  i_max_n_zero = ARRAY_LEN (a->as_u16);
121  max_n_zeros = 0;
122  i_first_zero = i_max_n_zero;
123  n_zeros = 0;
124  for (i = 0; i < ARRAY_LEN (a->as_u16); i++)
125    {
126      u32 is_zero = a->as_u16[i] == 0;
127      if (is_zero && i_first_zero >= ARRAY_LEN (a->as_u16))
128	{
129	  i_first_zero = i;
130	  n_zeros = 0;
131	}
132      n_zeros += is_zero;
133      if ((!is_zero && n_zeros > max_n_zeros)
134	  || (i + 1 >= ARRAY_LEN (a->as_u16) && n_zeros > max_n_zeros))
135	{
136	  i_max_n_zero = i_first_zero;
137	  max_n_zeros = n_zeros;
138	  i_first_zero = ARRAY_LEN (a->as_u16);
139	  n_zeros = 0;
140	}
141    }
142
143  last_double_colon = 0;
144  for (i = 0; i < ARRAY_LEN (a->as_u16); i++)
145    {
146      if (i == i_max_n_zero && max_n_zeros > 1)
147	{
148	  s = format (s, "::");
149	  i += max_n_zeros - 1;
150	  last_double_colon = 1;
151	}
152      else
153	{
154	  s = format (s, "%s%x",
155		      (last_double_colon || i == 0) ? "" : ":",
156		      clib_net_to_host_u16 (a->as_u16[i]));
157	  last_double_colon = 0;
158	}
159    }
160
161  return s;
162}
163
164/* Format an IP46 address. */
165u8 *
166format_ip46_address (u8 * s, va_list * args)
167{
168  ip46_address_t *ip46 = va_arg (*args, ip46_address_t *);
169  ip46_type_t type = va_arg (*args, ip46_type_t);
170  int is_ip4 = 1;
171
172  switch (type)
173    {
174    case IP46_TYPE_ANY:
175      is_ip4 = ip46_address_is_ip4 (ip46);
176      break;
177    case IP46_TYPE_IP4:
178      is_ip4 = 1;
179      break;
180    case IP46_TYPE_IP6:
181      is_ip4 = 0;
182      break;
183    }
184
185  return is_ip4 ?
186    format (s, "%U", format_ip4_address, &ip46->ip4) :
187    format (s, "%U", format_ip6_address, &ip46->ip6);
188}
189
190/*
191 * VPPCOM Utility Functions
192 */
193
194static void
195vcl_send_session_listen (vcl_worker_t * wrk, vcl_session_t * s)
196{
197  app_session_evt_t _app_evt, *app_evt = &_app_evt;
198  session_listen_msg_t *mp;
199  svm_msg_q_t *mq;
200
201  mq = vcl_worker_ctrl_mq (wrk);
202  app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_LISTEN);
203  mp = (session_listen_msg_t *) app_evt->evt->data;
204  memset (mp, 0, sizeof (*mp));
205  mp->client_index = wrk->my_client_index;
206  mp->context = s->session_index;
207  mp->wrk_index = wrk->vpp_wrk_index;
208  mp->is_ip4 = s->transport.is_ip4;
209  clib_memcpy_fast (&mp->ip, &s->transport.lcl_ip, sizeof (mp->ip));
210  mp->port = s->transport.lcl_port;
211  mp->proto = s->session_type;
212  if (s->flags & VCL_SESSION_F_CONNECTED)
213    mp->flags = TRANSPORT_CFG_F_CONNECTED;
214  app_send_ctrl_evt_to_vpp (mq, app_evt);
215}
216
217static void
218vcl_send_session_connect (vcl_worker_t * wrk, vcl_session_t * s)
219{
220  app_session_evt_t _app_evt, *app_evt = &_app_evt;
221  session_connect_msg_t *mp;
222  svm_msg_q_t *mq;
223
224  mq = vcl_worker_ctrl_mq (wrk);
225  app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_CONNECT);
226  mp = (session_connect_msg_t *) app_evt->evt->data;
227  memset (mp, 0, sizeof (*mp));
228  mp->client_index = wrk->my_client_index;
229  mp->context = s->session_index;
230  mp->wrk_index = wrk->vpp_wrk_index;
231  mp->is_ip4 = s->transport.is_ip4;
232  mp->parent_handle = s->parent_handle;
233  clib_memcpy_fast (&mp->ip, &s->transport.rmt_ip, sizeof (mp->ip));
234  clib_memcpy_fast (&mp->lcl_ip, &s->transport.lcl_ip, sizeof (mp->lcl_ip));
235  mp->port = s->transport.rmt_port;
236  mp->lcl_port = s->transport.lcl_port;
237  mp->proto = s->session_type;
238  if (s->flags & VCL_SESSION_F_CONNECTED)
239    mp->flags |= TRANSPORT_CFG_F_CONNECTED;
240  app_send_ctrl_evt_to_vpp (mq, app_evt);
241}
242
243void
244vcl_send_session_unlisten (vcl_worker_t * wrk, vcl_session_t * s)
245{
246  app_session_evt_t _app_evt, *app_evt = &_app_evt;
247  session_unlisten_msg_t *mp;
248  svm_msg_q_t *mq;
249
250  mq = vcl_worker_ctrl_mq (wrk);
251  app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_UNLISTEN);
252  mp = (session_unlisten_msg_t *) app_evt->evt->data;
253  memset (mp, 0, sizeof (*mp));
254  mp->client_index = wrk->my_client_index;
255  mp->wrk_index = wrk->vpp_wrk_index;
256  mp->handle = s->vpp_handle;
257  mp->context = wrk->wrk_index;
258  app_send_ctrl_evt_to_vpp (mq, app_evt);
259}
260
261static void
262vcl_send_session_disconnect (vcl_worker_t * wrk, vcl_session_t * s)
263{
264  app_session_evt_t _app_evt, *app_evt = &_app_evt;
265  session_disconnect_msg_t *mp;
266  svm_msg_q_t *mq;
267
268  /* Send to thread that owns the session */
269  mq = s->vpp_evt_q;
270  app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_DISCONNECT);
271  mp = (session_disconnect_msg_t *) app_evt->evt->data;
272  memset (mp, 0, sizeof (*mp));
273  mp->client_index = wrk->my_client_index;
274  mp->handle = s->vpp_handle;
275  app_send_ctrl_evt_to_vpp (mq, app_evt);
276}
277
278static void
279vcl_send_app_detach (vcl_worker_t * wrk)
280{
281  app_session_evt_t _app_evt, *app_evt = &_app_evt;
282  session_app_detach_msg_t *mp;
283  svm_msg_q_t *mq;
284
285  mq = vcl_worker_ctrl_mq (wrk);
286  app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_APP_DETACH);
287  mp = (session_app_detach_msg_t *) app_evt->evt->data;
288  memset (mp, 0, sizeof (*mp));
289  mp->client_index = wrk->my_client_index;
290  app_send_ctrl_evt_to_vpp (mq, app_evt);
291}
292
293static void
294vcl_send_session_accepted_reply (svm_msg_q_t * mq, u32 context,
295				 session_handle_t handle, int retval)
296{
297  app_session_evt_t _app_evt, *app_evt = &_app_evt;
298  session_accepted_reply_msg_t *rmp;
299  app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_ACCEPTED_REPLY);
300  rmp = (session_accepted_reply_msg_t *) app_evt->evt->data;
301  rmp->handle = handle;
302  rmp->context = context;
303  rmp->retval = retval;
304  app_send_ctrl_evt_to_vpp (mq, app_evt);
305}
306
307static void
308vcl_send_session_disconnected_reply (svm_msg_q_t * mq, u32 context,
309				     session_handle_t handle, int retval)
310{
311  app_session_evt_t _app_evt, *app_evt = &_app_evt;
312  session_disconnected_reply_msg_t *rmp;
313  app_alloc_ctrl_evt_to_vpp (mq, app_evt,
314			     SESSION_CTRL_EVT_DISCONNECTED_REPLY);
315  rmp = (session_disconnected_reply_msg_t *) app_evt->evt->data;
316  rmp->handle = handle;
317  rmp->context = context;
318  rmp->retval = retval;
319  app_send_ctrl_evt_to_vpp (mq, app_evt);
320}
321
322static void
323vcl_send_session_reset_reply (svm_msg_q_t * mq, u32 context,
324			      session_handle_t handle, int retval)
325{
326  app_session_evt_t _app_evt, *app_evt = &_app_evt;
327  session_reset_reply_msg_t *rmp;
328  app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_RESET_REPLY);
329  rmp = (session_reset_reply_msg_t *) app_evt->evt->data;
330  rmp->handle = handle;
331  rmp->context = context;
332  rmp->retval = retval;
333  app_send_ctrl_evt_to_vpp (mq, app_evt);
334}
335
336void
337vcl_send_session_worker_update (vcl_worker_t * wrk, vcl_session_t * s,
338				u32 wrk_index)
339{
340  app_session_evt_t _app_evt, *app_evt = &_app_evt;
341  session_worker_update_msg_t *mp;
342  svm_msg_q_t *mq;
343
344  mq = vcl_session_vpp_evt_q (wrk, s);
345  app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_WORKER_UPDATE);
346  mp = (session_worker_update_msg_t *) app_evt->evt->data;
347  mp->client_index = wrk->my_client_index;
348  mp->handle = s->vpp_handle;
349  mp->req_wrk_index = wrk->vpp_wrk_index;
350  mp->wrk_index = wrk_index;
351  app_send_ctrl_evt_to_vpp (mq, app_evt);
352}
353
354static u32
355vcl_session_accepted_handler (vcl_worker_t * wrk, session_accepted_msg_t * mp,
356			      u32 ls_index)
357{
358  vcl_session_t *session, *listen_session;
359  svm_fifo_t *rx_fifo, *tx_fifo;
360  u32 vpp_wrk_index;
361  svm_msg_q_t *evt_q;
362
363  session = vcl_session_alloc (wrk);
364
365  listen_session = vcl_session_get (wrk, ls_index);
366  if (listen_session->vpp_handle != mp->listener_handle)
367    {
368      VDBG (0, "ERROR: listener handle %lu does not match session %u",
369	    mp->listener_handle, ls_index);
370      goto error;
371    }
372
373  if (vcl_segment_is_not_mounted (wrk, mp->segment_handle))
374    {
375      VDBG (0, "ERROR: segment for session %u is not mounted!",
376	    session->session_index);
377      goto error;
378    }
379
380  rx_fifo = uword_to_pointer (mp->server_rx_fifo, svm_fifo_t *);
381  tx_fifo = uword_to_pointer (mp->server_tx_fifo, svm_fifo_t *);
382  session->vpp_evt_q = uword_to_pointer (mp->vpp_event_queue_address,
383					 svm_msg_q_t *);
384  rx_fifo->client_session_index = session->session_index;
385  tx_fifo->client_session_index = session->session_index;
386  rx_fifo->client_thread_index = vcl_get_worker_index ();
387  tx_fifo->client_thread_index = vcl_get_worker_index ();
388  vpp_wrk_index = tx_fifo->master_thread_index;
389  vec_validate (wrk->vpp_event_queues, vpp_wrk_index);
390  wrk->vpp_event_queues[vpp_wrk_index] = session->vpp_evt_q;
391
392  session->vpp_handle = mp->handle;
393  session->vpp_thread_index = rx_fifo->master_thread_index;
394  session->rx_fifo = rx_fifo;
395  session->tx_fifo = tx_fifo;
396
397  session->session_state = STATE_ACCEPT;
398  session->transport.rmt_port = mp->rmt.port;
399  session->transport.is_ip4 = mp->rmt.is_ip4;
400  clib_memcpy_fast (&session->transport.rmt_ip, &mp->rmt.ip,
401		    sizeof (ip46_address_t));
402
403  vcl_session_table_add_vpp_handle (wrk, mp->handle, session->session_index);
404  session->transport.lcl_port = listen_session->transport.lcl_port;
405  session->transport.lcl_ip = listen_session->transport.lcl_ip;
406  session->session_type = listen_session->session_type;
407  session->is_dgram = vcl_proto_is_dgram (session->session_type);
408  session->listener_index = listen_session->session_index;
409  listen_session->n_accepted_sessions++;
410
411  VDBG (1, "session %u [0x%llx]: client accept request from %s address %U"
412	" port %d queue %p!", session->session_index, mp->handle,
413	mp->rmt.is_ip4 ? "IPv4" : "IPv6", format_ip46_address, &mp->rmt.ip,
414	mp->rmt.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
415	clib_net_to_host_u16 (mp->rmt.port), session->vpp_evt_q);
416  vcl_evt (VCL_EVT_ACCEPT, session, listen_session, session_index);
417
418  vcl_send_session_accepted_reply (session->vpp_evt_q, mp->context,
419				   session->vpp_handle, 0);
420
421  return session->session_index;
422
423error:
424  evt_q = uword_to_pointer (mp->vpp_event_queue_address, svm_msg_q_t *);
425  vcl_send_session_accepted_reply (evt_q, mp->context, mp->handle,
426				   VNET_API_ERROR_INVALID_ARGUMENT);
427  vcl_session_free (wrk, session);
428  return VCL_INVALID_SESSION_INDEX;
429}
430
431static u32
432vcl_session_connected_handler (vcl_worker_t * wrk,
433			       session_connected_msg_t * mp)
434{
435  u32 session_index, vpp_wrk_index;
436  svm_fifo_t *rx_fifo, *tx_fifo;
437  vcl_session_t *session = 0;
438
439  session_index = mp->context;
440  session = vcl_session_get (wrk, session_index);
441  if (!session)
442    {
443      VDBG (0, "ERROR: vpp handle 0x%llx has no session index (%u)!",
444	    mp->handle, session_index);
445      return VCL_INVALID_SESSION_INDEX;
446    }
447  if (mp->retval)
448    {
449      VDBG (0, "ERROR: session index %u: connect failed! %U",
450	    session_index, format_session_error, mp->retval);
451      session->session_state = STATE_DETACHED | STATE_DISCONNECT;
452      session->vpp_handle = mp->handle;
453      return session_index;
454    }
455
456  session->vpp_handle = mp->handle;
457  session->vpp_evt_q = uword_to_pointer (mp->vpp_event_queue_address,
458					 svm_msg_q_t *);
459  rx_fifo = uword_to_pointer (mp->server_rx_fifo, svm_fifo_t *);
460  tx_fifo = uword_to_pointer (mp->server_tx_fifo, svm_fifo_t *);
461  if (vcl_segment_is_not_mounted (wrk, mp->segment_handle))
462    {
463      VDBG (0, "segment for session %u is not mounted!",
464	    session->session_index);
465      session->session_state = STATE_DETACHED | STATE_DISCONNECT;
466      vcl_send_session_disconnect (wrk, session);
467      return session_index;
468    }
469
470  rx_fifo->client_session_index = session_index;
471  tx_fifo->client_session_index = session_index;
472  rx_fifo->client_thread_index = vcl_get_worker_index ();
473  tx_fifo->client_thread_index = vcl_get_worker_index ();
474
475  vpp_wrk_index = tx_fifo->master_thread_index;
476  vec_validate (wrk->vpp_event_queues, vpp_wrk_index);
477  wrk->vpp_event_queues[vpp_wrk_index] = session->vpp_evt_q;
478
479  if (mp->ct_rx_fifo)
480    {
481      session->ct_rx_fifo = uword_to_pointer (mp->ct_rx_fifo, svm_fifo_t *);
482      session->ct_tx_fifo = uword_to_pointer (mp->ct_tx_fifo, svm_fifo_t *);
483      if (vcl_segment_is_not_mounted (wrk, mp->ct_segment_handle))
484	{
485	  VDBG (0, "ct segment for session %u is not mounted!",
486		session->session_index);
487	  session->session_state = STATE_DETACHED | STATE_DISCONNECT;
488	  vcl_send_session_disconnect (wrk, session);
489	  return session_index;
490	}
491    }
492
493  session->rx_fifo = rx_fifo;
494  session->tx_fifo = tx_fifo;
495  session->vpp_thread_index = rx_fifo->master_thread_index;
496  session->transport.is_ip4 = mp->lcl.is_ip4;
497  clib_memcpy_fast (&session->transport.lcl_ip, &mp->lcl.ip,
498		    sizeof (session->transport.lcl_ip));
499  session->transport.lcl_port = mp->lcl.port;
500  session->session_state = STATE_CONNECT;
501
502  /* Add it to lookup table */
503  vcl_session_table_add_vpp_handle (wrk, mp->handle, session_index);
504
505  VDBG (1, "session %u [0x%llx] connected! rx_fifo %p, refcnt %d, tx_fifo %p,"
506	" refcnt %d", session_index, mp->handle, session->rx_fifo,
507	session->rx_fifo->refcnt, session->tx_fifo, session->tx_fifo->refcnt);
508
509  return session_index;
510}
511
512static int
513vcl_flag_accepted_session (vcl_session_t * session, u64 handle, u32 flags)
514{
515  vcl_session_msg_t *accepted_msg;
516  int i;
517
518  for (i = 0; i < vec_len (session->accept_evts_fifo); i++)
519    {
520      accepted_msg = &session->accept_evts_fifo[i];
521      if (accepted_msg->accepted_msg.handle == handle)
522	{
523	  accepted_msg->flags |= flags;
524	  return 1;
525	}
526    }
527  return 0;
528}
529
530static u32
531vcl_session_reset_handler (vcl_worker_t * wrk,
532			   session_reset_msg_t * reset_msg)
533{
534  vcl_session_t *session;
535  u32 sid;
536
537  sid = vcl_session_index_from_vpp_handle (wrk, reset_msg->handle);
538  session = vcl_session_get (wrk, sid);
539  if (!session)
540    {
541      VDBG (0, "request to reset unknown handle 0x%llx", reset_msg->handle);
542      return VCL_INVALID_SESSION_INDEX;
543    }
544
545  /* Caught a reset before actually accepting the session */
546  if (session->session_state == STATE_LISTEN)
547    {
548
549      if (!vcl_flag_accepted_session (session, reset_msg->handle,
550				      VCL_ACCEPTED_F_RESET))
551	VDBG (0, "session was not accepted!");
552      return VCL_INVALID_SESSION_INDEX;
553    }
554
555  if (session->session_state != STATE_CLOSED)
556    session->session_state = STATE_DISCONNECT;
557  VDBG (0, "reset session %u [0x%llx]", sid, reset_msg->handle);
558  return sid;
559}
560
561static u32
562vcl_session_bound_handler (vcl_worker_t * wrk, session_bound_msg_t * mp)
563{
564  vcl_session_t *session;
565  u32 sid = mp->context;
566
567  session = vcl_session_get (wrk, sid);
568  if (mp->retval)
569    {
570      VERR ("session %u [0x%llx]: bind failed: %U", sid, mp->handle,
571	    format_session_error, mp->retval);
572      if (session)
573	{
574	  session->session_state = STATE_DETACHED;
575	  session->vpp_handle = mp->handle;
576	  return sid;
577	}
578      else
579	{
580	  VDBG (0, "ERROR: session %u [0x%llx]: Invalid session index!",
581		sid, mp->handle);
582	  return VCL_INVALID_SESSION_INDEX;
583	}
584    }
585
586  session->vpp_handle = mp->handle;
587  session->transport.is_ip4 = mp->lcl_is_ip4;
588  clib_memcpy_fast (&session->transport.lcl_ip, mp->lcl_ip,
589		    sizeof (ip46_address_t));
590  session->transport.lcl_port = mp->lcl_port;
591  vcl_session_table_add_listener (wrk, mp->handle, sid);
592  session->session_state = STATE_LISTEN;
593
594  session->vpp_evt_q = uword_to_pointer (mp->vpp_evt_q, svm_msg_q_t *);
595  vec_validate (wrk->vpp_event_queues, 0);
596  wrk->vpp_event_queues[0] = session->vpp_evt_q;
597
598  if (vcl_session_is_cl (session))
599    {
600      svm_fifo_t *rx_fifo, *tx_fifo;
601      session->vpp_evt_q = uword_to_pointer (mp->vpp_evt_q, svm_msg_q_t *);
602      rx_fifo = uword_to_pointer (mp->rx_fifo, svm_fifo_t *);
603      rx_fifo->client_session_index = sid;
604      tx_fifo = uword_to_pointer (mp->tx_fifo, svm_fifo_t *);
605      tx_fifo->client_session_index = sid;
606      session->rx_fifo = rx_fifo;
607      session->tx_fifo = tx_fifo;
608    }
609
610  VDBG (0, "session %u [0x%llx]: listen succeeded!", sid, mp->handle);
611  return sid;
612}
613
614static void
615vcl_session_unlisten_reply_handler (vcl_worker_t * wrk, void *data)
616{
617  session_unlisten_reply_msg_t *mp = (session_unlisten_reply_msg_t *) data;
618  vcl_session_t *s;
619
620  s = vcl_session_get_w_vpp_handle (wrk, mp->handle);
621  if (!s)
622    {
623      VDBG (0, "Unlisten reply with wrong handle %llx", mp->handle);
624      return;
625    }
626  if (s->session_state != STATE_DISCONNECT)
627    {
628      /* Connected udp listener */
629      if (s->session_type == VPPCOM_PROTO_UDP
630	  && s->session_state == STATE_CLOSED)
631	return;
632
633      VDBG (0, "Unlisten session in wrong state %llx", mp->handle);
634      return;
635    }
636
637  if (mp->retval)
638    VDBG (0, "ERROR: session %u [0xllx]: unlisten failed: %U",
639	  s->session_index, mp->handle, format_session_error, mp->retval);
640
641  if (mp->context != wrk->wrk_index)
642    VDBG (0, "wrong context");
643
644  vcl_session_table_del_vpp_handle (wrk, mp->handle);
645  vcl_session_free (wrk, s);
646}
647
648static void
649vcl_session_migrated_handler (vcl_worker_t * wrk, void *data)
650{
651  session_migrated_msg_t *mp = (session_migrated_msg_t *) data;
652  vcl_session_t *s;
653
654  s = vcl_session_get_w_vpp_handle (wrk, mp->handle);
655  if (!s)
656    {
657      VDBG (0, "Migrated notification with wrong handle %llx", mp->handle);
658      return;
659    }
660
661  s->vpp_thread_index = mp->vpp_thread_index;
662  s->vpp_handle = mp->new_handle;
663  s->vpp_evt_q = uword_to_pointer (mp->vpp_evt_q, svm_msg_q_t *);
664
665  vec_validate (wrk->vpp_event_queues, s->vpp_thread_index);
666  wrk->vpp_event_queues[s->vpp_thread_index] = s->vpp_evt_q;
667
668  vcl_session_table_del_vpp_handle (wrk, mp->handle);
669  vcl_session_table_add_vpp_handle (wrk, mp->new_handle, s->session_index);
670
671  /* Generate new tx event if we have outstanding data */
672  if (svm_fifo_has_event (s->tx_fifo))
673    app_send_io_evt_to_vpp (s->vpp_evt_q, s->tx_fifo->master_session_index,
674			    SESSION_IO_EVT_TX, SVM_Q_WAIT);
675
676  VDBG (0, "Migrated 0x%lx to thread %u 0x%lx", mp->handle,
677	s->vpp_thread_index, mp->new_handle);
678}
679
680static vcl_session_t *
681vcl_session_accepted (vcl_worker_t * wrk, session_accepted_msg_t * msg)
682{
683  vcl_session_msg_t *vcl_msg;
684  vcl_session_t *session;
685
686  session = vcl_session_get_w_vpp_handle (wrk, msg->handle);
687  if (PREDICT_FALSE (session != 0))
688    VWRN ("session overlap handle %lu state %u!", msg->handle,
689	  session->session_state);
690
691  session = vcl_session_table_lookup_listener (wrk, msg->listener_handle);
692  if (!session)
693    {
694      VERR ("couldn't find listen session: listener handle %llx",
695	    msg->listener_handle);
696      return 0;
697    }
698
699  clib_fifo_add2 (session->accept_evts_fifo, vcl_msg);
700  vcl_msg->flags = 0;
701  vcl_msg->accepted_msg = *msg;
702  /* Session handle points to listener until fully accepted by app */
703  vcl_session_table_add_vpp_handle (wrk, msg->handle, session->session_index);
704
705  return session;
706}
707
708static vcl_session_t *
709vcl_session_disconnected_handler (vcl_worker_t * wrk,
710				  session_disconnected_msg_t * msg)
711{
712  vcl_session_t *session;
713
714  session = vcl_session_get_w_vpp_handle (wrk, msg->handle);
715  if (!session)
716    {
717      VDBG (0, "request to disconnect unknown handle 0x%llx", msg->handle);
718      return 0;
719    }
720
721  /* Late disconnect notification on a session that has been closed */
722  if (session->session_state == STATE_CLOSED)
723    return 0;
724
725  /* Caught a disconnect before actually accepting the session */
726  if (session->session_state == STATE_LISTEN)
727    {
728      if (!vcl_flag_accepted_session (session, msg->handle,
729				      VCL_ACCEPTED_F_CLOSED))
730	VDBG (0, "session was not accepted!");
731      return 0;
732    }
733
734  /* If not already reset change state */
735  if (session->session_state != STATE_DISCONNECT)
736    session->session_state = STATE_VPP_CLOSING;
737
738  return session;
739}
740
741static void
742vcl_session_cleanup_handler (vcl_worker_t * wrk, void *data)
743{
744  session_cleanup_msg_t *msg;
745  vcl_session_t *session;
746
747  msg = (session_cleanup_msg_t *) data;
748  session = vcl_session_get_w_vpp_handle (wrk, msg->handle);
749  if (!session)
750    {
751      VDBG (0, "disconnect confirmed for unknown handle 0x%llx", msg->handle);
752      return;
753    }
754
755  if (msg->type == SESSION_CLEANUP_TRANSPORT)
756    {
757      /* Transport was cleaned up before we confirmed close. Probably the
758       * app is still waiting for some data that cannot be delivered.
759       * Confirm close to make sure everything is cleaned up */
760      if (session->session_state == STATE_VPP_CLOSING)
761	vcl_session_cleanup (wrk, session, vcl_session_handle (session),
762			     1 /* do_disconnect */ );
763      return;
764    }
765
766  vcl_session_table_del_vpp_handle (wrk, msg->handle);
767  /* Should not happen. App did not close the connection so don't free it. */
768  if (session->session_state != STATE_CLOSED)
769    {
770      VDBG (0, "app did not close session %d", session->session_index);
771      session->session_state = STATE_DETACHED;
772      session->vpp_handle = VCL_INVALID_SESSION_HANDLE;
773      return;
774    }
775  vcl_session_free (wrk, session);
776}
777
778static void
779vcl_session_req_worker_update_handler (vcl_worker_t * wrk, void *data)
780{
781  session_req_worker_update_msg_t *msg;
782  vcl_session_t *s;
783
784  msg = (session_req_worker_update_msg_t *) data;
785  s = vcl_session_get_w_vpp_handle (wrk, msg->session_handle);
786  if (!s)
787    return;
788
789  vec_add1 (wrk->pending_session_wrk_updates, s->session_index);
790}
791
792static void
793vcl_session_worker_update_reply_handler (vcl_worker_t * wrk, void *data)
794{
795  session_worker_update_reply_msg_t *msg;
796  vcl_session_t *s;
797
798  msg = (session_worker_update_reply_msg_t *) data;
799  s = vcl_session_get_w_vpp_handle (wrk, msg->handle);
800  if (!s)
801    {
802      VDBG (0, "unknown handle 0x%llx", msg->handle);
803      return;
804    }
805  if (vcl_segment_is_not_mounted (wrk, msg->segment_handle))
806    {
807      clib_warning ("segment for session %u is not mounted!",
808		    s->session_index);
809      return;
810    }
811
812  if (s->rx_fifo)
813    {
814      s->rx_fifo = uword_to_pointer (msg->rx_fifo, svm_fifo_t *);
815      s->tx_fifo = uword_to_pointer (msg->tx_fifo, svm_fifo_t *);
816      s->rx_fifo->client_session_index = s->session_index;
817      s->tx_fifo->client_session_index = s->session_index;
818      s->rx_fifo->client_thread_index = wrk->wrk_index;
819      s->tx_fifo->client_thread_index = wrk->wrk_index;
820    }
821  s->session_state = STATE_UPDATED;
822
823  VDBG (0, "session %u[0x%llx] moved to worker %u", s->session_index,
824	s->vpp_handle, wrk->wrk_index);
825}
826
827static void
828vcl_session_app_add_segment_handler (vcl_worker_t * wrk, void *data)
829{
830  ssvm_segment_type_t seg_type = SSVM_SEGMENT_SHM;
831  session_app_add_segment_msg_t *msg;
832  u64 segment_handle;
833  int fd = -1;
834
835  msg = (session_app_add_segment_msg_t *) data;
836
837  if (msg->fd_flags)
838    {
839      vl_socket_client_recv_fd_msg2 (&wrk->bapi_sock_ctx, &fd, 1, 5);
840      seg_type = SSVM_SEGMENT_MEMFD;
841    }
842
843  segment_handle = msg->segment_handle;
844  if (segment_handle == VCL_INVALID_SEGMENT_HANDLE)
845    {
846      clib_warning ("invalid segment handle");
847      return;
848    }
849
850  if (vcl_segment_attach (segment_handle, (char *) msg->segment_name,
851			  seg_type, fd))
852    {
853      VDBG (0, "vcl_segment_attach ('%s') failed", msg->segment_name);
854      return;
855    }
856
857  VDBG (1, "mapped new segment '%s' size %d", msg->segment_name,
858	msg->segment_size);
859}
860
861static void
862vcl_session_app_del_segment_handler (vcl_worker_t * wrk, void *data)
863{
864  session_app_del_segment_msg_t *msg = (session_app_del_segment_msg_t *) data;
865  vcl_segment_detach (msg->segment_handle);
866  VDBG (1, "Unmapped segment: %d", msg->segment_handle);
867}
868
869static int
870vcl_handle_mq_event (vcl_worker_t * wrk, session_event_t * e)
871{
872  session_disconnected_msg_t *disconnected_msg;
873  vcl_session_t *session;
874
875  switch (e->event_type)
876    {
877    case SESSION_IO_EVT_RX:
878    case SESSION_IO_EVT_TX:
879      session = vcl_session_get (wrk, e->session_index);
880      if (!session || !(session->session_state & STATE_OPEN))
881	break;
882      vec_add1 (wrk->unhandled_evts_vector, *e);
883      break;
884    case SESSION_CTRL_EVT_ACCEPTED:
885      vcl_session_accepted (wrk, (session_accepted_msg_t *) e->data);
886      break;
887    case SESSION_CTRL_EVT_CONNECTED:
888      vcl_session_connected_handler (wrk,
889				     (session_connected_msg_t *) e->data);
890      break;
891    case SESSION_CTRL_EVT_DISCONNECTED:
892      disconnected_msg = (session_disconnected_msg_t *) e->data;
893      session = vcl_session_disconnected_handler (wrk, disconnected_msg);
894      if (!session)
895	break;
896      VDBG (0, "disconnected session %u [0x%llx]", session->session_index,
897	    session->vpp_handle);
898      break;
899    case SESSION_CTRL_EVT_RESET:
900      vcl_session_reset_handler (wrk, (session_reset_msg_t *) e->data);
901      break;
902    case SESSION_CTRL_EVT_BOUND:
903      vcl_session_bound_handler (wrk, (session_bound_msg_t *) e->data);
904      break;
905    case SESSION_CTRL_EVT_UNLISTEN_REPLY:
906      vcl_session_unlisten_reply_handler (wrk, e->data);
907      break;
908    case SESSION_CTRL_EVT_MIGRATED:
909      vcl_session_migrated_handler (wrk, e->data);
910      break;
911    case SESSION_CTRL_EVT_CLEANUP:
912      vcl_session_cleanup_handler (wrk, e->data);
913      break;
914    case SESSION_CTRL_EVT_REQ_WORKER_UPDATE:
915      vcl_session_req_worker_update_handler (wrk, e->data);
916      break;
917    case SESSION_CTRL_EVT_WORKER_UPDATE_REPLY:
918      vcl_session_worker_update_reply_handler (wrk, e->data);
919      break;
920    case SESSION_CTRL_EVT_APP_ADD_SEGMENT:
921      vcl_session_app_add_segment_handler (wrk, e->data);
922      break;
923    case SESSION_CTRL_EVT_APP_DEL_SEGMENT:
924      vcl_session_app_del_segment_handler (wrk, e->data);
925      break;
926    default:
927      clib_warning ("unhandled %u", e->event_type);
928    }
929  return VPPCOM_OK;
930}
931
932static int
933vppcom_wait_for_session_state_change (u32 session_index,
934				      vcl_session_state_t state,
935				      f64 wait_for_time)
936{
937  vcl_worker_t *wrk = vcl_worker_get_current ();
938  f64 timeout = clib_time_now (&wrk->clib_time) + wait_for_time;
939  vcl_session_t *volatile session;
940  svm_msg_q_msg_t msg;
941  session_event_t *e;
942
943  do
944    {
945      session = vcl_session_get (wrk, session_index);
946      if (PREDICT_FALSE (!session))
947	{
948	  return VPPCOM_EBADFD;
949	}
950      if (session->session_state & state)
951	{
952	  return VPPCOM_OK;
953	}
954      if (session->session_state & STATE_DETACHED)
955	{
956	  return VPPCOM_ECONNREFUSED;
957	}
958
959      if (svm_msg_q_sub (wrk->app_event_queue, &msg, SVM_Q_NOWAIT, 0))
960	{
961	  usleep (100);
962	  continue;
963	}
964      e = svm_msg_q_msg_data (wrk->app_event_queue, &msg);
965      vcl_handle_mq_event (wrk, e);
966      svm_msg_q_free_msg (wrk->app_event_queue, &msg);
967    }
968  while (clib_time_now (&wrk->clib_time) < timeout);
969
970  VDBG (0, "timeout waiting for state 0x%x (%s)", state,
971	vppcom_session_state_str (state));
972  vcl_evt (VCL_EVT_SESSION_TIMEOUT, session, session_state);
973
974  return VPPCOM_ETIMEDOUT;
975}
976
977static void
978vcl_handle_pending_wrk_updates (vcl_worker_t * wrk)
979{
980  vcl_session_state_t state;
981  vcl_session_t *s;
982  u32 *sip;
983
984  if (PREDICT_TRUE (vec_len (wrk->pending_session_wrk_updates) == 0))
985    return;
986
987  vec_foreach (sip, wrk->pending_session_wrk_updates)
988  {
989    s = vcl_session_get (wrk, *sip);
990    vcl_send_session_worker_update (wrk, s, wrk->wrk_index);
991    state = s->session_state;
992    vppcom_wait_for_session_state_change (s->session_index, STATE_UPDATED, 5);
993    s->session_state = state;
994  }
995  vec_reset_length (wrk->pending_session_wrk_updates);
996}
997
998void
999vcl_flush_mq_events (void)
1000{
1001  vcl_worker_t *wrk = vcl_worker_get_current ();
1002  svm_msg_q_msg_t *msg;
1003  session_event_t *e;
1004  svm_msg_q_t *mq;
1005  int i;
1006
1007  mq = wrk->app_event_queue;
1008  svm_msg_q_lock (mq);
1009  vcl_mq_dequeue_batch (wrk, mq, ~0);
1010  svm_msg_q_unlock (mq);
1011
1012  for (i = 0; i < vec_len (wrk->mq_msg_vector); i++)
1013    {
1014      msg = vec_elt_at_index (wrk->mq_msg_vector, i);
1015      e = svm_msg_q_msg_data (mq, msg);
1016      vcl_handle_mq_event (wrk, e);
1017      svm_msg_q_free_msg (mq, msg);
1018    }
1019  vec_reset_length (wrk->mq_msg_vector);
1020  vcl_handle_pending_wrk_updates (wrk);
1021}
1022
1023static int
1024vppcom_app_session_enable (void)
1025{
1026  int rv;
1027
1028  if (vcm->app_state != STATE_APP_ENABLED)
1029    {
1030      vppcom_send_session_enable_disable (1 /* is_enabled == TRUE */ );
1031      rv = vcl_wait_for_app_state_change (STATE_APP_ENABLED);
1032      if (PREDICT_FALSE (rv))
1033	{
1034	  VDBG (0, "application session enable timed out! returning %d (%s)",
1035		rv, vppcom_retval_str (rv));
1036	  return rv;
1037	}
1038    }
1039  return VPPCOM_OK;
1040}
1041
1042static int
1043vppcom_app_attach (void)
1044{
1045  int rv;
1046
1047  vppcom_app_send_attach ();
1048  rv = vcl_wait_for_app_state_change (STATE_APP_ATTACHED);
1049  if (PREDICT_FALSE (rv))
1050    {
1051      VDBG (0, "application attach timed out! returning %d (%s)", rv,
1052	    vppcom_retval_str (rv));
1053      return rv;
1054    }
1055
1056  return VPPCOM_OK;
1057}
1058
1059static int
1060vppcom_session_unbind (u32 session_handle)
1061{
1062  vcl_worker_t *wrk = vcl_worker_get_current ();
1063  session_accepted_msg_t *accepted_msg;
1064  vcl_session_t *session = 0;
1065  vcl_session_msg_t *evt;
1066
1067  session = vcl_session_get_w_handle (wrk, session_handle);
1068  if (!session)
1069    return VPPCOM_EBADFD;
1070
1071  /* Flush pending accept events, if any */
1072  while (clib_fifo_elts (session->accept_evts_fifo))
1073    {
1074      clib_fifo_sub2 (session->accept_evts_fifo, evt);
1075      accepted_msg = &evt->accepted_msg;
1076      vcl_session_table_del_vpp_handle (wrk, accepted_msg->handle);
1077      vcl_send_session_accepted_reply (session->vpp_evt_q,
1078				       accepted_msg->context,
1079				       session->vpp_handle, -1);
1080    }
1081  clib_fifo_free (session->accept_evts_fifo);
1082
1083  vcl_send_session_unlisten (wrk, session);
1084
1085  VDBG (1, "session %u [0x%llx]: sending unbind!", session->session_index,
1086	session->vpp_handle);
1087  vcl_evt (VCL_EVT_UNBIND, session);
1088
1089  session->vpp_handle = ~0;
1090  session->session_state = STATE_DISCONNECT;
1091
1092  return VPPCOM_OK;
1093}
1094
1095static int
1096vppcom_session_disconnect (u32 session_handle)
1097{
1098  vcl_worker_t *wrk = vcl_worker_get_current ();
1099  svm_msg_q_t *vpp_evt_q;
1100  vcl_session_t *session, *listen_session;
1101  vcl_session_state_t state;
1102  u64 vpp_handle;
1103
1104  session = vcl_session_get_w_handle (wrk, session_handle);
1105  if (!session)
1106    return VPPCOM_EBADFD;
1107
1108  vpp_handle = session->vpp_handle;
1109  state = session->session_state;
1110
1111  VDBG (1, "session %u [0x%llx] state 0x%x (%s)", session->session_index,
1112	vpp_handle, state, vppcom_session_state_str (state));
1113
1114  if (PREDICT_FALSE (state & STATE_LISTEN))
1115    {
1116      VDBG (0, "ERROR: Cannot disconnect a listen socket!");
1117      return VPPCOM_EBADFD;
1118    }
1119
1120  if (state & STATE_VPP_CLOSING)
1121    {
1122      vpp_evt_q = vcl_session_vpp_evt_q (wrk, session);
1123      vcl_send_session_disconnected_reply (vpp_evt_q, wrk->my_client_index,
1124					   vpp_handle, 0);
1125      VDBG (1, "session %u [0x%llx]: sending disconnect REPLY...",
1126	    session->session_index, vpp_handle);
1127    }
1128  else
1129    {
1130      VDBG (1, "session %u [0x%llx]: sending disconnect...",
1131	    session->session_index, vpp_handle);
1132      vcl_send_session_disconnect (wrk, session);
1133    }
1134
1135  if (session->listener_index != VCL_INVALID_SESSION_INDEX)
1136    {
1137      listen_session = vcl_session_get (wrk, session->listener_index);
1138      listen_session->n_accepted_sessions--;
1139    }
1140
1141  return VPPCOM_OK;
1142}
1143
1144/**
1145 * Handle app exit
1146 *
1147 * Notify vpp of the disconnect and mark the worker as free. If we're the
1148 * last worker, do a full cleanup otherwise, since we're probably a forked
1149 * child, avoid syscalls as much as possible. We might've lost privileges.
1150 */
1151void
1152vppcom_app_exit (void)
1153{
1154  if (!pool_elts (vcm->workers))
1155    return;
1156  vcl_worker_cleanup (vcl_worker_get_current (), 1 /* notify vpp */ );
1157  vcl_set_worker_index (~0);
1158  vcl_elog_stop (vcm);
1159}
1160
1161/*
1162 * VPPCOM Public API functions
1163 */
1164int
1165vppcom_app_create (char *app_name)
1166{
1167  vppcom_cfg_t *vcl_cfg = &vcm->cfg;
1168  int rv;
1169
1170  if (vcm->is_init)
1171    {
1172      VDBG (1, "already initialized");
1173      return VPPCOM_EEXIST;
1174    }
1175
1176  vcm->is_init = 1;
1177  vppcom_cfg (&vcm->cfg);
1178  vcl_cfg = &vcm->cfg;
1179
1180  vcm->main_cpu = pthread_self ();
1181  vcm->main_pid = getpid ();
1182  vcm->app_name = format (0, "%s", app_name);
1183  vppcom_init_error_string_table ();
1184  fifo_segment_main_init (&vcm->segment_main, vcl_cfg->segment_baseva,
1185			  20 /* timeout in secs */ );
1186  pool_alloc (vcm->workers, vcl_cfg->max_workers);
1187  clib_spinlock_init (&vcm->workers_lock);
1188  clib_rwlock_init (&vcm->segment_table_lock);
1189  atexit (vppcom_app_exit);
1190
1191  /* Allocate default worker */
1192  vcl_worker_alloc_and_init ();
1193
1194  /* API hookup and connect to VPP */
1195  vcl_elog_init (vcm);
1196  vcm->app_state = STATE_APP_START;
1197  rv = vppcom_connect_to_vpp (app_name);
1198  if (rv)
1199    {
1200      VERR ("couldn't connect to VPP!");
1201      return rv;
1202    }
1203  VDBG (0, "sending session enable");
1204  rv = vppcom_app_session_enable ();
1205  if (rv)
1206    {
1207      VERR ("vppcom_app_session_enable() failed!");
1208      return rv;
1209    }
1210
1211  VDBG (0, "sending app attach");
1212  rv = vppcom_app_attach ();
1213  if (rv)
1214    {
1215      VERR ("vppcom_app_attach() failed!");
1216      return rv;
1217    }
1218
1219  VDBG (0, "app_name '%s', my_client_index %d (0x%x)", app_name,
1220	vcm->workers[0].my_client_index, vcm->workers[0].my_client_index);
1221
1222  return VPPCOM_OK;
1223}
1224
1225void
1226vppcom_app_destroy (void)
1227{
1228  vcl_worker_t *wrk, *current_wrk;
1229  struct dlmallinfo mi;
1230  mspace heap;
1231
1232  if (!pool_elts (vcm->workers))
1233    return;
1234
1235  vcl_evt (VCL_EVT_DETACH, vcm);
1236
1237  current_wrk = vcl_worker_get_current ();
1238
1239  /* *INDENT-OFF* */
1240  pool_foreach (wrk, vcm->workers, ({
1241    if (current_wrk != wrk)
1242      vcl_worker_cleanup (wrk, 0 /* notify vpp */ );
1243  }));
1244  /* *INDENT-ON* */
1245
1246  vcl_send_app_detach (current_wrk);
1247  vppcom_disconnect_from_vpp ();
1248  vcl_worker_cleanup (current_wrk, 0 /* notify vpp */ );
1249
1250  vcl_elog_stop (vcm);
1251
1252  /*
1253   * Free the heap and fix vcm
1254   */
1255  heap = clib_mem_get_heap ();
1256  mi = mspace_mallinfo (heap);
1257  munmap (mspace_least_addr (heap), mi.arena);
1258
1259  vcm = &_vppcom_main;
1260  vcm->is_init = 0;
1261}
1262
1263int
1264vppcom_session_create (u8 proto, u8 is_nonblocking)
1265{
1266  vcl_worker_t *wrk = vcl_worker_get_current ();
1267  vcl_session_t *session;
1268
1269  session = vcl_session_alloc (wrk);
1270
1271  session->session_type = proto;
1272  session->session_state = STATE_CLOSED;
1273  session->vpp_handle = ~0;
1274  session->is_dgram = vcl_proto_is_dgram (proto);
1275
1276  if (is_nonblocking)
1277    VCL_SESS_ATTR_SET (session->attr, VCL_SESS_ATTR_NONBLOCK);
1278
1279  vcl_evt (VCL_EVT_CREATE, session, session_type, session->session_state,
1280	   is_nonblocking, session_index);
1281
1282  VDBG (0, "created session %u", session->session_index);
1283
1284  return vcl_session_handle (session);
1285}
1286
1287int
1288vcl_session_cleanup (vcl_worker_t * wrk, vcl_session_t * session,
1289		     vcl_session_handle_t sh, u8 do_disconnect)
1290{
1291  vcl_session_state_t state;
1292  u32 next_sh, vep_sh;
1293  int rv = VPPCOM_OK;
1294  u64 vpp_handle;
1295  u8 is_vep;
1296
1297  is_vep = session->is_vep;
1298  next_sh = session->vep.next_sh;
1299  vep_sh = session->vep.vep_sh;
1300  state = session->session_state;
1301  vpp_handle = session->vpp_handle;
1302
1303  VDBG (1, "session %u [0x%llx] closing", session->session_index, vpp_handle);
1304
1305  if (is_vep)
1306    {
1307      while (next_sh != ~0)
1308	{
1309	  rv = vppcom_epoll_ctl (sh, EPOLL_CTL_DEL, next_sh, 0);
1310	  if (PREDICT_FALSE (rv < 0))
1311	    VDBG (0, "vpp handle 0x%llx, sh %u: EPOLL_CTL_DEL vep_idx %u"
1312		  " failed! rv %d (%s)", vpp_handle, next_sh, vep_sh, rv,
1313		  vppcom_retval_str (rv));
1314
1315	  next_sh = session->vep.next_sh;
1316	}
1317      goto cleanup;
1318    }
1319
1320  if (session->is_vep_session)
1321    {
1322      rv = vppcom_epoll_ctl (vep_sh, EPOLL_CTL_DEL, sh, 0);
1323      if (rv < 0)
1324	VDBG (0, "session %u [0x%llx]: EPOLL_CTL_DEL vep_idx %u "
1325	      "failed! rv %d (%s)", session->session_index, vpp_handle,
1326	      vep_sh, rv, vppcom_retval_str (rv));
1327    }
1328
1329  if (!do_disconnect)
1330    {
1331      VDBG (1, "session %u [0x%llx] disconnect skipped",
1332	    session->session_index, vpp_handle);
1333      goto cleanup;
1334    }
1335
1336  if (state & STATE_LISTEN)
1337    {
1338      rv = vppcom_session_unbind (sh);
1339      if (PREDICT_FALSE (rv < 0))
1340	VDBG (0, "session %u [0x%llx]: listener unbind failed! "
1341	      "rv %d (%s)", session->session_index, vpp_handle, rv,
1342	      vppcom_retval_str (rv));
1343      return rv;
1344    }
1345  else if ((state & STATE_OPEN)
1346	   || (vcl_session_is_connectable_listener (wrk, session)))
1347    {
1348      rv = vppcom_session_disconnect (sh);
1349      if (PREDICT_FALSE (rv < 0))
1350	VDBG (0, "ERROR: session %u [0x%llx]: disconnect failed!"
1351	      " rv %d (%s)", session->session_index, vpp_handle,
1352	      rv, vppcom_retval_str (rv));
1353    }
1354  else if (state == STATE_DISCONNECT)
1355    {
1356      svm_msg_q_t *mq = vcl_session_vpp_evt_q (wrk, session);
1357      vcl_send_session_reset_reply (mq, wrk->my_client_index,
1358				    session->vpp_handle, 0);
1359    }
1360  else if (state == STATE_DETACHED)
1361    {
1362      /* Should not happen. VPP cleaned up before app confirmed close */
1363      VDBG (0, "vpp freed session %d before close", session->session_index);
1364      goto free_session;
1365    }
1366
1367  session->session_state = STATE_CLOSED;
1368
1369  /* Session is removed only after vpp confirms the disconnect */
1370  return rv;
1371
1372cleanup:
1373  vcl_session_table_del_vpp_handle (wrk, vpp_handle);
1374free_session:
1375  vcl_session_free (wrk, session);
1376  vcl_evt (VCL_EVT_CLOSE, session, rv);
1377
1378  return rv;
1379}
1380
1381int
1382vppcom_session_close (uint32_t session_handle)
1383{
1384  vcl_worker_t *wrk = vcl_worker_get_current ();
1385  vcl_session_t *session;
1386
1387  session = vcl_session_get_w_handle (wrk, session_handle);
1388  if (!session)
1389    return VPPCOM_EBADFD;
1390  return vcl_session_cleanup (wrk, session, session_handle,
1391			      1 /* do_disconnect */ );
1392}
1393
1394int
1395vppcom_session_bind (uint32_t session_handle, vppcom_endpt_t * ep)
1396{
1397  vcl_worker_t *wrk = vcl_worker_get_current ();
1398  vcl_session_t *session = 0;
1399
1400  if (!ep || !ep->ip)
1401    return VPPCOM_EINVAL;
1402
1403  session = vcl_session_get_w_handle (wrk, session_handle);
1404  if (!session)
1405    return VPPCOM_EBADFD;
1406
1407  if (session->is_vep)
1408    {
1409      VDBG (0, "ERROR: cannot bind to epoll session %u!",
1410	    session->session_index);
1411      return VPPCOM_EBADFD;
1412    }
1413
1414  session->transport.is_ip4 = ep->is_ip4;
1415  if (ep->is_ip4)
1416    clib_memcpy_fast (&session->transport.lcl_ip.ip4, ep->ip,
1417		      sizeof (ip4_address_t));
1418  else
1419    clib_memcpy_fast (&session->transport.lcl_ip.ip6, ep->ip,
1420		      sizeof (ip6_address_t));
1421  session->transport.lcl_port = ep->port;
1422
1423  VDBG (0, "session %u handle %u: binding to local %s address %U port %u, "
1424	"proto %s", session->session_index, session_handle,
1425	session->transport.is_ip4 ? "IPv4" : "IPv6",
1426	format_ip46_address, &session->transport.lcl_ip,
1427	session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
1428	clib_net_to_host_u16 (session->transport.lcl_port),
1429	vppcom_proto_str (session->session_type));
1430  vcl_evt (VCL_EVT_BIND, session);
1431
1432  if (session->session_type == VPPCOM_PROTO_UDP)
1433    vppcom_session_listen (session_handle, 10);
1434
1435  return VPPCOM_OK;
1436}
1437
1438int
1439vppcom_session_listen (uint32_t listen_sh, uint32_t q_len)
1440{
1441  vcl_worker_t *wrk = vcl_worker_get_current ();
1442  vcl_session_t *listen_session = 0;
1443  u64 listen_vpp_handle;
1444  int rv;
1445
1446  listen_session = vcl_session_get_w_handle (wrk, listen_sh);
1447  if (!listen_session || listen_session->is_vep)
1448    return VPPCOM_EBADFD;
1449
1450  if (q_len == 0 || q_len == ~0)
1451    q_len = vcm->cfg.listen_queue_size;
1452
1453  listen_vpp_handle = listen_session->vpp_handle;
1454  if (listen_session->session_state & STATE_LISTEN)
1455    {
1456      VDBG (0, "session %u [0x%llx]: already in listen state!",
1457	    listen_sh, listen_vpp_handle);
1458      return VPPCOM_OK;
1459    }
1460
1461  VDBG (0, "session %u: sending vpp listen request...", listen_sh);
1462
1463  /*
1464   * Send listen request to vpp and wait for reply
1465   */
1466  vcl_send_session_listen (wrk, listen_session);
1467  rv = vppcom_wait_for_session_state_change (listen_session->session_index,
1468					     STATE_LISTEN,
1469					     vcm->cfg.session_timeout);
1470
1471  if (PREDICT_FALSE (rv))
1472    {
1473      listen_session = vcl_session_get_w_handle (wrk, listen_sh);
1474      VDBG (0, "session %u [0x%llx]: listen failed! returning %d (%s)",
1475	    listen_sh, listen_session->vpp_handle, rv,
1476	    vppcom_retval_str (rv));
1477      return rv;
1478    }
1479
1480  return VPPCOM_OK;
1481}
1482
1483int
1484vppcom_session_tls_add_cert (uint32_t session_handle, char *cert,
1485			     uint32_t cert_len)
1486{
1487
1488  vcl_worker_t *wrk = vcl_worker_get_current ();
1489  vcl_session_t *session = 0;
1490
1491  session = vcl_session_get_w_handle (wrk, session_handle);
1492  if (!session)
1493    return VPPCOM_EBADFD;
1494
1495  if (cert_len == 0 || cert_len == ~0)
1496    return VPPCOM_EBADFD;
1497
1498  /*
1499   * Send listen request to vpp and wait for reply
1500   */
1501  vppcom_send_application_tls_cert_add (session, cert, cert_len);
1502  vcm->app_state = STATE_APP_ADDING_TLS_DATA;
1503  vcl_wait_for_app_state_change (STATE_APP_READY);
1504  return VPPCOM_OK;
1505
1506}
1507
1508int
1509vppcom_session_tls_add_key (uint32_t session_handle, char *key,
1510			    uint32_t key_len)
1511{
1512
1513  vcl_worker_t *wrk = vcl_worker_get_current ();
1514  vcl_session_t *session = 0;
1515
1516  session = vcl_session_get_w_handle (wrk, session_handle);
1517  if (!session)
1518    return VPPCOM_EBADFD;
1519
1520  if (key_len == 0 || key_len == ~0)
1521    return VPPCOM_EBADFD;
1522
1523  vppcom_send_application_tls_key_add (session, key, key_len);
1524  vcm->app_state = STATE_APP_ADDING_TLS_DATA;
1525  vcl_wait_for_app_state_change (STATE_APP_READY);
1526  return VPPCOM_OK;
1527}
1528
1529static int
1530validate_args_session_accept_ (vcl_worker_t * wrk, vcl_session_t * ls)
1531{
1532  if (ls->is_vep)
1533    {
1534      VDBG (0, "ERROR: cannot accept on epoll session %u!",
1535	    ls->session_index);
1536      return VPPCOM_EBADFD;
1537    }
1538
1539  if ((ls->session_state != STATE_LISTEN)
1540      && (!vcl_session_is_connectable_listener (wrk, ls)))
1541    {
1542      VDBG (0,
1543	    "ERROR: session [0x%llx]: not in listen state! state 0x%x"
1544	    " (%s)", ls->vpp_handle, ls->session_state,
1545	    vppcom_session_state_str (ls->session_state));
1546      return VPPCOM_EBADFD;
1547    }
1548  return VPPCOM_OK;
1549}
1550
1551int
1552vppcom_unformat_proto (uint8_t * proto, char *proto_str)
1553{
1554  if (!strcmp (proto_str, "TCP"))
1555    *proto = VPPCOM_PROTO_TCP;
1556  else if (!strcmp (proto_str, "tcp"))
1557    *proto = VPPCOM_PROTO_TCP;
1558  else if (!strcmp (proto_str, "UDP"))
1559    *proto = VPPCOM_PROTO_UDP;
1560  else if (!strcmp (proto_str, "udp"))
1561    *proto = VPPCOM_PROTO_UDP;
1562  else if (!strcmp (proto_str, "TLS"))
1563    *proto = VPPCOM_PROTO_TLS;
1564  else if (!strcmp (proto_str, "tls"))
1565    *proto = VPPCOM_PROTO_TLS;
1566  else if (!strcmp (proto_str, "QUIC"))
1567    *proto = VPPCOM_PROTO_QUIC;
1568  else if (!strcmp (proto_str, "quic"))
1569    *proto = VPPCOM_PROTO_QUIC;
1570  else
1571    return 1;
1572  return 0;
1573}
1574
1575int
1576vppcom_session_accept (uint32_t listen_session_handle, vppcom_endpt_t * ep,
1577		       uint32_t flags)
1578{
1579  u32 client_session_index = ~0, listen_session_index, accept_flags = 0;
1580  vcl_worker_t *wrk = vcl_worker_get_current ();
1581  session_accepted_msg_t accepted_msg;
1582  vcl_session_t *listen_session = 0;
1583  vcl_session_t *client_session = 0;
1584  vcl_session_msg_t *evt;
1585  svm_msg_q_msg_t msg;
1586  session_event_t *e;
1587  u8 is_nonblocking;
1588  int rv;
1589
1590  listen_session = vcl_session_get_w_handle (wrk, listen_session_handle);
1591  if (!listen_session)
1592    return VPPCOM_EBADFD;
1593
1594  listen_session_index = listen_session->session_index;
1595  if ((rv = validate_args_session_accept_ (wrk, listen_session)))
1596    return rv;
1597
1598  if (clib_fifo_elts (listen_session->accept_evts_fifo))
1599    {
1600      clib_fifo_sub2 (listen_session->accept_evts_fifo, evt);
1601      accept_flags = evt->flags;
1602      accepted_msg = evt->accepted_msg;
1603      goto handle;
1604    }
1605
1606  is_nonblocking = VCL_SESS_ATTR_TEST (listen_session->attr,
1607				       VCL_SESS_ATTR_NONBLOCK);
1608  while (1)
1609    {
1610      if (svm_msg_q_is_empty (wrk->app_event_queue) && is_nonblocking)
1611	return VPPCOM_EAGAIN;
1612
1613      if (svm_msg_q_sub (wrk->app_event_queue, &msg, SVM_Q_WAIT, 0))
1614	return VPPCOM_EAGAIN;
1615
1616      e = svm_msg_q_msg_data (wrk->app_event_queue, &msg);
1617      if (e->event_type != SESSION_CTRL_EVT_ACCEPTED)
1618	{
1619	  vcl_handle_mq_event (wrk, e);
1620	  svm_msg_q_free_msg (wrk->app_event_queue, &msg);
1621	  continue;
1622	}
1623      clib_memcpy_fast (&accepted_msg, e->data, sizeof (accepted_msg));
1624      svm_msg_q_free_msg (wrk->app_event_queue, &msg);
1625      break;
1626    }
1627
1628handle:
1629
1630  client_session_index = vcl_session_accepted_handler (wrk, &accepted_msg,
1631						       listen_session_index);
1632  if (client_session_index == VCL_INVALID_SESSION_INDEX)
1633    return VPPCOM_ECONNABORTED;
1634
1635  listen_session = vcl_session_get (wrk, listen_session_index);
1636  client_session = vcl_session_get (wrk, client_session_index);
1637
1638  if (flags & O_NONBLOCK)
1639    VCL_SESS_ATTR_SET (client_session->attr, VCL_SESS_ATTR_NONBLOCK);
1640
1641  VDBG (1, "listener %u [0x%llx]: Got a connect request! session %u [0x%llx],"
1642	" flags %d, is_nonblocking %u", listen_session->session_index,
1643	listen_session->vpp_handle, client_session_index,
1644	client_session->vpp_handle, flags,
1645	VCL_SESS_ATTR_TEST (client_session->attr, VCL_SESS_ATTR_NONBLOCK));
1646
1647  if (ep)
1648    {
1649      ep->is_ip4 = client_session->transport.is_ip4;
1650      ep->port = client_session->transport.rmt_port;
1651      if (client_session->transport.is_ip4)
1652	clib_memcpy_fast (ep->ip, &client_session->transport.rmt_ip.ip4,
1653			  sizeof (ip4_address_t));
1654      else
1655	clib_memcpy_fast (ep->ip, &client_session->transport.rmt_ip.ip6,
1656			  sizeof (ip6_address_t));
1657    }
1658
1659  VDBG (0, "listener %u [0x%llx] accepted %u [0x%llx] peer: %U:%u "
1660	"local: %U:%u", listen_session_handle, listen_session->vpp_handle,
1661	client_session_index, client_session->vpp_handle,
1662	format_ip46_address, &client_session->transport.rmt_ip,
1663	client_session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
1664	clib_net_to_host_u16 (client_session->transport.rmt_port),
1665	format_ip46_address, &client_session->transport.lcl_ip,
1666	client_session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
1667	clib_net_to_host_u16 (client_session->transport.lcl_port));
1668  vcl_evt (VCL_EVT_ACCEPT, client_session, listen_session,
1669	   client_session_index);
1670
1671  /*
1672   * Session might have been closed already
1673   */
1674  if (accept_flags)
1675    {
1676      if (accept_flags & VCL_ACCEPTED_F_CLOSED)
1677	client_session->session_state = STATE_VPP_CLOSING;
1678      else if (accept_flags & VCL_ACCEPTED_F_RESET)
1679	client_session->session_state = STATE_DISCONNECT;
1680    }
1681  return vcl_session_handle (client_session);
1682}
1683
1684int
1685vppcom_session_connect (uint32_t session_handle, vppcom_endpt_t * server_ep)
1686{
1687  vcl_worker_t *wrk = vcl_worker_get_current ();
1688  vcl_session_t *session = 0;
1689  u32 session_index;
1690  int rv;
1691
1692  session = vcl_session_get_w_handle (wrk, session_handle);
1693  if (!session)
1694    return VPPCOM_EBADFD;
1695  session_index = session->session_index;
1696
1697  if (PREDICT_FALSE (session->is_vep))
1698    {
1699      VDBG (0, "ERROR: cannot connect epoll session %u!",
1700	    session->session_index);
1701      return VPPCOM_EBADFD;
1702    }
1703
1704  if (PREDICT_FALSE (session->session_state & CLIENT_STATE_OPEN))
1705    {
1706      VDBG (0, "session handle %u [0x%llx]: session already "
1707	    "connected to %s %U port %d proto %s, state 0x%x (%s)",
1708	    session_handle, session->vpp_handle,
1709	    session->transport.is_ip4 ? "IPv4" : "IPv6", format_ip46_address,
1710	    &session->transport.rmt_ip, session->transport.is_ip4 ?
1711	    IP46_TYPE_IP4 : IP46_TYPE_IP6,
1712	    clib_net_to_host_u16 (session->transport.rmt_port),
1713	    vppcom_proto_str (session->session_type), session->session_state,
1714	    vppcom_session_state_str (session->session_state));
1715      return VPPCOM_OK;
1716    }
1717
1718  /* Attempt to connect a connectionless listener */
1719  if (PREDICT_FALSE (session->session_state & STATE_LISTEN))
1720    {
1721      if (session->session_type != VPPCOM_PROTO_UDP)
1722	return VPPCOM_EINVAL;
1723      vcl_send_session_unlisten (wrk, session);
1724      session->session_state = STATE_CLOSED;
1725    }
1726
1727  session->transport.is_ip4 = server_ep->is_ip4;
1728  vcl_ip_copy_from_ep (&session->transport.rmt_ip, server_ep);
1729  session->transport.rmt_port = server_ep->port;
1730  session->parent_handle = VCL_INVALID_SESSION_HANDLE;
1731  session->flags |= VCL_SESSION_F_CONNECTED;
1732
1733  VDBG (0, "session handle %u (%s): connecting to peer %s %U "
1734	"port %d proto %s", session_handle,
1735	vppcom_session_state_str (session->session_state),
1736	session->transport.is_ip4 ? "IPv4" : "IPv6",
1737	format_ip46_address,
1738	&session->transport.rmt_ip, session->transport.is_ip4 ?
1739	IP46_TYPE_IP4 : IP46_TYPE_IP6,
1740	clib_net_to_host_u16 (session->transport.rmt_port),
1741	vppcom_proto_str (session->session_type));
1742
1743  vcl_send_session_connect (wrk, session);
1744
1745  if (VCL_SESS_ATTR_TEST (session->attr, VCL_SESS_ATTR_NONBLOCK))
1746    return VPPCOM_EINPROGRESS;
1747
1748  /*
1749   * Wait for reply from vpp if blocking
1750   */
1751  rv = vppcom_wait_for_session_state_change (session_index, STATE_CONNECT,
1752					     vcm->cfg.session_timeout);
1753
1754  session = vcl_session_get (wrk, session_index);
1755  VDBG (0, "session %u [0x%llx]: connect %s!", session->session_index,
1756	session->vpp_handle, rv ? "failed" : "succeeded");
1757
1758  return rv;
1759}
1760
1761int
1762vppcom_session_stream_connect (uint32_t session_handle,
1763			       uint32_t parent_session_handle)
1764{
1765  vcl_worker_t *wrk = vcl_worker_get_current ();
1766  vcl_session_t *session, *parent_session;
1767  u32 session_index, parent_session_index;
1768  int rv;
1769
1770  session = vcl_session_get_w_handle (wrk, session_handle);
1771  if (!session)
1772    return VPPCOM_EBADFD;
1773  parent_session = vcl_session_get_w_handle (wrk, parent_session_handle);
1774  if (!parent_session)
1775    return VPPCOM_EBADFD;
1776
1777  session_index = session->session_index;
1778  parent_session_index = parent_session->session_index;
1779  if (PREDICT_FALSE (session->is_vep))
1780    {
1781      VDBG (0, "ERROR: cannot connect epoll session %u!",
1782	    session->session_index);
1783      return VPPCOM_EBADFD;
1784    }
1785
1786  if (PREDICT_FALSE (session->session_state & CLIENT_STATE_OPEN))
1787    {
1788      VDBG (0, "session handle %u [0x%llx]: session already "
1789	    "connected to session %u [0x%llx] proto %s, state 0x%x (%s)",
1790	    session_handle, session->vpp_handle,
1791	    parent_session_handle, parent_session->vpp_handle,
1792	    vppcom_proto_str (session->session_type), session->session_state,
1793	    vppcom_session_state_str (session->session_state));
1794      return VPPCOM_OK;
1795    }
1796
1797  /* Connect to quic session specifics */
1798  session->transport.is_ip4 = parent_session->transport.is_ip4;
1799  session->transport.rmt_ip.ip4.as_u32 = (uint32_t) 1;
1800  session->transport.rmt_port = 0;
1801  session->parent_handle = parent_session->vpp_handle;
1802
1803  VDBG (0, "session handle %u: connecting to session %u [0x%llx]",
1804	session_handle, parent_session_handle, parent_session->vpp_handle);
1805
1806  /*
1807   * Send connect request and wait for reply from vpp
1808   */
1809  vcl_send_session_connect (wrk, session);
1810  rv = vppcom_wait_for_session_state_change (session_index, STATE_CONNECT,
1811					     vcm->cfg.session_timeout);
1812
1813  session->listener_index = parent_session_index;
1814  parent_session = vcl_session_get_w_handle (wrk, parent_session_handle);
1815  if (parent_session)
1816    parent_session->n_accepted_sessions++;
1817
1818  session = vcl_session_get (wrk, session_index);
1819  VDBG (0, "session %u [0x%llx]: connect %s!", session->session_index,
1820	session->vpp_handle, rv ? "failed" : "succeeded");
1821
1822  return rv;
1823}
1824
1825static u8
1826vcl_is_rx_evt_for_session (session_event_t * e, u32 sid, u8 is_ct)
1827{
1828  return (e->event_type == SESSION_IO_EVT_RX && e->session_index == sid);
1829}
1830
1831static inline int
1832vppcom_session_read_internal (uint32_t session_handle, void *buf, int n,
1833			      u8 peek)
1834{
1835  vcl_worker_t *wrk = vcl_worker_get_current ();
1836  int n_read = 0, is_nonblocking;
1837  vcl_session_t *s = 0;
1838  svm_fifo_t *rx_fifo;
1839  svm_msg_q_msg_t msg;
1840  session_event_t *e;
1841  svm_msg_q_t *mq;
1842  u8 is_ct;
1843
1844  if (PREDICT_FALSE (!buf))
1845    return VPPCOM_EINVAL;
1846
1847  s = vcl_session_get_w_handle (wrk, session_handle);
1848  if (PREDICT_FALSE (!s || s->is_vep))
1849    return VPPCOM_EBADFD;
1850
1851  if (PREDICT_FALSE (!vcl_session_is_open (s)))
1852    {
1853      VDBG (0, "session %u[0x%llx] is not open! state 0x%x (%s)",
1854	    s->session_index, s->vpp_handle, s->session_state,
1855	    vppcom_session_state_str (s->session_state));
1856      return vcl_session_closed_error (s);
1857    }
1858
1859  is_nonblocking = VCL_SESS_ATTR_TEST (s->attr, VCL_SESS_ATTR_NONBLOCK);
1860  is_ct = vcl_session_is_ct (s);
1861  mq = wrk->app_event_queue;
1862  rx_fifo = is_ct ? s->ct_rx_fifo : s->rx_fifo;
1863  s->has_rx_evt = 0;
1864
1865  if (svm_fifo_is_empty_cons (rx_fifo))
1866    {
1867      if (is_nonblocking)
1868	{
1869	  if (vcl_session_is_closing (s))
1870	    return vcl_session_closing_error (s);
1871	  svm_fifo_unset_event (s->rx_fifo);
1872	  return VPPCOM_EWOULDBLOCK;
1873	}
1874      while (svm_fifo_is_empty_cons (rx_fifo))
1875	{
1876	  if (vcl_session_is_closing (s))
1877	    return vcl_session_closing_error (s);
1878
1879	  svm_fifo_unset_event (s->rx_fifo);
1880	  svm_msg_q_lock (mq);
1881	  if (svm_msg_q_is_empty (mq))
1882	    svm_msg_q_wait (mq);
1883
1884	  svm_msg_q_sub_w_lock (mq, &msg);
1885	  e = svm_msg_q_msg_data (mq, &msg);
1886	  svm_msg_q_unlock (mq);
1887	  if (!vcl_is_rx_evt_for_session (e, s->session_index, is_ct))
1888	    vcl_handle_mq_event (wrk, e);
1889	  svm_msg_q_free_msg (mq, &msg);
1890	}
1891    }
1892
1893  if (s->is_dgram)
1894    n_read = app_recv_dgram_raw (rx_fifo, buf, n, &s->transport, 0, peek);
1895  else
1896    n_read = app_recv_stream_raw (rx_fifo, buf, n, 0, peek);
1897
1898  if (svm_fifo_is_empty_cons (rx_fifo))
1899    svm_fifo_unset_event (s->rx_fifo);
1900
1901  /* Cut-through sessions might request tx notifications on rx fifos */
1902  if (PREDICT_FALSE (rx_fifo->want_deq_ntf))
1903    {
1904      app_send_io_evt_to_vpp (s->vpp_evt_q, s->rx_fifo->master_session_index,
1905			      SESSION_IO_EVT_RX, SVM_Q_WAIT);
1906      svm_fifo_reset_has_deq_ntf (s->rx_fifo);
1907    }
1908
1909  VDBG (2, "session %u[0x%llx]: read %d bytes from (%p)", s->session_index,
1910	s->vpp_handle, n_read, rx_fifo);
1911
1912  return n_read;
1913}
1914
1915int
1916vppcom_session_read (uint32_t session_handle, void *buf, size_t n)
1917{
1918  return (vppcom_session_read_internal (session_handle, buf, n, 0));
1919}
1920
1921static int
1922vppcom_session_peek (uint32_t session_handle, void *buf, int n)
1923{
1924  return (vppcom_session_read_internal (session_handle, buf, n, 1));
1925}
1926
1927int
1928vppcom_session_read_segments (uint32_t session_handle,
1929			      vppcom_data_segments_t ds)
1930{
1931  vcl_worker_t *wrk = vcl_worker_get_current ();
1932  int n_read = 0, is_nonblocking;
1933  vcl_session_t *s = 0;
1934  svm_fifo_t *rx_fifo;
1935  svm_msg_q_msg_t msg;
1936  session_event_t *e;
1937  svm_msg_q_t *mq;
1938  u8 is_ct;
1939
1940  s = vcl_session_get_w_handle (wrk, session_handle);
1941  if (PREDICT_FALSE (!s || s->is_vep))
1942    return VPPCOM_EBADFD;
1943
1944  if (PREDICT_FALSE (!vcl_session_is_open (s)))
1945    return vcl_session_closed_error (s);
1946
1947  is_nonblocking = VCL_SESS_ATTR_TEST (s->attr, VCL_SESS_ATTR_NONBLOCK);
1948  is_ct = vcl_session_is_ct (s);
1949  mq = is_ct ? s->our_evt_q : wrk->app_event_queue;
1950  rx_fifo = s->rx_fifo;
1951  s->has_rx_evt = 0;
1952
1953  if (is_ct)
1954    svm_fifo_unset_event (s->rx_fifo);
1955
1956  if (svm_fifo_is_empty_cons (rx_fifo))
1957    {
1958      if (is_nonblocking)
1959	{
1960	  svm_fifo_unset_event (rx_fifo);
1961	  return VPPCOM_EWOULDBLOCK;
1962	}
1963      while (svm_fifo_is_empty_cons (rx_fifo))
1964	{
1965	  if (vcl_session_is_closing (s))
1966	    return vcl_session_closing_error (s);
1967
1968	  svm_fifo_unset_event (rx_fifo);
1969	  svm_msg_q_lock (mq);
1970	  if (svm_msg_q_is_empty (mq))
1971	    svm_msg_q_wait (mq);
1972
1973	  svm_msg_q_sub_w_lock (mq, &msg);
1974	  e = svm_msg_q_msg_data (mq, &msg);
1975	  svm_msg_q_unlock (mq);
1976	  if (!vcl_is_rx_evt_for_session (e, s->session_index, is_ct))
1977	    vcl_handle_mq_event (wrk, e);
1978	  svm_msg_q_free_msg (mq, &msg);
1979	}
1980    }
1981
1982  n_read = svm_fifo_segments (rx_fifo, (svm_fifo_seg_t *) ds);
1983  svm_fifo_unset_event (rx_fifo);
1984
1985  return n_read;
1986}
1987
1988void
1989vppcom_session_free_segments (uint32_t session_handle,
1990			      vppcom_data_segments_t ds)
1991{
1992  vcl_worker_t *wrk = vcl_worker_get_current ();
1993  vcl_session_t *s;
1994
1995  s = vcl_session_get_w_handle (wrk, session_handle);
1996  if (PREDICT_FALSE (!s || s->is_vep))
1997    return;
1998
1999  svm_fifo_segments_free (s->rx_fifo, (svm_fifo_seg_t *) ds);
2000}
2001
2002int
2003vppcom_data_segment_copy (void *buf, vppcom_data_segments_t ds, u32 max_bytes)
2004{
2005  u32 first_copy = clib_min (ds[0].len, max_bytes);
2006  clib_memcpy_fast (buf, ds[0].data, first_copy);
2007  if (first_copy < max_bytes)
2008    {
2009      clib_memcpy_fast (buf + first_copy, ds[1].data,
2010			clib_min (ds[1].len, max_bytes - first_copy));
2011    }
2012  return 0;
2013}
2014
2015static u8
2016vcl_is_tx_evt_for_session (session_event_t * e, u32 sid, u8 is_ct)
2017{
2018  return (e->event_type == SESSION_IO_EVT_TX && e->session_index == sid);
2019}
2020
2021always_inline u8
2022vcl_fifo_is_writeable (svm_fifo_t * f, u32 len, u8 is_dgram)
2023{
2024  u32 max_enq = svm_fifo_max_enqueue_prod (f);
2025  if (is_dgram)
2026    return max_enq >= (sizeof (session_dgram_hdr_t) + len);
2027  else
2028    return max_enq > 0;
2029}
2030
2031always_inline int
2032vppcom_session_write_inline (vcl_worker_t * wrk, vcl_session_t * s, void *buf,
2033			     size_t n, u8 is_flush, u8 is_dgram)
2034{
2035  int n_write, is_nonblocking;
2036  session_evt_type_t et;
2037  svm_msg_q_msg_t msg;
2038  svm_fifo_t *tx_fifo;
2039  session_event_t *e;
2040  svm_msg_q_t *mq;
2041  u8 is_ct;
2042
2043  if (PREDICT_FALSE (!buf || n == 0))
2044    return VPPCOM_EINVAL;
2045
2046  if (PREDICT_FALSE (s->is_vep))
2047    {
2048      VDBG (0, "ERROR: session %u [0x%llx]: cannot write to an epoll"
2049	    " session!", s->session_index, s->vpp_handle);
2050      return VPPCOM_EBADFD;
2051    }
2052
2053  if (PREDICT_FALSE (!vcl_session_is_open (s)))
2054    {
2055      VDBG (1, "session %u [0x%llx]: is not open! state 0x%x (%s)",
2056	    s->session_index, s->vpp_handle, s->session_state,
2057	    vppcom_session_state_str (s->session_state));
2058      return vcl_session_closed_error (s);;
2059    }
2060
2061  is_ct = vcl_session_is_ct (s);
2062  tx_fifo = is_ct ? s->ct_tx_fifo : s->tx_fifo;
2063  is_nonblocking = VCL_SESS_ATTR_TEST (s->attr, VCL_SESS_ATTR_NONBLOCK);
2064
2065  mq = wrk->app_event_queue;
2066  if (!vcl_fifo_is_writeable (tx_fifo, n, is_dgram))
2067    {
2068      if (is_nonblocking)
2069	{
2070	  return VPPCOM_EWOULDBLOCK;
2071	}
2072      while (!vcl_fifo_is_writeable (tx_fifo, n, is_dgram))
2073	{
2074	  svm_fifo_add_want_deq_ntf (tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF);
2075	  if (vcl_session_is_closing (s))
2076	    return vcl_session_closing_error (s);
2077	  svm_msg_q_lock (mq);
2078	  if (svm_msg_q_is_empty (mq))
2079	    svm_msg_q_wait (mq);
2080
2081	  svm_msg_q_sub_w_lock (mq, &msg);
2082	  e = svm_msg_q_msg_data (mq, &msg);
2083	  svm_msg_q_unlock (mq);
2084
2085	  if (!vcl_is_tx_evt_for_session (e, s->session_index, is_ct))
2086	    vcl_handle_mq_event (wrk, e);
2087	  svm_msg_q_free_msg (mq, &msg);
2088	}
2089    }
2090
2091  et = SESSION_IO_EVT_TX;
2092  if (is_flush && !is_ct)
2093    et = SESSION_IO_EVT_TX_FLUSH;
2094
2095  if (is_dgram)
2096    n_write = app_send_dgram_raw (tx_fifo, &s->transport,
2097				  s->vpp_evt_q, buf, n, et,
2098				  0 /* do_evt */ , SVM_Q_WAIT);
2099  else
2100    n_write = app_send_stream_raw (tx_fifo, s->vpp_evt_q, buf, n, et,
2101				   0 /* do_evt */ , SVM_Q_WAIT);
2102
2103  if (svm_fifo_set_event (s->tx_fifo))
2104    app_send_io_evt_to_vpp (s->vpp_evt_q, s->tx_fifo->master_session_index,
2105			    et, SVM_Q_WAIT);
2106
2107  ASSERT (n_write > 0);
2108
2109  VDBG (2, "session %u [0x%llx]: wrote %d bytes", s->session_index,
2110	s->vpp_handle, n_write);
2111
2112  return n_write;
2113}
2114
2115int
2116vppcom_session_write (uint32_t session_handle, void *buf, size_t n)
2117{
2118  vcl_worker_t *wrk = vcl_worker_get_current ();
2119  vcl_session_t *s;
2120
2121  s = vcl_session_get_w_handle (wrk, session_handle);
2122  if (PREDICT_FALSE (!s))
2123    return VPPCOM_EBADFD;
2124
2125  return vppcom_session_write_inline (wrk, s, buf, n,
2126				      0 /* is_flush */ , s->is_dgram ? 1 : 0);
2127}
2128
2129int
2130vppcom_session_write_msg (uint32_t session_handle, void *buf, size_t n)
2131{
2132  vcl_worker_t *wrk = vcl_worker_get_current ();
2133  vcl_session_t *s;
2134
2135  s = vcl_session_get_w_handle (wrk, session_handle);
2136  if (PREDICT_FALSE (!s))
2137    return VPPCOM_EBADFD;
2138
2139  return vppcom_session_write_inline (wrk, s, buf, n,
2140				      1 /* is_flush */ , s->is_dgram ? 1 : 0);
2141}
2142
2143#define vcl_fifo_rx_evt_valid_or_break(_s)				\
2144if (PREDICT_FALSE (!_s->rx_fifo))					\
2145  break;								\
2146if (PREDICT_FALSE (svm_fifo_is_empty (_s->rx_fifo)))			\
2147  {									\
2148    if (!vcl_session_is_ct (_s))					\
2149      {									\
2150	svm_fifo_unset_event (_s->rx_fifo);				\
2151	if (svm_fifo_is_empty (_s->rx_fifo))				\
2152	  break;							\
2153      }									\
2154    else if (svm_fifo_is_empty (_s->ct_rx_fifo))			\
2155      {									\
2156	svm_fifo_unset_event (_s->ct_rx_fifo);				\
2157	if (svm_fifo_is_empty (_s->ct_rx_fifo))				\
2158	  break;							\
2159      }									\
2160  }									\
2161
2162static void
2163vcl_select_handle_mq_event (vcl_worker_t * wrk, session_event_t * e,
2164			    unsigned long n_bits, unsigned long *read_map,
2165			    unsigned long *write_map,
2166			    unsigned long *except_map, u32 * bits_set)
2167{
2168  session_disconnected_msg_t *disconnected_msg;
2169  session_connected_msg_t *connected_msg;
2170  vcl_session_t *session;
2171  u32 sid;
2172
2173  switch (e->event_type)
2174    {
2175    case SESSION_IO_EVT_RX:
2176      sid = e->session_index;
2177      session = vcl_session_get (wrk, sid);
2178      if (!session || !vcl_session_is_open (session))
2179	break;
2180      vcl_fifo_rx_evt_valid_or_break (session);
2181      if (sid < n_bits && read_map)
2182	{
2183	  clib_bitmap_set_no_check ((uword *) read_map, sid, 1);
2184	  *bits_set += 1;
2185	}
2186      break;
2187    case SESSION_IO_EVT_TX:
2188      sid = e->session_index;
2189      session = vcl_session_get (wrk, sid);
2190      if (!session || !vcl_session_is_open (session))
2191	break;
2192      if (sid < n_bits && write_map)
2193	{
2194	  clib_bitmap_set_no_check ((uword *) write_map, sid, 1);
2195	  *bits_set += 1;
2196	}
2197      break;
2198    case SESSION_CTRL_EVT_ACCEPTED:
2199      session = vcl_session_accepted (wrk,
2200				      (session_accepted_msg_t *) e->data);
2201      if (!session)
2202	break;
2203      sid = session->session_index;
2204      if (sid < n_bits && read_map)
2205	{
2206	  clib_bitmap_set_no_check ((uword *) read_map, sid, 1);
2207	  *bits_set += 1;
2208	}
2209      break;
2210    case SESSION_CTRL_EVT_CONNECTED:
2211      connected_msg = (session_connected_msg_t *) e->data;
2212      sid = vcl_session_connected_handler (wrk, connected_msg);
2213      if (sid == VCL_INVALID_SESSION_INDEX)
2214	break;
2215      if (sid < n_bits && write_map)
2216	{
2217	  clib_bitmap_set_no_check ((uword *) write_map, sid, 1);
2218	  *bits_set += 1;
2219	}
2220      break;
2221    case SESSION_CTRL_EVT_DISCONNECTED:
2222      disconnected_msg = (session_disconnected_msg_t *) e->data;
2223      session = vcl_session_disconnected_handler (wrk, disconnected_msg);
2224      if (!session)
2225	break;
2226      sid = session->session_index;
2227      if (sid < n_bits && except_map)
2228	{
2229	  clib_bitmap_set_no_check ((uword *) except_map, sid, 1);
2230	  *bits_set += 1;
2231	}
2232      break;
2233    case SESSION_CTRL_EVT_RESET:
2234      sid = vcl_session_reset_handler (wrk, (session_reset_msg_t *) e->data);
2235      if (sid < n_bits && except_map)
2236	{
2237	  clib_bitmap_set_no_check ((uword *) except_map, sid, 1);
2238	  *bits_set += 1;
2239	}
2240      break;
2241    case SESSION_CTRL_EVT_UNLISTEN_REPLY:
2242      vcl_session_unlisten_reply_handler (wrk, e->data);
2243      break;
2244    case SESSION_CTRL_EVT_MIGRATED:
2245      vcl_session_migrated_handler (wrk, e->data);
2246      break;
2247    case SESSION_CTRL_EVT_CLEANUP:
2248      vcl_session_cleanup_handler (wrk, e->data);
2249      break;
2250    case SESSION_CTRL_EVT_WORKER_UPDATE_REPLY:
2251      vcl_session_worker_update_reply_handler (wrk, e->data);
2252      break;
2253    case SESSION_CTRL_EVT_REQ_WORKER_UPDATE:
2254      vcl_session_req_worker_update_handler (wrk, e->data);
2255      break;
2256    case SESSION_CTRL_EVT_APP_ADD_SEGMENT:
2257      vcl_session_app_add_segment_handler (wrk, e->data);
2258      break;
2259    case SESSION_CTRL_EVT_APP_DEL_SEGMENT:
2260      vcl_session_app_del_segment_handler (wrk, e->data);
2261      break;
2262    default:
2263      clib_warning ("unhandled: %u", e->event_type);
2264      break;
2265    }
2266}
2267
2268static int
2269vcl_select_handle_mq (vcl_worker_t * wrk, svm_msg_q_t * mq,
2270		      unsigned long n_bits, unsigned long *read_map,
2271		      unsigned long *write_map, unsigned long *except_map,
2272		      double time_to_wait, u32 * bits_set)
2273{
2274  svm_msg_q_msg_t *msg;
2275  session_event_t *e;
2276  u32 i;
2277
2278  svm_msg_q_lock (mq);
2279  if (svm_msg_q_is_empty (mq))
2280    {
2281      if (*bits_set)
2282	{
2283	  svm_msg_q_unlock (mq);
2284	  return 0;
2285	}
2286
2287      if (!time_to_wait)
2288	{
2289	  svm_msg_q_unlock (mq);
2290	  return 0;
2291	}
2292      else if (time_to_wait < 0)
2293	{
2294	  svm_msg_q_wait (mq);
2295	}
2296      else
2297	{
2298	  if (svm_msg_q_timedwait (mq, time_to_wait))
2299	    {
2300	      svm_msg_q_unlock (mq);
2301	      return 0;
2302	    }
2303	}
2304    }
2305  vcl_mq_dequeue_batch (wrk, mq, ~0);
2306  svm_msg_q_unlock (mq);
2307
2308  for (i = 0; i < vec_len (wrk->mq_msg_vector); i++)
2309    {
2310      msg = vec_elt_at_index (wrk->mq_msg_vector, i);
2311      e = svm_msg_q_msg_data (mq, msg);
2312      vcl_select_handle_mq_event (wrk, e, n_bits, read_map, write_map,
2313				  except_map, bits_set);
2314      svm_msg_q_free_msg (mq, msg);
2315    }
2316  vec_reset_length (wrk->mq_msg_vector);
2317  vcl_handle_pending_wrk_updates (wrk);
2318  return *bits_set;
2319}
2320
2321static int
2322vppcom_select_condvar (vcl_worker_t * wrk, int n_bits,
2323		       vcl_si_set * read_map, vcl_si_set * write_map,
2324		       vcl_si_set * except_map, double time_to_wait,
2325		       u32 * bits_set)
2326{
2327  double wait = 0, start = 0;
2328
2329  if (!*bits_set)
2330    {
2331      wait = time_to_wait;
2332      start = clib_time_now (&wrk->clib_time);
2333    }
2334
2335  do
2336    {
2337      vcl_select_handle_mq (wrk, wrk->app_event_queue, n_bits, read_map,
2338			    write_map, except_map, wait, bits_set);
2339      if (*bits_set)
2340	return *bits_set;
2341      if (wait == -1)
2342	continue;
2343
2344      wait = wait - (clib_time_now (&wrk->clib_time) - start);
2345    }
2346  while (wait > 0);
2347
2348  return 0;
2349}
2350
2351static int
2352vppcom_select_eventfd (vcl_worker_t * wrk, int n_bits,
2353		       vcl_si_set * read_map, vcl_si_set * write_map,
2354		       vcl_si_set * except_map, double time_to_wait,
2355		       u32 * bits_set)
2356{
2357  vcl_mq_evt_conn_t *mqc;
2358  int __clib_unused n_read;
2359  int n_mq_evts, i;
2360  u64 buf;
2361
2362  vec_validate (wrk->mq_events, pool_elts (wrk->mq_evt_conns));
2363  n_mq_evts = epoll_wait (wrk->mqs_epfd, wrk->mq_events,
2364			  vec_len (wrk->mq_events), time_to_wait);
2365  for (i = 0; i < n_mq_evts; i++)
2366    {
2367      mqc = vcl_mq_evt_conn_get (wrk, wrk->mq_events[i].data.u32);
2368      n_read = read (mqc->mq_fd, &buf, sizeof (buf));
2369      vcl_select_handle_mq (wrk, mqc->mq, n_bits, read_map, write_map,
2370			    except_map, 0, bits_set);
2371    }
2372
2373  return (n_mq_evts > 0 ? (int) *bits_set : 0);
2374}
2375
2376int
2377vppcom_select (int n_bits, vcl_si_set * read_map, vcl_si_set * write_map,
2378	       vcl_si_set * except_map, double time_to_wait)
2379{
2380  u32 sid, minbits = clib_max (n_bits, BITS (uword)), bits_set = 0;
2381  vcl_worker_t *wrk = vcl_worker_get_current ();
2382  vcl_session_t *session = 0;
2383  int i;
2384
2385  if (n_bits && read_map)
2386    {
2387      clib_bitmap_validate (wrk->rd_bitmap, minbits);
2388      clib_memcpy_fast (wrk->rd_bitmap, read_map,
2389			vec_len (wrk->rd_bitmap) * sizeof (vcl_si_set));
2390      memset (read_map, 0, vec_len (wrk->rd_bitmap) * sizeof (vcl_si_set));
2391    }
2392  if (n_bits && write_map)
2393    {
2394      clib_bitmap_validate (wrk->wr_bitmap, minbits);
2395      clib_memcpy_fast (wrk->wr_bitmap, write_map,
2396			vec_len (wrk->wr_bitmap) * sizeof (vcl_si_set));
2397      memset (write_map, 0, vec_len (wrk->wr_bitmap) * sizeof (vcl_si_set));
2398    }
2399  if (n_bits && except_map)
2400    {
2401      clib_bitmap_validate (wrk->ex_bitmap, minbits);
2402      clib_memcpy_fast (wrk->ex_bitmap, except_map,
2403			vec_len (wrk->ex_bitmap) * sizeof (vcl_si_set));
2404      memset (except_map, 0, vec_len (wrk->ex_bitmap) * sizeof (vcl_si_set));
2405    }
2406
2407  if (!n_bits)
2408    return 0;
2409
2410  if (!write_map)
2411    goto check_rd;
2412
2413  /* *INDENT-OFF* */
2414  clib_bitmap_foreach (sid, wrk->wr_bitmap, ({
2415    if (!(session = vcl_session_get (wrk, sid)))
2416      {
2417	clib_bitmap_set_no_check ((uword*)write_map, sid, 1);
2418	bits_set++;
2419	continue;
2420      }
2421
2422    if (vcl_session_write_ready (session))
2423      {
2424        clib_bitmap_set_no_check ((uword*)write_map, sid, 1);
2425        bits_set++;
2426      }
2427    else
2428      svm_fifo_add_want_deq_ntf (session->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF);
2429  }));
2430
2431check_rd:
2432  if (!read_map)
2433    goto check_mq;
2434
2435  clib_bitmap_foreach (sid, wrk->rd_bitmap, ({
2436    if (!(session = vcl_session_get (wrk, sid)))
2437      {
2438	clib_bitmap_set_no_check ((uword*)read_map, sid, 1);
2439	bits_set++;
2440	continue;
2441      }
2442
2443    if (vcl_session_read_ready (session))
2444      {
2445        clib_bitmap_set_no_check ((uword*)read_map, sid, 1);
2446        bits_set++;
2447      }
2448  }));
2449  /* *INDENT-ON* */
2450
2451check_mq:
2452
2453  for (i = 0; i < vec_len (wrk->unhandled_evts_vector); i++)
2454    {
2455      vcl_select_handle_mq_event (wrk, &wrk->unhandled_evts_vector[i], n_bits,
2456				  read_map, write_map, except_map, &bits_set);
2457    }
2458  vec_reset_length (wrk->unhandled_evts_vector);
2459
2460  if (vcm->cfg.use_mq_eventfd)
2461    vppcom_select_eventfd (wrk, n_bits, read_map, write_map, except_map,
2462			   time_to_wait, &bits_set);
2463  else
2464    vppcom_select_condvar (wrk, n_bits, read_map, write_map, except_map,
2465			   time_to_wait, &bits_set);
2466
2467  return (bits_set);
2468}
2469
2470static inline void
2471vep_verify_epoll_chain (vcl_worker_t * wrk, u32 vep_handle)
2472{
2473  vcl_session_t *session;
2474  vppcom_epoll_t *vep;
2475  u32 sh = vep_handle;
2476
2477  if (VPPCOM_DEBUG <= 2)
2478    return;
2479
2480  session = vcl_session_get_w_handle (wrk, vep_handle);
2481  if (PREDICT_FALSE (!session))
2482    {
2483      VDBG (0, "ERROR: Invalid vep_sh (%u)!", vep_handle);
2484      goto done;
2485    }
2486  if (PREDICT_FALSE (!session->is_vep))
2487    {
2488      VDBG (0, "ERROR: vep_sh (%u) is not a vep!", vep_handle);
2489      goto done;
2490    }
2491  vep = &session->vep;
2492  VDBG (0, "vep_sh (%u): Dumping epoll chain\n"
2493	"{\n"
2494	"   is_vep         = %u\n"
2495	"   is_vep_session = %u\n"
2496	"   next_sh        = 0x%x (%u)\n"
2497	"}\n", vep_handle, session->is_vep, session->is_vep_session,
2498	vep->next_sh, vep->next_sh);
2499
2500  for (sh = vep->next_sh; sh != ~0; sh = vep->next_sh)
2501    {
2502      session = vcl_session_get_w_handle (wrk, sh);
2503      if (PREDICT_FALSE (!session))
2504	{
2505	  VDBG (0, "ERROR: Invalid sh (%u)!", sh);
2506	  goto done;
2507	}
2508      if (PREDICT_FALSE (session->is_vep))
2509	{
2510	  VDBG (0, "ERROR: sh (%u) is a vep!", vep_handle);
2511	}
2512      else if (PREDICT_FALSE (!session->is_vep_session))
2513	{
2514	  VDBG (0, "ERROR: sh (%u) is not a vep session handle!", sh);
2515	  goto done;
2516	}
2517      vep = &session->vep;
2518      if (PREDICT_FALSE (vep->vep_sh != vep_handle))
2519	VDBG (0, "ERROR: session (%u) vep_sh (%u) != vep_sh (%u)!",
2520	      sh, session->vep.vep_sh, vep_handle);
2521      if (session->is_vep_session)
2522	{
2523	  VDBG (0, "vep_sh[%u]: sh 0x%x (%u)\n"
2524		"{\n"
2525		"   next_sh        = 0x%x (%u)\n"
2526		"   prev_sh        = 0x%x (%u)\n"
2527		"   vep_sh         = 0x%x (%u)\n"
2528		"   ev.events      = 0x%x\n"
2529		"   ev.data.u64    = 0x%llx\n"
2530		"   et_mask        = 0x%x\n"
2531		"}\n",
2532		vep_handle, sh, sh, vep->next_sh, vep->next_sh, vep->prev_sh,
2533		vep->prev_sh, vep->vep_sh, vep->vep_sh, vep->ev.events,
2534		vep->ev.data.u64, vep->et_mask);
2535	}
2536    }
2537
2538done:
2539  VDBG (0, "vep_sh (%u): Dump complete!\n", vep_handle);
2540}
2541
2542int
2543vppcom_epoll_create (void)
2544{
2545  vcl_worker_t *wrk = vcl_worker_get_current ();
2546  vcl_session_t *vep_session;
2547
2548  vep_session = vcl_session_alloc (wrk);
2549
2550  vep_session->is_vep = 1;
2551  vep_session->vep.vep_sh = ~0;
2552  vep_session->vep.next_sh = ~0;
2553  vep_session->vep.prev_sh = ~0;
2554  vep_session->vpp_handle = ~0;
2555
2556  vcl_evt (VCL_EVT_EPOLL_CREATE, vep_session, vep_session->session_index);
2557  VDBG (0, "Created vep_idx %u", vep_session->session_index);
2558
2559  return vcl_session_handle (vep_session);
2560}
2561
2562int
2563vppcom_epoll_ctl (uint32_t vep_handle, int op, uint32_t session_handle,
2564		  struct epoll_event *event)
2565{
2566  vcl_worker_t *wrk = vcl_worker_get_current ();
2567  vcl_session_t *vep_session;
2568  vcl_session_t *session;
2569  int rv = VPPCOM_OK;
2570
2571  if (vep_handle == session_handle)
2572    {
2573      VDBG (0, "vep_sh == session handle (%u)!", vep_handle);
2574      return VPPCOM_EINVAL;
2575    }
2576
2577  vep_session = vcl_session_get_w_handle (wrk, vep_handle);
2578  if (PREDICT_FALSE (!vep_session))
2579    {
2580      VDBG (0, "Invalid vep_sh (%u)!", vep_handle);
2581      return VPPCOM_EBADFD;
2582    }
2583  if (PREDICT_FALSE (!vep_session->is_vep))
2584    {
2585      VDBG (0, "vep_sh (%u) is not a vep!", vep_handle);
2586      return VPPCOM_EINVAL;
2587    }
2588
2589  ASSERT (vep_session->vep.vep_sh == ~0);
2590  ASSERT (vep_session->vep.prev_sh == ~0);
2591
2592  session = vcl_session_get_w_handle (wrk, session_handle);
2593  if (PREDICT_FALSE (!session))
2594    {
2595      VDBG (0, "Invalid session_handle (%u)!", session_handle);
2596      return VPPCOM_EBADFD;
2597    }
2598  if (PREDICT_FALSE (session->is_vep))
2599    {
2600      VDBG (0, "session_handle (%u) is a vep!", vep_handle);
2601      return VPPCOM_EINVAL;
2602    }
2603
2604  switch (op)
2605    {
2606    case EPOLL_CTL_ADD:
2607      if (PREDICT_FALSE (!event))
2608	{
2609	  VDBG (0, "EPOLL_CTL_ADD: NULL pointer to epoll_event structure!");
2610	  return VPPCOM_EINVAL;
2611	}
2612      if (vep_session->vep.next_sh != ~0)
2613	{
2614	  vcl_session_t *next_session;
2615	  next_session = vcl_session_get_w_handle (wrk,
2616						   vep_session->vep.next_sh);
2617	  if (PREDICT_FALSE (!next_session))
2618	    {
2619	      VDBG (0, "EPOLL_CTL_ADD: Invalid vep.next_sh (%u) on "
2620		    "vep_idx (%u)!", vep_session->vep.next_sh, vep_handle);
2621	      return VPPCOM_EBADFD;
2622	    }
2623	  ASSERT (next_session->vep.prev_sh == vep_handle);
2624	  next_session->vep.prev_sh = session_handle;
2625	}
2626      session->vep.next_sh = vep_session->vep.next_sh;
2627      session->vep.prev_sh = vep_handle;
2628      session->vep.vep_sh = vep_handle;
2629      session->vep.et_mask = VEP_DEFAULT_ET_MASK;
2630      session->vep.ev = *event;
2631      session->is_vep = 0;
2632      session->is_vep_session = 1;
2633      vep_session->vep.next_sh = session_handle;
2634
2635      if (session->tx_fifo)
2636	svm_fifo_add_want_deq_ntf (session->tx_fifo,
2637				   SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
2638
2639      /* Generate EPOLLOUT if tx fifo not full */
2640      if ((event->events & EPOLLOUT) &&
2641	  (vcl_session_write_ready (session) > 0))
2642	{
2643	  session_event_t e = { 0 };
2644	  e.event_type = SESSION_IO_EVT_TX;
2645	  e.session_index = session->session_index;
2646	  vec_add1 (wrk->unhandled_evts_vector, e);
2647	}
2648      /* Generate EPOLLIN if rx fifo has data */
2649      if ((event->events & EPOLLIN) && (vcl_session_read_ready (session) > 0))
2650	{
2651	  session_event_t e = { 0 };
2652	  e.event_type = SESSION_IO_EVT_RX;
2653	  e.session_index = session->session_index;
2654	  vec_add1 (wrk->unhandled_evts_vector, e);
2655	}
2656      VDBG (1, "EPOLL_CTL_ADD: vep_sh %u, sh %u, events 0x%x, data 0x%llx!",
2657	    vep_handle, session_handle, event->events, event->data.u64);
2658      vcl_evt (VCL_EVT_EPOLL_CTLADD, session, event->events, event->data.u64);
2659      break;
2660
2661    case EPOLL_CTL_MOD:
2662      if (PREDICT_FALSE (!event))
2663	{
2664	  VDBG (0, "EPOLL_CTL_MOD: NULL pointer to epoll_event structure!");
2665	  rv = VPPCOM_EINVAL;
2666	  goto done;
2667	}
2668      else if (PREDICT_FALSE (!session->is_vep_session))
2669	{
2670	  VDBG (0, "sh %u EPOLL_CTL_MOD: not a vep session!", session_handle);
2671	  rv = VPPCOM_EINVAL;
2672	  goto done;
2673	}
2674      else if (PREDICT_FALSE (session->vep.vep_sh != vep_handle))
2675	{
2676	  VDBG (0, "EPOLL_CTL_MOD: sh %u vep_sh (%u) != vep_sh (%u)!",
2677		session_handle, session->vep.vep_sh, vep_handle);
2678	  rv = VPPCOM_EINVAL;
2679	  goto done;
2680	}
2681
2682      /* Generate EPOLLOUT when tx_fifo/ct_tx_fifo not full */
2683      if ((event->events & EPOLLOUT) &&
2684	  !(session->vep.ev.events & EPOLLOUT) &&
2685	  (vcl_session_write_ready (session) > 0))
2686	{
2687	  session_event_t e = { 0 };
2688	  e.event_type = SESSION_IO_EVT_TX;
2689	  e.session_index = session->session_index;
2690	  vec_add1 (wrk->unhandled_evts_vector, e);
2691	}
2692      session->vep.et_mask = VEP_DEFAULT_ET_MASK;
2693      session->vep.ev = *event;
2694      VDBG (1, "EPOLL_CTL_MOD: vep_sh %u, sh %u, events 0x%x, data 0x%llx!",
2695	    vep_handle, session_handle, event->events, event->data.u64);
2696      break;
2697
2698    case EPOLL_CTL_DEL:
2699      if (PREDICT_FALSE (!session->is_vep_session))
2700	{
2701	  VDBG (0, "EPOLL_CTL_DEL: %u not a vep session!", session_handle);
2702	  rv = VPPCOM_EINVAL;
2703	  goto done;
2704	}
2705      else if (PREDICT_FALSE (session->vep.vep_sh != vep_handle))
2706	{
2707	  VDBG (0, "EPOLL_CTL_DEL: sh %u vep_sh (%u) != vep_sh (%u)!",
2708		session_handle, session->vep.vep_sh, vep_handle);
2709	  rv = VPPCOM_EINVAL;
2710	  goto done;
2711	}
2712
2713      if (session->vep.prev_sh == vep_handle)
2714	vep_session->vep.next_sh = session->vep.next_sh;
2715      else
2716	{
2717	  vcl_session_t *prev_session;
2718	  prev_session = vcl_session_get_w_handle (wrk, session->vep.prev_sh);
2719	  if (PREDICT_FALSE (!prev_session))
2720	    {
2721	      VDBG (0, "EPOLL_CTL_DEL: Invalid prev_sh (%u) on sh (%u)!",
2722		    session->vep.prev_sh, session_handle);
2723	      return VPPCOM_EBADFD;
2724	    }
2725	  ASSERT (prev_session->vep.next_sh == session_handle);
2726	  prev_session->vep.next_sh = session->vep.next_sh;
2727	}
2728      if (session->vep.next_sh != ~0)
2729	{
2730	  vcl_session_t *next_session;
2731	  next_session = vcl_session_get_w_handle (wrk, session->vep.next_sh);
2732	  if (PREDICT_FALSE (!next_session))
2733	    {
2734	      VDBG (0, "EPOLL_CTL_DEL: Invalid next_sh (%u) on sh (%u)!",
2735		    session->vep.next_sh, session_handle);
2736	      return VPPCOM_EBADFD;
2737	    }
2738	  ASSERT (next_session->vep.prev_sh == session_handle);
2739	  next_session->vep.prev_sh = session->vep.prev_sh;
2740	}
2741
2742      memset (&session->vep, 0, sizeof (session->vep));
2743      session->vep.next_sh = ~0;
2744      session->vep.prev_sh = ~0;
2745      session->vep.vep_sh = ~0;
2746      session->is_vep_session = 0;
2747
2748      if (session->tx_fifo)
2749	svm_fifo_del_want_deq_ntf (session->tx_fifo, SVM_FIFO_NO_DEQ_NOTIF);
2750
2751      VDBG (1, "EPOLL_CTL_DEL: vep_idx %u, sh %u!", vep_handle,
2752	    session_handle);
2753      vcl_evt (VCL_EVT_EPOLL_CTLDEL, session, vep_sh);
2754      break;
2755
2756    default:
2757      VDBG (0, "Invalid operation (%d)!", op);
2758      rv = VPPCOM_EINVAL;
2759    }
2760
2761  vep_verify_epoll_chain (wrk, vep_handle);
2762
2763done:
2764  return rv;
2765}
2766
2767static inline void
2768vcl_epoll_wait_handle_mq_event (vcl_worker_t * wrk, session_event_t * e,
2769				struct epoll_event *events, u32 * num_ev)
2770{
2771  session_disconnected_msg_t *disconnected_msg;
2772  session_connected_msg_t *connected_msg;
2773  u32 sid = ~0, session_events;
2774  u64 session_evt_data = ~0;
2775  vcl_session_t *session;
2776  u8 add_event = 0;
2777
2778  switch (e->event_type)
2779    {
2780    case SESSION_IO_EVT_RX:
2781      sid = e->session_index;
2782      session = vcl_session_get (wrk, sid);
2783      if (vcl_session_is_closed (session))
2784	break;
2785      vcl_fifo_rx_evt_valid_or_break (session);
2786      session_events = session->vep.ev.events;
2787      if (!(EPOLLIN & session->vep.ev.events) || session->has_rx_evt)
2788	break;
2789      add_event = 1;
2790      events[*num_ev].events |= EPOLLIN;
2791      session_evt_data = session->vep.ev.data.u64;
2792      session->has_rx_evt = 1;
2793      break;
2794    case SESSION_IO_EVT_TX:
2795      sid = e->session_index;
2796      session = vcl_session_get (wrk, sid);
2797      if (vcl_session_is_closed (session))
2798	break;
2799      session_events = session->vep.ev.events;
2800      if (!(EPOLLOUT & session_events))
2801	break;
2802      add_event = 1;
2803      events[*num_ev].events |= EPOLLOUT;
2804      session_evt_data = session->vep.ev.data.u64;
2805      svm_fifo_reset_has_deq_ntf (session->tx_fifo);
2806      break;
2807    case SESSION_CTRL_EVT_ACCEPTED:
2808      session = vcl_session_accepted (wrk,
2809				      (session_accepted_msg_t *) e->data);
2810      if (!session)
2811	break;
2812
2813      session_events = session->vep.ev.events;
2814      if (!(EPOLLIN & session_events))
2815	break;
2816
2817      add_event = 1;
2818      events[*num_ev].events |= EPOLLIN;
2819      session_evt_data = session->vep.ev.data.u64;
2820      break;
2821    case SESSION_CTRL_EVT_CONNECTED:
2822      connected_msg = (session_connected_msg_t *) e->data;
2823      sid = vcl_session_connected_handler (wrk, connected_msg);
2824      /* Generate EPOLLOUT because there's no connected event */
2825      session = vcl_session_get (wrk, sid);
2826      if (vcl_session_is_closed (session))
2827	break;
2828      session_events = session->vep.ev.events;
2829      if (!(EPOLLOUT & session_events))
2830	break;
2831      add_event = 1;
2832      events[*num_ev].events |= EPOLLOUT;
2833      session_evt_data = session->vep.ev.data.u64;
2834      if (session->session_state & STATE_DETACHED)
2835	events[*num_ev].events |= EPOLLHUP;
2836      break;
2837    case SESSION_CTRL_EVT_DISCONNECTED:
2838      disconnected_msg = (session_disconnected_msg_t *) e->data;
2839      session = vcl_session_disconnected_handler (wrk, disconnected_msg);
2840      if (vcl_session_is_closed (session))
2841	break;
2842      session_events = session->vep.ev.events;
2843      add_event = 1;
2844      events[*num_ev].events |= EPOLLHUP | EPOLLRDHUP;
2845      session_evt_data = session->vep.ev.data.u64;
2846      break;
2847    case SESSION_CTRL_EVT_RESET:
2848      sid = vcl_session_reset_handler (wrk, (session_reset_msg_t *) e->data);
2849      session = vcl_session_get (wrk, sid);
2850      if (vcl_session_is_closed (session))
2851	break;
2852      session_events = session->vep.ev.events;
2853      add_event = 1;
2854      events[*num_ev].events |= EPOLLHUP | EPOLLRDHUP;
2855      session_evt_data = session->vep.ev.data.u64;
2856      break;
2857    case SESSION_CTRL_EVT_UNLISTEN_REPLY:
2858      vcl_session_unlisten_reply_handler (wrk, e->data);
2859      break;
2860    case SESSION_CTRL_EVT_MIGRATED:
2861      vcl_session_migrated_handler (wrk, e->data);
2862      break;
2863    case SESSION_CTRL_EVT_CLEANUP:
2864      vcl_session_cleanup_handler (wrk, e->data);
2865      break;
2866    case SESSION_CTRL_EVT_REQ_WORKER_UPDATE:
2867      vcl_session_req_worker_update_handler (wrk, e->data);
2868      break;
2869    case SESSION_CTRL_EVT_WORKER_UPDATE_REPLY:
2870      vcl_session_worker_update_reply_handler (wrk, e->data);
2871      break;
2872    case SESSION_CTRL_EVT_APP_ADD_SEGMENT:
2873      vcl_session_app_add_segment_handler (wrk, e->data);
2874      break;
2875    case SESSION_CTRL_EVT_APP_DEL_SEGMENT:
2876      vcl_session_app_del_segment_handler (wrk, e->data);
2877      break;
2878    default:
2879      VDBG (0, "unhandled: %u", e->event_type);
2880      break;
2881    }
2882
2883  if (add_event)
2884    {
2885      events[*num_ev].data.u64 = session_evt_data;
2886      if (EPOLLONESHOT & session_events)
2887	{
2888	  session = vcl_session_get (wrk, sid);
2889	  session->vep.ev.events = 0;
2890	}
2891      *num_ev += 1;
2892    }
2893}
2894
2895static int
2896vcl_epoll_wait_handle_mq (vcl_worker_t * wrk, svm_msg_q_t * mq,
2897			  struct epoll_event *events, u32 maxevents,
2898			  double wait_for_time, u32 * num_ev)
2899{
2900  svm_msg_q_msg_t *msg;
2901  session_event_t *e;
2902  int i;
2903
2904  if (vec_len (wrk->mq_msg_vector) && svm_msg_q_is_empty (mq))
2905    goto handle_dequeued;
2906
2907  svm_msg_q_lock (mq);
2908  if (svm_msg_q_is_empty (mq))
2909    {
2910      if (!wait_for_time)
2911	{
2912	  svm_msg_q_unlock (mq);
2913	  return 0;
2914	}
2915      else if (wait_for_time < 0)
2916	{
2917	  svm_msg_q_wait (mq);
2918	}
2919      else
2920	{
2921	  if (svm_msg_q_timedwait (mq, wait_for_time / 1e3))
2922	    {
2923	      svm_msg_q_unlock (mq);
2924	      return 0;
2925	    }
2926	}
2927    }
2928  ASSERT (maxevents > *num_ev);
2929  vcl_mq_dequeue_batch (wrk, mq, ~0);
2930  svm_msg_q_unlock (mq);
2931
2932handle_dequeued:
2933  for (i = 0; i < vec_len (wrk->mq_msg_vector); i++)
2934    {
2935      msg = vec_elt_at_index (wrk->mq_msg_vector, i);
2936      e = svm_msg_q_msg_data (mq, msg);
2937      if (*num_ev < maxevents)
2938	vcl_epoll_wait_handle_mq_event (wrk, e, events, num_ev);
2939      else
2940	vcl_handle_mq_event (wrk, e);
2941      svm_msg_q_free_msg (mq, msg);
2942    }
2943  vec_reset_length (wrk->mq_msg_vector);
2944  vcl_handle_pending_wrk_updates (wrk);
2945  return *num_ev;
2946}
2947
2948static int
2949vppcom_epoll_wait_condvar (vcl_worker_t * wrk, struct epoll_event *events,
2950			   int maxevents, u32 n_evts, double wait_for_time)
2951{
2952  double wait = 0, start = 0, now;
2953
2954  if (!n_evts)
2955    {
2956      wait = wait_for_time;
2957      start = clib_time_now (&wrk->clib_time);
2958    }
2959
2960  do
2961    {
2962      vcl_epoll_wait_handle_mq (wrk, wrk->app_event_queue, events, maxevents,
2963				wait, &n_evts);
2964      if (n_evts)
2965	return n_evts;
2966      if (wait == -1)
2967	continue;
2968
2969      now = clib_time_now (&wrk->clib_time);
2970      wait -= now - start;
2971      start = now;
2972    }
2973  while (wait > 0);
2974
2975  return 0;
2976}
2977
2978static int
2979vppcom_epoll_wait_eventfd (vcl_worker_t * wrk, struct epoll_event *events,
2980			   int maxevents, u32 n_evts, double wait_for_time)
2981{
2982  vcl_mq_evt_conn_t *mqc;
2983  int __clib_unused n_read;
2984  int n_mq_evts, i;
2985  u64 buf;
2986
2987  vec_validate (wrk->mq_events, pool_elts (wrk->mq_evt_conns));
2988again:
2989  n_mq_evts = epoll_wait (wrk->mqs_epfd, wrk->mq_events,
2990			  vec_len (wrk->mq_events), wait_for_time);
2991  for (i = 0; i < n_mq_evts; i++)
2992    {
2993      mqc = vcl_mq_evt_conn_get (wrk, wrk->mq_events[i].data.u32);
2994      n_read = read (mqc->mq_fd, &buf, sizeof (buf));
2995      vcl_epoll_wait_handle_mq (wrk, mqc->mq, events, maxevents, 0, &n_evts);
2996    }
2997  if (!n_evts && n_mq_evts > 0)
2998    goto again;
2999
3000  return (int) n_evts;
3001}
3002
3003int
3004vppcom_epoll_wait (uint32_t vep_handle, struct epoll_event *events,
3005		   int maxevents, double wait_for_time)
3006{
3007  vcl_worker_t *wrk = vcl_worker_get_current ();
3008  vcl_session_t *vep_session;
3009  u32 n_evts = 0;
3010  int i;
3011
3012  if (PREDICT_FALSE (maxevents <= 0))
3013    {
3014      VDBG (0, "ERROR: Invalid maxevents (%d)!", maxevents);
3015      return VPPCOM_EINVAL;
3016    }
3017
3018  vep_session = vcl_session_get_w_handle (wrk, vep_handle);
3019  if (!vep_session)
3020    return VPPCOM_EBADFD;
3021
3022  if (PREDICT_FALSE (!vep_session->is_vep))
3023    {
3024      VDBG (0, "ERROR: vep_idx (%u) is not a vep!", vep_handle);
3025      return VPPCOM_EINVAL;
3026    }
3027
3028  memset (events, 0, sizeof (*events) * maxevents);
3029
3030  if (vec_len (wrk->unhandled_evts_vector))
3031    {
3032      for (i = 0; i < vec_len (wrk->unhandled_evts_vector); i++)
3033	{
3034	  vcl_epoll_wait_handle_mq_event (wrk, &wrk->unhandled_evts_vector[i],
3035					  events, &n_evts);
3036	  if (n_evts == maxevents)
3037	    {
3038	      vec_delete (wrk->unhandled_evts_vector, i + 1, 0);
3039	      return n_evts;
3040	    }
3041	}
3042      vec_reset_length (wrk->unhandled_evts_vector);
3043    }
3044
3045  if (vcm->cfg.use_mq_eventfd)
3046    return vppcom_epoll_wait_eventfd (wrk, events, maxevents, n_evts,
3047				      wait_for_time);
3048
3049  return vppcom_epoll_wait_condvar (wrk, events, maxevents, n_evts,
3050				    wait_for_time);
3051}
3052
3053int
3054vppcom_session_attr (uint32_t session_handle, uint32_t op,
3055		     void *buffer, uint32_t * buflen)
3056{
3057  vcl_worker_t *wrk = vcl_worker_get_current ();
3058  vcl_session_t *session;
3059  int rv = VPPCOM_OK;
3060  u32 *flags = buffer, tmp_flags = 0;
3061  vppcom_endpt_t *ep = buffer;
3062
3063  session = vcl_session_get_w_handle (wrk, session_handle);
3064  if (!session)
3065    return VPPCOM_EBADFD;
3066
3067  switch (op)
3068    {
3069    case VPPCOM_ATTR_GET_NREAD:
3070      rv = vcl_session_read_ready (session);
3071      VDBG (2, "VPPCOM_ATTR_GET_NREAD: sh %u, nread = %d", session_handle,
3072	    rv);
3073      break;
3074
3075    case VPPCOM_ATTR_GET_NWRITE:
3076      rv = vcl_session_write_ready (session);
3077      VDBG (2, "VPPCOM_ATTR_GET_NWRITE: sh %u, nwrite = %d", session_handle,
3078	    rv);
3079      break;
3080
3081    case VPPCOM_ATTR_GET_FLAGS:
3082      if (PREDICT_TRUE (buffer && buflen && (*buflen >= sizeof (*flags))))
3083	{
3084	  *flags = O_RDWR | (VCL_SESS_ATTR_TEST (session->attr,
3085						 VCL_SESS_ATTR_NONBLOCK));
3086	  *buflen = sizeof (*flags);
3087	  VDBG (2, "VPPCOM_ATTR_GET_FLAGS: sh %u, flags = 0x%08x, "
3088		"is_nonblocking = %u", session_handle, *flags,
3089		VCL_SESS_ATTR_TEST (session->attr, VCL_SESS_ATTR_NONBLOCK));
3090	}
3091      else
3092	rv = VPPCOM_EINVAL;
3093      break;
3094
3095    case VPPCOM_ATTR_SET_FLAGS:
3096      if (PREDICT_TRUE (buffer && buflen && (*buflen == sizeof (*flags))))
3097	{
3098	  if (*flags & O_NONBLOCK)
3099	    VCL_SESS_ATTR_SET (session->attr, VCL_SESS_ATTR_NONBLOCK);
3100	  else
3101	    VCL_SESS_ATTR_CLR (session->attr, VCL_SESS_ATTR_NONBLOCK);
3102
3103	  VDBG (2, "VPPCOM_ATTR_SET_FLAGS: sh %u, flags = 0x%08x,"
3104		" is_nonblocking = %u", session_handle, *flags,
3105		VCL_SESS_ATTR_TEST (session->attr, VCL_SESS_ATTR_NONBLOCK));
3106	}
3107      else
3108	rv = VPPCOM_EINVAL;
3109      break;
3110
3111    case VPPCOM_ATTR_GET_PEER_ADDR:
3112      if (PREDICT_TRUE (buffer && buflen &&
3113			(*buflen >= sizeof (*ep)) && ep->ip))
3114	{
3115	  ep->is_ip4 = session->transport.is_ip4;
3116	  ep->port = session->transport.rmt_port;
3117	  if (session->transport.is_ip4)
3118	    clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip4,
3119			      sizeof (ip4_address_t));
3120	  else
3121	    clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip6,
3122			      sizeof (ip6_address_t));
3123	  *buflen = sizeof (*ep);
3124	  VDBG (1, "VPPCOM_ATTR_GET_PEER_ADDR: sh %u, is_ip4 = %u, "
3125		"addr = %U, port %u", session_handle, ep->is_ip4,
3126		format_ip46_address, &session->transport.rmt_ip,
3127		ep->is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
3128		clib_net_to_host_u16 (ep->port));
3129	}
3130      else
3131	rv = VPPCOM_EINVAL;
3132      break;
3133
3134    case VPPCOM_ATTR_GET_LCL_ADDR:
3135      if (PREDICT_TRUE (buffer && buflen &&
3136			(*buflen >= sizeof (*ep)) && ep->ip))
3137	{
3138	  ep->is_ip4 = session->transport.is_ip4;
3139	  ep->port = session->transport.lcl_port;
3140	  if (session->transport.is_ip4)
3141	    clib_memcpy_fast (ep->ip, &session->transport.lcl_ip.ip4,
3142			      sizeof (ip4_address_t));
3143	  else
3144	    clib_memcpy_fast (ep->ip, &session->transport.lcl_ip.ip6,
3145			      sizeof (ip6_address_t));
3146	  *buflen = sizeof (*ep);
3147	  VDBG (1, "VPPCOM_ATTR_GET_LCL_ADDR: sh %u, is_ip4 = %u, addr = %U"
3148		" port %d", session_handle, ep->is_ip4, format_ip46_address,
3149		&session->transport.lcl_ip,
3150		ep->is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
3151		clib_net_to_host_u16 (ep->port));
3152	}
3153      else
3154	rv = VPPCOM_EINVAL;
3155      break;
3156
3157    case VPPCOM_ATTR_SET_LCL_ADDR:
3158      if (PREDICT_TRUE (buffer && buflen &&
3159			(*buflen >= sizeof (*ep)) && ep->ip))
3160	{
3161	  session->transport.is_ip4 = ep->is_ip4;
3162	  session->transport.lcl_port = ep->port;
3163	  vcl_ip_copy_from_ep (&session->transport.lcl_ip, ep);
3164	  *buflen = sizeof (*ep);
3165	  VDBG (1, "VPPCOM_ATTR_SET_LCL_ADDR: sh %u, is_ip4 = %u, addr = %U"
3166		" port %d", session_handle, ep->is_ip4, format_ip46_address,
3167		&session->transport.lcl_ip,
3168		ep->is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
3169		clib_net_to_host_u16 (ep->port));
3170	}
3171      else
3172	rv = VPPCOM_EINVAL;
3173      break;
3174
3175    case VPPCOM_ATTR_GET_LIBC_EPFD:
3176      rv = session->libc_epfd;
3177      VDBG (2, "VPPCOM_ATTR_GET_LIBC_EPFD: libc_epfd %d", rv);
3178      break;
3179
3180    case VPPCOM_ATTR_SET_LIBC_EPFD:
3181      if (PREDICT_TRUE (buffer && buflen &&
3182			(*buflen == sizeof (session->libc_epfd))))
3183	{
3184	  session->libc_epfd = *(int *) buffer;
3185	  *buflen = sizeof (session->libc_epfd);
3186
3187	  VDBG (2, "VPPCOM_ATTR_SET_LIBC_EPFD: libc_epfd %d, buflen %d",
3188		session->libc_epfd, *buflen);
3189	}
3190      else
3191	rv = VPPCOM_EINVAL;
3192      break;
3193
3194    case VPPCOM_ATTR_GET_PROTOCOL:
3195      if (buffer && buflen && (*buflen >= sizeof (int)))
3196	{
3197	  *(int *) buffer = session->session_type;
3198	  *buflen = sizeof (int);
3199
3200	  VDBG (2, "VPPCOM_ATTR_GET_PROTOCOL: %d (%s), buflen %d",
3201		*(int *) buffer, *(int *) buffer ? "UDP" : "TCP", *buflen);
3202	}
3203      else
3204	rv = VPPCOM_EINVAL;
3205      break;
3206
3207    case VPPCOM_ATTR_GET_LISTEN:
3208      if (buffer && buflen && (*buflen >= sizeof (int)))
3209	{
3210	  *(int *) buffer = VCL_SESS_ATTR_TEST (session->attr,
3211						VCL_SESS_ATTR_LISTEN);
3212	  *buflen = sizeof (int);
3213
3214	  VDBG (2, "VPPCOM_ATTR_GET_LISTEN: %d, buflen %d", *(int *) buffer,
3215		*buflen);
3216	}
3217      else
3218	rv = VPPCOM_EINVAL;
3219      break;
3220
3221    case VPPCOM_ATTR_GET_ERROR:
3222      if (buffer && buflen && (*buflen >= sizeof (int)))
3223	{
3224	  *(int *) buffer = 0;
3225	  *buflen = sizeof (int);
3226
3227	  VDBG (2, "VPPCOM_ATTR_GET_ERROR: %d, buflen %d, #VPP-TBD#",
3228		*(int *) buffer, *buflen);
3229	}
3230      else
3231	rv = VPPCOM_EINVAL;
3232      break;
3233
3234    case VPPCOM_ATTR_GET_TX_FIFO_LEN:
3235      if (buffer && buflen && (*buflen >= sizeof (u32)))
3236	{
3237
3238	  /* VPP-TBD */
3239	  *(size_t *) buffer = (session->sndbuf_size ? session->sndbuf_size :
3240				session->tx_fifo ?
3241				svm_fifo_size (session->tx_fifo) :
3242				vcm->cfg.tx_fifo_size);
3243	  *buflen = sizeof (u32);
3244
3245	  VDBG (2, "VPPCOM_ATTR_GET_TX_FIFO_LEN: %u (0x%x), buflen %d,"
3246		" #VPP-TBD#", *(size_t *) buffer, *(size_t *) buffer,
3247		*buflen);
3248	}
3249      else
3250	rv = VPPCOM_EINVAL;
3251      break;
3252
3253    case VPPCOM_ATTR_SET_TX_FIFO_LEN:
3254      if (buffer && buflen && (*buflen == sizeof (u32)))
3255	{
3256	  /* VPP-TBD */
3257	  session->sndbuf_size = *(u32 *) buffer;
3258	  VDBG (2, "VPPCOM_ATTR_SET_TX_FIFO_LEN: %u (0x%x), buflen %d,"
3259		" #VPP-TBD#", session->sndbuf_size, session->sndbuf_size,
3260		*buflen);
3261	}
3262      else
3263	rv = VPPCOM_EINVAL;
3264      break;
3265
3266    case VPPCOM_ATTR_GET_RX_FIFO_LEN:
3267      if (buffer && buflen && (*buflen >= sizeof (u32)))
3268	{
3269
3270	  /* VPP-TBD */
3271	  *(size_t *) buffer = (session->rcvbuf_size ? session->rcvbuf_size :
3272				session->rx_fifo ?
3273				svm_fifo_size (session->rx_fifo) :
3274				vcm->cfg.rx_fifo_size);
3275	  *buflen = sizeof (u32);
3276
3277	  VDBG (2, "VPPCOM_ATTR_GET_RX_FIFO_LEN: %u (0x%x), buflen %d, "
3278		"#VPP-TBD#", *(size_t *) buffer, *(size_t *) buffer, *buflen);
3279	}
3280      else
3281	rv = VPPCOM_EINVAL;
3282      break;
3283
3284    case VPPCOM_ATTR_SET_RX_FIFO_LEN:
3285      if (buffer && buflen && (*buflen == sizeof (u32)))
3286	{
3287	  /* VPP-TBD */
3288	  session->rcvbuf_size = *(u32 *) buffer;
3289	  VDBG (2, "VPPCOM_ATTR_SET_RX_FIFO_LEN: %u (0x%x), buflen %d,"
3290		" #VPP-TBD#", session->sndbuf_size, session->sndbuf_size,
3291		*buflen);
3292	}
3293      else
3294	rv = VPPCOM_EINVAL;
3295      break;
3296
3297    case VPPCOM_ATTR_GET_REUSEADDR:
3298      if (buffer && buflen && (*buflen >= sizeof (int)))
3299	{
3300	  /* VPP-TBD */
3301	  *(int *) buffer = VCL_SESS_ATTR_TEST (session->attr,
3302						VCL_SESS_ATTR_REUSEADDR);
3303	  *buflen = sizeof (int);
3304
3305	  VDBG (2, "VPPCOM_ATTR_GET_REUSEADDR: %d, buflen %d, #VPP-TBD#",
3306		*(int *) buffer, *buflen);
3307	}
3308      else
3309	rv = VPPCOM_EINVAL;
3310      break;
3311
3312    case VPPCOM_ATTR_SET_REUSEADDR:
3313      if (buffer && buflen && (*buflen == sizeof (int)) &&
3314	  !VCL_SESS_ATTR_TEST (session->attr, VCL_SESS_ATTR_LISTEN))
3315	{
3316	  /* VPP-TBD */
3317	  if (*(int *) buffer)
3318	    VCL_SESS_ATTR_SET (session->attr, VCL_SESS_ATTR_REUSEADDR);
3319	  else
3320	    VCL_SESS_ATTR_CLR (session->attr, VCL_SESS_ATTR_REUSEADDR);
3321
3322	  VDBG (2, "VPPCOM_ATTR_SET_REUSEADDR: %d, buflen %d, #VPP-TBD#",
3323		VCL_SESS_ATTR_TEST (session->attr, VCL_SESS_ATTR_REUSEADDR),
3324		*buflen);
3325	}
3326      else
3327	rv = VPPCOM_EINVAL;
3328      break;
3329
3330    case VPPCOM_ATTR_GET_REUSEPORT:
3331      if (buffer && buflen && (*buflen >= sizeof (int)))
3332	{
3333	  /* VPP-TBD */
3334	  *(int *) buffer = VCL_SESS_ATTR_TEST (session->attr,
3335						VCL_SESS_ATTR_REUSEPORT);
3336	  *buflen = sizeof (int);
3337
3338	  VDBG (2, "VPPCOM_ATTR_GET_REUSEPORT: %d, buflen %d, #VPP-TBD#",
3339		*(int *) buffer, *buflen);
3340	}
3341      else
3342	rv = VPPCOM_EINVAL;
3343      break;
3344
3345    case VPPCOM_ATTR_SET_REUSEPORT:
3346      if (buffer && buflen && (*buflen == sizeof (int)) &&
3347	  !VCL_SESS_ATTR_TEST (session->attr, VCL_SESS_ATTR_LISTEN))
3348	{
3349	  /* VPP-TBD */
3350	  if (*(int *) buffer)
3351	    VCL_SESS_ATTR_SET (session->attr, VCL_SESS_ATTR_REUSEPORT);
3352	  else
3353	    VCL_SESS_ATTR_CLR (session->attr, VCL_SESS_ATTR_REUSEPORT);
3354
3355	  VDBG (2, "VPPCOM_ATTR_SET_REUSEPORT: %d, buflen %d, #VPP-TBD#",
3356		VCL_SESS_ATTR_TEST (session->attr, VCL_SESS_ATTR_REUSEPORT),
3357		*buflen);
3358	}
3359      else
3360	rv = VPPCOM_EINVAL;
3361      break;
3362
3363    case VPPCOM_ATTR_GET_BROADCAST:
3364      if (buffer && buflen && (*buflen >= sizeof (int)))
3365	{
3366	  /* VPP-TBD */
3367	  *(int *) buffer = VCL_SESS_ATTR_TEST (session->attr,
3368						VCL_SESS_ATTR_BROADCAST);
3369	  *buflen = sizeof (int);
3370
3371	  VDBG (2, "VPPCOM_ATTR_GET_BROADCAST: %d, buflen %d, #VPP-TBD#",
3372		*(int *) buffer, *buflen);
3373	}
3374      else
3375	rv = VPPCOM_EINVAL;
3376      break;
3377
3378    case VPPCOM_ATTR_SET_BROADCAST:
3379      if (buffer && buflen && (*buflen == sizeof (int)))
3380	{
3381	  /* VPP-TBD */
3382	  if (*(int *) buffer)
3383	    VCL_SESS_ATTR_SET (session->attr, VCL_SESS_ATTR_BROADCAST);
3384	  else
3385	    VCL_SESS_ATTR_CLR (session->attr, VCL_SESS_ATTR_BROADCAST);
3386
3387	  VDBG (2, "VPPCOM_ATTR_SET_BROADCAST: %d, buflen %d, #VPP-TBD#",
3388		VCL_SESS_ATTR_TEST (session->attr, VCL_SESS_ATTR_BROADCAST),
3389		*buflen);
3390	}
3391      else
3392	rv = VPPCOM_EINVAL;
3393      break;
3394
3395    case VPPCOM_ATTR_GET_V6ONLY:
3396      if (buffer && buflen && (*buflen >= sizeof (int)))
3397	{
3398	  /* VPP-TBD */
3399	  *(int *) buffer = VCL_SESS_ATTR_TEST (session->attr,
3400						VCL_SESS_ATTR_V6ONLY);
3401	  *buflen = sizeof (int);
3402
3403	  VDBG (2, "VPPCOM_ATTR_GET_V6ONLY: %d, buflen %d, #VPP-TBD#",
3404		*(int *) buffer, *buflen);
3405	}
3406      else
3407	rv = VPPCOM_EINVAL;
3408      break;
3409
3410    case VPPCOM_ATTR_SET_V6ONLY:
3411      if (buffer && buflen && (*buflen == sizeof (int)))
3412	{
3413	  /* VPP-TBD */
3414	  if (*(int *) buffer)
3415	    VCL_SESS_ATTR_SET (session->attr, VCL_SESS_ATTR_V6ONLY);
3416	  else
3417	    VCL_SESS_ATTR_CLR (session->attr, VCL_SESS_ATTR_V6ONLY);
3418
3419	  VDBG (2, "VPPCOM_ATTR_SET_V6ONLY: %d, buflen %d, #VPP-TBD#",
3420		VCL_SESS_ATTR_TEST (session->attr, VCL_SESS_ATTR_V6ONLY),
3421		*buflen);
3422	}
3423      else
3424	rv = VPPCOM_EINVAL;
3425      break;
3426
3427    case VPPCOM_ATTR_GET_KEEPALIVE:
3428      if (buffer && buflen && (*buflen >= sizeof (int)))
3429	{
3430	  /* VPP-TBD */
3431	  *(int *) buffer = VCL_SESS_ATTR_TEST (session->attr,
3432						VCL_SESS_ATTR_KEEPALIVE);
3433	  *buflen = sizeof (int);
3434
3435	  VDBG (2, "VPPCOM_ATTR_GET_KEEPALIVE: %d, buflen %d, #VPP-TBD#",
3436		*(int *) buffer, *buflen);
3437	}
3438      else
3439	rv = VPPCOM_EINVAL;
3440      break;
3441
3442    case VPPCOM_ATTR_SET_KEEPALIVE:
3443      if (buffer && buflen && (*buflen == sizeof (int)))
3444	{
3445	  /* VPP-TBD */
3446	  if (*(int *) buffer)
3447	    VCL_SESS_ATTR_SET (session->attr, VCL_SESS_ATTR_KEEPALIVE);
3448	  else
3449	    VCL_SESS_ATTR_CLR (session->attr, VCL_SESS_ATTR_KEEPALIVE);
3450
3451	  VDBG (2, "VPPCOM_ATTR_SET_KEEPALIVE: %d, buflen %d, #VPP-TBD#",
3452		VCL_SESS_ATTR_TEST (session->attr, VCL_SESS_ATTR_KEEPALIVE),
3453		*buflen);
3454	}
3455      else
3456	rv = VPPCOM_EINVAL;
3457      break;
3458
3459    case VPPCOM_ATTR_GET_TCP_NODELAY:
3460      if (buffer && buflen && (*buflen >= sizeof (int)))
3461	{
3462	  /* VPP-TBD */
3463	  *(int *) buffer = VCL_SESS_ATTR_TEST (session->attr,
3464						VCL_SESS_ATTR_TCP_NODELAY);
3465	  *buflen = sizeof (int);
3466
3467	  VDBG (2, "VPPCOM_ATTR_GET_TCP_NODELAY: %d, buflen %d, #VPP-TBD#",
3468		*(int *) buffer, *buflen);
3469	}
3470      else
3471	rv = VPPCOM_EINVAL;
3472      break;
3473
3474    case VPPCOM_ATTR_SET_TCP_NODELAY:
3475      if (buffer && buflen && (*buflen == sizeof (int)))
3476	{
3477	  /* VPP-TBD */
3478	  if (*(int *) buffer)
3479	    VCL_SESS_ATTR_SET (session->attr, VCL_SESS_ATTR_TCP_NODELAY);
3480	  else
3481	    VCL_SESS_ATTR_CLR (session->attr, VCL_SESS_ATTR_TCP_NODELAY);
3482
3483	  VDBG (2, "VPPCOM_ATTR_SET_TCP_NODELAY: %d, buflen %d, #VPP-TBD#",
3484		VCL_SESS_ATTR_TEST (session->attr, VCL_SESS_ATTR_TCP_NODELAY),
3485		*buflen);
3486	}
3487      else
3488	rv = VPPCOM_EINVAL;
3489      break;
3490
3491    case VPPCOM_ATTR_GET_TCP_KEEPIDLE:
3492      if (buffer && buflen && (*buflen >= sizeof (int)))
3493	{
3494	  /* VPP-TBD */
3495	  *(int *) buffer = VCL_SESS_ATTR_TEST (session->attr,
3496						VCL_SESS_ATTR_TCP_KEEPIDLE);
3497	  *buflen = sizeof (int);
3498
3499	  VDBG (2, "VPPCOM_ATTR_GET_TCP_KEEPIDLE: %d, buflen %d, #VPP-TBD#",
3500		*(int *) buffer, *buflen);
3501	}
3502      else
3503	rv = VPPCOM_EINVAL;
3504      break;
3505
3506    case VPPCOM_ATTR_SET_TCP_KEEPIDLE:
3507      if (buffer && buflen && (*buflen == sizeof (int)))
3508	{
3509	  /* VPP-TBD */
3510	  if (*(int *) buffer)
3511	    VCL_SESS_ATTR_SET (session->attr, VCL_SESS_ATTR_TCP_KEEPIDLE);
3512	  else
3513	    VCL_SESS_ATTR_CLR (session->attr, VCL_SESS_ATTR_TCP_KEEPIDLE);
3514
3515	  VDBG (2, "VPPCOM_ATTR_SET_TCP_KEEPIDLE: %d, buflen %d, #VPP-TBD#",
3516		VCL_SESS_ATTR_TEST (session->attr,
3517				    VCL_SESS_ATTR_TCP_KEEPIDLE), *buflen);
3518	}
3519      else
3520	rv = VPPCOM_EINVAL;
3521      break;
3522
3523    case VPPCOM_ATTR_GET_TCP_KEEPINTVL:
3524      if (buffer && buflen && (*buflen >= sizeof (int)))
3525	{
3526	  /* VPP-TBD */
3527	  *(int *) buffer = VCL_SESS_ATTR_TEST (session->attr,
3528						VCL_SESS_ATTR_TCP_KEEPINTVL);
3529	  *buflen = sizeof (int);
3530
3531	  VDBG (2, "VPPCOM_ATTR_GET_TCP_KEEPINTVL: %d, buflen %d, #VPP-TBD#",
3532		*(int *) buffer, *buflen);
3533	}
3534      else
3535	rv = VPPCOM_EINVAL;
3536      break;
3537
3538    case VPPCOM_ATTR_SET_TCP_KEEPINTVL:
3539      if (buffer && buflen && (*buflen == sizeof (int)))
3540	{
3541	  /* VPP-TBD */
3542	  if (*(int *) buffer)
3543	    VCL_SESS_ATTR_SET (session->attr, VCL_SESS_ATTR_TCP_KEEPINTVL);
3544	  else
3545	    VCL_SESS_ATTR_CLR (session->attr, VCL_SESS_ATTR_TCP_KEEPINTVL);
3546
3547	  VDBG (2, "VPPCOM_ATTR_SET_TCP_KEEPINTVL: %d, buflen %d, #VPP-TBD#",
3548		VCL_SESS_ATTR_TEST (session->attr,
3549				    VCL_SESS_ATTR_TCP_KEEPINTVL), *buflen);
3550	}
3551      else
3552	rv = VPPCOM_EINVAL;
3553      break;
3554
3555    case VPPCOM_ATTR_GET_TCP_USER_MSS:
3556      if (buffer && buflen && (*buflen >= sizeof (u32)))
3557	{
3558	  /* VPP-TBD */
3559	  *(u32 *) buffer = session->user_mss;
3560	  *buflen = sizeof (int);
3561
3562	  VDBG (2, "VPPCOM_ATTR_GET_TCP_USER_MSS: %d, buflen %d, #VPP-TBD#",
3563		*(int *) buffer, *buflen);
3564	}
3565      else
3566	rv = VPPCOM_EINVAL;
3567      break;
3568
3569    case VPPCOM_ATTR_SET_TCP_USER_MSS:
3570      if (buffer && buflen && (*buflen == sizeof (u32)))
3571	{
3572	  /* VPP-TBD */
3573	  session->user_mss = *(u32 *) buffer;
3574
3575	  VDBG (2, "VPPCOM_ATTR_SET_TCP_USER_MSS: %u, buflen %d, #VPP-TBD#",
3576		session->user_mss, *buflen);
3577	}
3578      else
3579	rv = VPPCOM_EINVAL;
3580      break;
3581
3582    case VPPCOM_ATTR_SET_SHUT:
3583      if (*flags == SHUT_RD || *flags == SHUT_RDWR)
3584	VCL_SESS_ATTR_SET (session->attr, VCL_SESS_ATTR_SHUT_RD);
3585      if (*flags == SHUT_WR || *flags == SHUT_RDWR)
3586	VCL_SESS_ATTR_SET (session->attr, VCL_SESS_ATTR_SHUT_WR);
3587      break;
3588
3589    case VPPCOM_ATTR_GET_SHUT:
3590      if (VCL_SESS_ATTR_TEST (session->attr, VCL_SESS_ATTR_SHUT_RD))
3591	tmp_flags = 1;
3592      if (VCL_SESS_ATTR_TEST (session->attr, VCL_SESS_ATTR_SHUT_WR))
3593	tmp_flags |= 2;
3594      if (tmp_flags == 1)
3595	*(int *) buffer = SHUT_RD;
3596      else if (tmp_flags == 2)
3597	*(int *) buffer = SHUT_WR;
3598      else if (tmp_flags == 3)
3599	*(int *) buffer = SHUT_RDWR;
3600      *buflen = sizeof (int);
3601      break;
3602
3603    case VPPCOM_ATTR_SET_CONNECTED:
3604      session->flags |= VCL_SESSION_F_CONNECTED;
3605      break;
3606
3607    default:
3608      rv = VPPCOM_EINVAL;
3609      break;
3610    }
3611
3612  return rv;
3613}
3614
3615int
3616vppcom_session_recvfrom (uint32_t session_handle, void *buffer,
3617			 uint32_t buflen, int flags, vppcom_endpt_t * ep)
3618{
3619  vcl_worker_t *wrk = vcl_worker_get_current ();
3620  vcl_session_t *session;
3621  int rv = VPPCOM_OK;
3622
3623  if (flags == 0)
3624    rv = vppcom_session_read (session_handle, buffer, buflen);
3625  else if (flags & MSG_PEEK)
3626    rv = vppcom_session_peek (session_handle, buffer, buflen);
3627  else
3628    {
3629      VDBG (0, "Unsupport flags for recvfrom %d", flags);
3630      return VPPCOM_EAFNOSUPPORT;
3631    }
3632
3633  if (ep && rv > 0)
3634    {
3635      session = vcl_session_get_w_handle (wrk, session_handle);
3636      if (session->transport.is_ip4)
3637	clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip4,
3638			  sizeof (ip4_address_t));
3639      else
3640	clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip6,
3641			  sizeof (ip6_address_t));
3642      ep->is_ip4 = session->transport.is_ip4;
3643      ep->port = session->transport.rmt_port;
3644    }
3645
3646  return rv;
3647}
3648
3649int
3650vppcom_session_sendto (uint32_t session_handle, void *buffer,
3651		       uint32_t buflen, int flags, vppcom_endpt_t * ep)
3652{
3653  vcl_worker_t *wrk = vcl_worker_get_current ();
3654  vcl_session_t *s;
3655
3656  s = vcl_session_get_w_handle (wrk, session_handle);
3657  if (!s)
3658    return VPPCOM_EBADFD;
3659
3660  if (!buffer)
3661    return VPPCOM_EINVAL;
3662
3663  if (ep)
3664    {
3665      if (s->session_type != VPPCOM_PROTO_UDP
3666	  || (s->flags & VCL_SESSION_F_CONNECTED))
3667	return VPPCOM_EINVAL;
3668
3669      /* Session not connected/bound in vpp. Create it by 'connecting' it */
3670      if (PREDICT_FALSE (s->session_state == STATE_CLOSED))
3671	{
3672	  vcl_send_session_connect (wrk, s);
3673	}
3674      else
3675	{
3676	  s->transport.is_ip4 = ep->is_ip4;
3677	  s->transport.rmt_port = ep->port;
3678	  vcl_ip_copy_from_ep (&s->transport.rmt_ip, ep);
3679	}
3680    }
3681
3682  if (flags)
3683    {
3684      // TBD check the flags and do the right thing
3685      VDBG (2, "handling flags 0x%u (%d) not implemented yet.", flags, flags);
3686    }
3687
3688  return (vppcom_session_write_inline (wrk, s, buffer, buflen, 1,
3689				       s->is_dgram ? 1 : 0));
3690}
3691
3692int
3693vppcom_poll (vcl_poll_t * vp, uint32_t n_sids, double wait_for_time)
3694{
3695  vcl_worker_t *wrk = vcl_worker_get_current ();
3696  f64 timeout = clib_time_now (&wrk->clib_time) + wait_for_time;
3697  u32 i, keep_trying = 1;
3698  svm_msg_q_msg_t msg;
3699  session_event_t *e;
3700  int rv, num_ev = 0;
3701
3702  VDBG (3, "vp %p, nsids %u, wait_for_time %f", vp, n_sids, wait_for_time);
3703
3704  if (!vp)
3705    return VPPCOM_EFAULT;
3706
3707  do
3708    {
3709      vcl_session_t *session;
3710
3711      /* Dequeue all events and drop all unhandled io events */
3712      while (svm_msg_q_sub (wrk->app_event_queue, &msg, SVM_Q_NOWAIT, 0) == 0)
3713	{
3714	  e = svm_msg_q_msg_data (wrk->app_event_queue, &msg);
3715	  vcl_handle_mq_event (wrk, e);
3716	  svm_msg_q_free_msg (wrk->app_event_queue, &msg);
3717	}
3718      vec_reset_length (wrk->unhandled_evts_vector);
3719
3720      for (i = 0; i < n_sids; i++)
3721	{
3722	  session = vcl_session_get (wrk, vp[i].sh);
3723	  if (!session)
3724	    {
3725	      vp[i].revents = POLLHUP;
3726	      num_ev++;
3727	      continue;
3728	    }
3729
3730	  vp[i].revents = 0;
3731
3732	  if (POLLIN & vp[i].events)
3733	    {
3734	      rv = vcl_session_read_ready (session);
3735	      if (rv > 0)
3736		{
3737		  vp[i].revents |= POLLIN;
3738		  num_ev++;
3739		}
3740	      else if (rv < 0)
3741		{
3742		  switch (rv)
3743		    {
3744		    case VPPCOM_ECONNRESET:
3745		      vp[i].revents = POLLHUP;
3746		      break;
3747
3748		    default:
3749		      vp[i].revents = POLLERR;
3750		      break;
3751		    }
3752		  num_ev++;
3753		}
3754	    }
3755
3756	  if (POLLOUT & vp[i].events)
3757	    {
3758	      rv = vcl_session_write_ready (session);
3759	      if (rv > 0)
3760		{
3761		  vp[i].revents |= POLLOUT;
3762		  num_ev++;
3763		}
3764	      else if (rv < 0)
3765		{
3766		  switch (rv)
3767		    {
3768		    case VPPCOM_ECONNRESET:
3769		      vp[i].revents = POLLHUP;
3770		      break;
3771
3772		    default:
3773		      vp[i].revents = POLLERR;
3774		      break;
3775		    }
3776		  num_ev++;
3777		}
3778	    }
3779
3780	  if (0)		// Note "done:" label used by VCL_SESSION_LOCK_AND_GET()
3781	    {
3782	      vp[i].revents = POLLNVAL;
3783	      num_ev++;
3784	    }
3785	}
3786      if (wait_for_time != -1)
3787	keep_trying = (clib_time_now (&wrk->clib_time) <= timeout) ? 1 : 0;
3788    }
3789  while ((num_ev == 0) && keep_trying);
3790
3791  return num_ev;
3792}
3793
3794int
3795vppcom_mq_epoll_fd (void)
3796{
3797  vcl_worker_t *wrk = vcl_worker_get_current ();
3798  return wrk->mqs_epfd;
3799}
3800
3801int
3802vppcom_session_index (vcl_session_handle_t session_handle)
3803{
3804  return session_handle & 0xFFFFFF;
3805}
3806
3807int
3808vppcom_session_worker (vcl_session_handle_t session_handle)
3809{
3810  return session_handle >> 24;
3811}
3812
3813int
3814vppcom_worker_register (void)
3815{
3816  vcl_worker_t *wrk;
3817  u8 *wrk_name = 0;
3818  int rv;
3819
3820  if (!vcl_worker_alloc_and_init ())
3821    return VPPCOM_EEXIST;
3822
3823  wrk = vcl_worker_get_current ();
3824  wrk_name = format (0, "%s-wrk-%u", vcm->app_name, wrk->wrk_index);
3825
3826  rv = vppcom_connect_to_vpp ((char *) wrk_name);
3827  vec_free (wrk_name);
3828
3829  if (rv)
3830    return VPPCOM_EFAULT;
3831
3832  if (vcl_worker_register_with_vpp ())
3833    return VPPCOM_EEXIST;
3834
3835  return VPPCOM_OK;
3836}
3837
3838void
3839vppcom_worker_unregister (void)
3840{
3841  vcl_worker_cleanup (vcl_worker_get_current (), 1 /* notify vpp */ );
3842  vcl_set_worker_index (~0);
3843}
3844
3845void
3846vppcom_worker_index_set (int index)
3847{
3848  vcl_set_worker_index (index);
3849}
3850
3851int
3852vppcom_worker_index (void)
3853{
3854  return vcl_get_worker_index ();
3855}
3856
3857int
3858vppcom_worker_mqs_epfd (void)
3859{
3860  vcl_worker_t *wrk = vcl_worker_get_current ();
3861  if (!vcm->cfg.use_mq_eventfd)
3862    return -1;
3863  return wrk->mqs_epfd;
3864}
3865
3866int
3867vppcom_session_is_connectable_listener (uint32_t session_handle)
3868{
3869  vcl_session_t *session;
3870  vcl_worker_t *wrk = vcl_worker_get_current ();
3871  session = vcl_session_get_w_handle (wrk, session_handle);
3872  if (!session)
3873    return VPPCOM_EBADFD;
3874  return vcl_session_is_connectable_listener (wrk, session);
3875}
3876
3877int
3878vppcom_session_listener (uint32_t session_handle)
3879{
3880  vcl_worker_t *wrk = vcl_worker_get_current ();
3881  vcl_session_t *listen_session, *session;
3882  session = vcl_session_get_w_handle (wrk, session_handle);
3883  if (!session)
3884    return VPPCOM_EBADFD;
3885  if (session->listener_index == VCL_INVALID_SESSION_INDEX)
3886    return VPPCOM_EBADFD;
3887  listen_session = vcl_session_get_w_handle (wrk, session->listener_index);
3888  if (!listen_session)
3889    return VPPCOM_EBADFD;
3890  return vcl_session_handle (listen_session);
3891}
3892
3893int
3894vppcom_session_n_accepted (uint32_t session_handle)
3895{
3896  vcl_worker_t *wrk = vcl_worker_get_current ();
3897  vcl_session_t *session = vcl_session_get_w_handle (wrk, session_handle);
3898  if (!session)
3899    return VPPCOM_EBADFD;
3900  return session->n_accepted_sessions;
3901}
3902
3903/*
3904 * fd.io coding-style-patch-verification: ON
3905 *
3906 * Local Variables:
3907 * eval: (c-set-style "gnu")
3908 * End:
3909 */
3910