memory_api.c revision 0aa4013e
1/*
2 *------------------------------------------------------------------
3 * Copyright (c) 2018 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
16 */
17#include <signal.h>
18
19#include <vlib/vlib.h>
20#include <vlibapi/api.h>
21#include <vlibmemory/api.h>
22#include <vlibmemory/memory_api.h>
23
24#include <vlibmemory/vl_memory_msg_enum.h>	/* enumerate all vlib messages */
25
26#define vl_typedefs		/* define message structures */
27#include <vlibmemory/vl_memory_api_h.h>
28#undef vl_typedefs
29
30/* instantiate all the print functions we know about */
31#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
32#define vl_printfun
33#include <vlibmemory/vl_memory_api_h.h>
34#undef vl_printfun
35
36/* instantiate all the endian swap functions we know about */
37#define vl_endianfun
38#include <vlibmemory/vl_memory_api_h.h>
39#undef vl_endianfun
40
41static inline void *
42vl_api_memclnt_create_t_print (vl_api_memclnt_create_t * a, void *handle)
43{
44  vl_print (handle, "vl_api_memclnt_create_t:\n");
45  vl_print (handle, "name: %s\n", a->name);
46  vl_print (handle, "input_queue: 0x%wx\n", a->input_queue);
47  vl_print (handle, "context: %u\n", (unsigned) a->context);
48  vl_print (handle, "ctx_quota: %ld\n", (long) a->ctx_quota);
49  return handle;
50}
51
52static inline void *
53vl_api_memclnt_delete_t_print (vl_api_memclnt_delete_t * a, void *handle)
54{
55  vl_print (handle, "vl_api_memclnt_delete_t:\n");
56  vl_print (handle, "index: %u\n", (unsigned) a->index);
57  vl_print (handle, "handle: 0x%wx\n", a->handle);
58  return handle;
59}
60
61volatile int **vl_api_queue_cursizes;
62
63static void
64memclnt_queue_callback (vlib_main_t * vm)
65{
66  int i;
67  api_main_t *am = vlibapi_get_main ();
68
69  if (PREDICT_FALSE (vec_len (vl_api_queue_cursizes) !=
70		     1 + vec_len (am->vlib_private_rps)))
71    {
72      vl_shmem_hdr_t *shmem_hdr = am->shmem_hdr;
73      svm_queue_t *q;
74
75      if (shmem_hdr == 0)
76	return;
77
78      q = shmem_hdr->vl_input_queue;
79      if (q == 0)
80	return;
81
82      vec_add1 (vl_api_queue_cursizes, &q->cursize);
83
84      for (i = 0; i < vec_len (am->vlib_private_rps); i++)
85	{
86	  svm_region_t *vlib_rp = am->vlib_private_rps[i];
87
88	  shmem_hdr = (void *) vlib_rp->user_ctx;
89	  q = shmem_hdr->vl_input_queue;
90	  vec_add1 (vl_api_queue_cursizes, &q->cursize);
91	}
92    }
93
94  for (i = 0; i < vec_len (vl_api_queue_cursizes); i++)
95    {
96      if (*vl_api_queue_cursizes[i])
97	{
98	  vm->queue_signal_pending = 1;
99	  vm->api_queue_nonempty = 1;
100	  vlib_process_signal_event (vm, vl_api_clnt_node.index,
101				     /* event_type */ QUEUE_SIGNAL_EVENT,
102				     /* event_data */ 0);
103	  break;
104	}
105    }
106  if (vec_len (vm->pending_rpc_requests))
107    {
108      vm->queue_signal_pending = 1;
109      vm->api_queue_nonempty = 1;
110      vlib_process_signal_event (vm, vl_api_clnt_node.index,
111				 /* event_type */ QUEUE_SIGNAL_EVENT,
112				 /* event_data */ 0);
113    }
114}
115
116/*
117 * vl_api_memclnt_create_internal
118 */
119u32
120vl_api_memclnt_create_internal (char *name, svm_queue_t * q)
121{
122  vl_api_registration_t **regpp;
123  vl_api_registration_t *regp;
124  void *oldheap;
125  api_main_t *am = vlibapi_get_main ();
126
127  ASSERT (vlib_get_thread_index () == 0);
128  pool_get (am->vl_clients, regpp);
129
130
131  oldheap = vl_msg_push_heap ();
132  *regpp = clib_mem_alloc (sizeof (vl_api_registration_t));
133
134  regp = *regpp;
135  clib_memset (regp, 0, sizeof (*regp));
136  regp->registration_type = REGISTRATION_TYPE_SHMEM;
137  regp->vl_api_registration_pool_index = regpp - am->vl_clients;
138  regp->vlib_rp = am->vlib_rp;
139  regp->shmem_hdr = am->shmem_hdr;
140
141  regp->vl_input_queue = q;
142  regp->name = format (0, "%s%c", name, 0);
143
144  vl_msg_pop_heap (oldheap);
145  return vl_msg_api_handle_from_index_and_epoch
146    (regp->vl_api_registration_pool_index,
147     am->shmem_hdr->application_restarts);
148}
149
150/*
151 * vl_api_memclnt_create_t_handler
152 */
153void
154vl_api_memclnt_create_t_handler (vl_api_memclnt_create_t * mp)
155{
156  vl_api_registration_t **regpp;
157  vl_api_registration_t *regp;
158  vl_api_memclnt_create_reply_t *rp;
159  svm_queue_t *q;
160  int rv = 0;
161  void *oldheap;
162  api_main_t *am = vlibapi_get_main ();
163  u8 *msg_table;
164
165  /*
166   * This is tortured. Maintain a vlib-address-space private
167   * pool of client registrations. We use the shared-memory virtual
168   * address of client structure as a handle, to allow direct
169   * manipulation of context quota vbls from the client library.
170   *
171   * This scheme causes trouble w/ API message trace replay, since
172   * some random VA from clib_mem_alloc() certainly won't
173   * occur in the Linux sim. The (very) few places
174   * that care need to use the pool index.
175   *
176   * Putting the registration object(s) into a pool in shared memory and
177   * using the pool index as a handle seems like a great idea.
178   * Unfortunately, each and every reference to that pool would need
179   * to be protected by a mutex:
180   *
181   *     Client                      VLIB
182   *     ------                      ----
183   *     convert pool index to
184   *     pointer.
185   *     <deschedule>
186   *                                 expand pool
187   *                                 <deschedule>
188   *     kaboom!
189   */
190
191  pool_get (am->vl_clients, regpp);
192
193  oldheap = vl_msg_push_heap ();
194  *regpp = clib_mem_alloc (sizeof (vl_api_registration_t));
195
196  regp = *regpp;
197  clib_memset (regp, 0, sizeof (*regp));
198  regp->registration_type = REGISTRATION_TYPE_SHMEM;
199  regp->vl_api_registration_pool_index = regpp - am->vl_clients;
200  regp->vlib_rp = am->vlib_rp;
201  regp->shmem_hdr = am->shmem_hdr;
202  regp->clib_file_index = am->shmem_hdr->clib_file_index;
203
204  q = regp->vl_input_queue = (svm_queue_t *) (uword) mp->input_queue;
205  VL_MSG_API_SVM_QUEUE_UNPOISON (q);
206
207  regp->name = format (0, "%s", mp->name);
208  vec_add1 (regp->name, 0);
209
210  if (am->serialized_message_table_in_shmem == 0)
211    am->serialized_message_table_in_shmem =
212      vl_api_serialize_message_table (am, 0);
213
214  if (am->vlib_rp != am->vlib_primary_rp)
215    msg_table = vl_api_serialize_message_table (am, 0);
216  else
217    msg_table = am->serialized_message_table_in_shmem;
218
219  vl_msg_pop_heap (oldheap);
220
221  rp = vl_msg_api_alloc (sizeof (*rp));
222  rp->_vl_msg_id = ntohs (VL_API_MEMCLNT_CREATE_REPLY);
223  rp->handle = (uword) regp;
224  rp->index = vl_msg_api_handle_from_index_and_epoch
225    (regp->vl_api_registration_pool_index,
226     am->shmem_hdr->application_restarts);
227  rp->context = mp->context;
228  rp->response = ntohl (rv);
229  rp->message_table = pointer_to_uword (msg_table);
230
231  vl_msg_api_send_shmem (q, (u8 *) & rp);
232}
233
234int
235vl_api_call_reaper_functions (u32 client_index)
236{
237  clib_error_t *error = 0;
238  _vl_msg_api_function_list_elt_t *i;
239
240  i = vlibapi_get_main ()->reaper_function_registrations;
241  while (i)
242    {
243      error = i->f (client_index);
244      if (error)
245	clib_error_report (error);
246      i = i->next_init_function;
247    }
248  return 0;
249}
250
251/*
252 * vl_api_memclnt_delete_t_handler
253 */
254void
255vl_api_memclnt_delete_t_handler (vl_api_memclnt_delete_t * mp)
256{
257  vl_api_registration_t **regpp;
258  vl_api_registration_t *regp;
259  vl_api_memclnt_delete_reply_t *rp;
260  void *oldheap;
261  api_main_t *am = vlibapi_get_main ();
262  u32 handle, client_index, epoch;
263
264  handle = mp->index;
265
266  if (vl_api_call_reaper_functions (handle))
267    return;
268
269  epoch = vl_msg_api_handle_get_epoch (handle);
270  client_index = vl_msg_api_handle_get_index (handle);
271
272  if (epoch != (am->shmem_hdr->application_restarts & VL_API_EPOCH_MASK))
273    {
274      clib_warning
275	("Stale clnt delete index %d old epoch %d cur epoch %d",
276	 client_index, epoch,
277	 (am->shmem_hdr->application_restarts & VL_API_EPOCH_MASK));
278      return;
279    }
280
281  regpp = pool_elt_at_index (am->vl_clients, client_index);
282
283  if (!pool_is_free (am->vl_clients, regpp))
284    {
285      int i;
286      regp = *regpp;
287      int private_registration = 0;
288
289      /* Send reply unless client asked us to do the cleanup */
290      if (!mp->do_cleanup)
291	{
292	  /*
293	   * Note: the API message handling path will set am->vlib_rp
294	   * as appropriate for pairwise / private memory segments
295	   */
296	  rp = vl_msg_api_alloc (sizeof (*rp));
297	  rp->_vl_msg_id = ntohs (VL_API_MEMCLNT_DELETE_REPLY);
298	  rp->handle = mp->handle;
299	  rp->response = 1;
300
301	  vl_msg_api_send_shmem (regp->vl_input_queue, (u8 *) & rp);
302	  if (client_index != regp->vl_api_registration_pool_index)
303	    {
304	      clib_warning ("mismatch client_index %d pool_index %d",
305			    client_index,
306			    regp->vl_api_registration_pool_index);
307	      vl_msg_api_free (rp);
308	      return;
309	    }
310	}
311
312      /* No dangling references, please */
313      *regpp = 0;
314
315      /* For horizontal scaling, add a hash table... */
316      for (i = 0; i < vec_len (am->vlib_private_rps); i++)
317	{
318	  /* Is this a pairwise / private API segment? */
319	  if (am->vlib_private_rps[i] == am->vlib_rp)
320	    {
321	      /* Note: account for the memfd header page */
322	      uword virtual_base = am->vlib_rp->virtual_base - MMAP_PAGESIZE;
323	      uword virtual_size = am->vlib_rp->virtual_size + MMAP_PAGESIZE;
324
325	      /*
326	       * Kill the registration pool element before we make
327	       * the index vanish forever
328	       */
329	      pool_put_index (am->vl_clients,
330			      regp->vl_api_registration_pool_index);
331
332	      vec_delete (am->vlib_private_rps, 1, i);
333	      /* Kill it, accounting for the memfd header page */
334	      if (munmap ((void *) virtual_base, virtual_size) < 0)
335		clib_unix_warning ("munmap");
336	      /* Reset the queue-length-address cache */
337	      vec_reset_length (vl_api_queue_cursizes);
338	      private_registration = 1;
339	      break;
340	    }
341	}
342
343      if (private_registration == 0)
344	{
345	  pool_put_index (am->vl_clients,
346			  regp->vl_api_registration_pool_index);
347	  oldheap = vl_msg_push_heap ();
348	  if (mp->do_cleanup)
349	    svm_queue_free (regp->vl_input_queue);
350	  vec_free (regp->name);
351	  /* Poison the old registration */
352	  clib_memset (regp, 0xF1, sizeof (*regp));
353	  clib_mem_free (regp);
354	  vl_msg_pop_heap (oldheap);
355	  /*
356	   * These messages must be freed manually, since they're set up
357	   * as "bounce" messages. In the private_registration == 1 case,
358	   * we kill the shared-memory segment which contains the message
359	   * with munmap.
360	   */
361	  vl_msg_api_free (mp);
362	}
363    }
364  else
365    {
366      clib_warning ("unknown client ID %d", mp->index);
367    }
368}
369
370/**
371 * client answered a ping, stave off the grim reaper...
372 */
373void
374  vl_api_memclnt_keepalive_reply_t_handler
375  (vl_api_memclnt_keepalive_reply_t * mp)
376{
377  vl_api_registration_t *regp;
378  vlib_main_t *vm = vlib_get_main ();
379
380  regp = vl_api_client_index_to_registration (mp->context);
381  if (regp)
382    {
383      regp->last_heard = vlib_time_now (vm);
384      regp->unanswered_pings = 0;
385    }
386  else
387    clib_warning ("BUG: anonymous memclnt_keepalive_reply");
388}
389
390/**
391 * We can send ourselves these messages if someone uses the
392 * builtin binary api test tool...
393 */
394static void
395vl_api_memclnt_keepalive_t_handler (vl_api_memclnt_keepalive_t * mp)
396{
397  vl_api_memclnt_keepalive_reply_t *rmp;
398  api_main_t *am;
399  vl_shmem_hdr_t *shmem_hdr;
400
401  am = vlibapi_get_main ();
402  shmem_hdr = am->shmem_hdr;
403
404  rmp = vl_msg_api_alloc_as_if_client (sizeof (*rmp));
405  clib_memset (rmp, 0, sizeof (*rmp));
406  rmp->_vl_msg_id = ntohs (VL_API_MEMCLNT_KEEPALIVE_REPLY);
407  rmp->context = mp->context;
408  vl_msg_api_send_shmem (shmem_hdr->vl_input_queue, (u8 *) & rmp);
409}
410
411/*
412 * To avoid filling the API trace buffer with boring messages,
413 * don't trace memclnt_keepalive[_reply] msgs
414 */
415
416#define foreach_vlib_api_msg                            \
417_(MEMCLNT_CREATE, memclnt_create, 1)                    \
418_(MEMCLNT_DELETE, memclnt_delete, 1)                    \
419_(MEMCLNT_KEEPALIVE, memclnt_keepalive, 0)              \
420_(MEMCLNT_KEEPALIVE_REPLY, memclnt_keepalive_reply, 0)
421
422/*
423 * memory_api_init
424 */
425int
426vl_mem_api_init (const char *region_name)
427{
428  int rv;
429  api_main_t *am = vlibapi_get_main ();
430  vl_msg_api_msg_config_t cfg;
431  vl_msg_api_msg_config_t *c = &cfg;
432  vl_shmem_hdr_t *shm;
433  vlib_main_t *vm = vlib_get_main ();
434
435  clib_memset (c, 0, sizeof (*c));
436
437  if ((rv = vl_map_shmem (region_name, 1 /* is_vlib */ )) < 0)
438    return rv;
439
440#define _(N,n,t) do {                                            \
441    c->id = VL_API_##N;                                         \
442    c->name = #n;                                               \
443    c->handler = vl_api_##n##_t_handler;                        \
444    c->cleanup = vl_noop_handler;                               \
445    c->endian = vl_api_##n##_t_endian;                          \
446    c->print = vl_api_##n##_t_print;                            \
447    c->size = sizeof(vl_api_##n##_t);                           \
448    c->traced = t; /* trace, so these msgs print */             \
449    c->replay = 0; /* don't replay client create/delete msgs */ \
450    c->message_bounce = 0; /* don't bounce this message */	\
451    vl_msg_api_config(c);} while (0);
452
453  foreach_vlib_api_msg;
454#undef _
455
456  /*
457   * special-case freeing of memclnt_delete messages, so we can
458   * simply munmap pairwise / private API segments...
459   */
460  am->message_bounce[VL_API_MEMCLNT_DELETE] = 1;
461  am->is_mp_safe[VL_API_MEMCLNT_KEEPALIVE_REPLY] = 1;
462  am->is_mp_safe[VL_API_MEMCLNT_KEEPALIVE] = 1;
463
464  vlib_set_queue_signal_callback (vm, memclnt_queue_callback);
465
466  shm = am->shmem_hdr;
467  ASSERT (shm && shm->vl_input_queue);
468
469  /* Make a note so we can always find the primary region easily */
470  am->vlib_primary_rp = am->vlib_rp;
471
472  return 0;
473}
474
475clib_error_t *
476map_api_segment_init (vlib_main_t * vm)
477{
478  api_main_t *am = vlibapi_get_main ();
479  int rv;
480
481  if ((rv = vl_mem_api_init (am->region_name)) < 0)
482    {
483      return clib_error_return (0, "vl_mem_api_init (%s) failed",
484				am->region_name);
485    }
486  return 0;
487}
488
489static void
490send_memclnt_keepalive (vl_api_registration_t * regp, f64 now)
491{
492  vl_api_memclnt_keepalive_t *mp;
493  svm_queue_t *q;
494  api_main_t *am = vlibapi_get_main ();
495
496  q = regp->vl_input_queue;
497
498  /*
499   * If the queue head is moving, assume that the client is processing
500   * messages and skip the ping. This heuristic may fail if the queue
501   * is in the same position as last time, net of wrapping; in which
502   * case, the client will receive a keepalive.
503   */
504  if (regp->last_queue_head != q->head)
505    {
506      regp->last_heard = now;
507      regp->unanswered_pings = 0;
508      regp->last_queue_head = q->head;
509      return;
510    }
511
512  /*
513   * push/pop shared memory segment, so this routine
514   * will work with "normal" as well as "private segment"
515   * memory clients..
516   */
517
518  mp = vl_mem_api_alloc_as_if_client_w_reg (regp, sizeof (*mp));
519  clib_memset (mp, 0, sizeof (*mp));
520  mp->_vl_msg_id = clib_host_to_net_u16 (VL_API_MEMCLNT_KEEPALIVE);
521  mp->context = mp->client_index =
522    vl_msg_api_handle_from_index_and_epoch
523    (regp->vl_api_registration_pool_index,
524     am->shmem_hdr->application_restarts);
525
526  regp->unanswered_pings++;
527
528  /* Failure-to-send due to a stuffed queue is absolutely expected */
529  if (svm_queue_add (q, (u8 *) & mp, 1 /* nowait */ ))
530    vl_msg_api_free_w_region (regp->vlib_rp, mp);
531}
532
533static void
534vl_mem_send_client_keepalive_w_reg (api_main_t * am, f64 now,
535				    vl_api_registration_t ** regpp,
536				    u32 ** dead_indices,
537				    u32 ** confused_indices)
538{
539  vl_api_registration_t *regp = *regpp;
540  if (regp)
541    {
542      /* If we haven't heard from this client recently... */
543      if (regp->last_heard < (now - 10.0))
544	{
545	  if (regp->unanswered_pings == 2)
546	    {
547	      svm_queue_t *q;
548	      q = regp->vl_input_queue;
549	      if (kill (q->consumer_pid, 0) >= 0)
550		{
551		  clib_warning ("REAPER: lazy binary API client '%s'",
552				regp->name);
553		  regp->unanswered_pings = 0;
554		  regp->last_heard = now;
555		}
556	      else
557		{
558		  clib_warning ("REAPER: binary API client '%s' died",
559				regp->name);
560		  vec_add1 (*dead_indices, regpp - am->vl_clients);
561		}
562	    }
563	  else
564	    send_memclnt_keepalive (regp, now);
565	}
566      else
567	regp->unanswered_pings = 0;
568    }
569  else
570    {
571      clib_warning ("NULL client registration index %d",
572		    regpp - am->vl_clients);
573      vec_add1 (*confused_indices, regpp - am->vl_clients);
574    }
575}
576
577void
578vl_mem_api_dead_client_scan (api_main_t * am, vl_shmem_hdr_t * shm, f64 now)
579{
580  vl_api_registration_t **regpp;
581  static u32 *dead_indices;
582  static u32 *confused_indices;
583
584  vec_reset_length (dead_indices);
585  vec_reset_length (confused_indices);
586
587  /* *INDENT-OFF* */
588  pool_foreach (regpp, am->vl_clients, ({
589      vl_mem_send_client_keepalive_w_reg (am, now, regpp, &dead_indices,
590                                          &confused_indices);
591  }));
592  /* *INDENT-ON* */
593
594  /* This should "never happen," but if it does, fix it... */
595  if (PREDICT_FALSE (vec_len (confused_indices) > 0))
596    {
597      int i;
598      for (i = 0; i < vec_len (confused_indices); i++)
599	{
600	  pool_put_index (am->vl_clients, confused_indices[i]);
601	}
602    }
603
604  if (PREDICT_FALSE (vec_len (dead_indices) > 0))
605    {
606      int i;
607      void *oldheap;
608
609      /* Allow the application to clean up its registrations */
610      for (i = 0; i < vec_len (dead_indices); i++)
611	{
612	  regpp = pool_elt_at_index (am->vl_clients, dead_indices[i]);
613	  if (regpp)
614	    {
615	      u32 handle;
616
617	      handle = vl_msg_api_handle_from_index_and_epoch
618		(dead_indices[i], shm->application_restarts);
619	      (void) vl_api_call_reaper_functions (handle);
620	    }
621	}
622
623      oldheap = vl_msg_push_heap ();
624
625      for (i = 0; i < vec_len (dead_indices); i++)
626	{
627	  regpp = pool_elt_at_index (am->vl_clients, dead_indices[i]);
628	  if (regpp)
629	    {
630	      /* Is this a pairwise SVM segment? */
631	      if ((*regpp)->vlib_rp != am->vlib_rp)
632		{
633		  int i;
634		  svm_region_t *dead_rp = (*regpp)->vlib_rp;
635		  /* Note: account for the memfd header page */
636		  uword virtual_base = dead_rp->virtual_base - MMAP_PAGESIZE;
637		  uword virtual_size = dead_rp->virtual_size + MMAP_PAGESIZE;
638
639		  /* For horizontal scaling, add a hash table... */
640		  for (i = 0; i < vec_len (am->vlib_private_rps); i++)
641		    if (am->vlib_private_rps[i] == dead_rp)
642		      {
643			vec_delete (am->vlib_private_rps, 1, i);
644			goto found;
645		      }
646		  svm_pop_heap (oldheap);
647		  clib_warning ("private rp %llx AWOL", dead_rp);
648		  oldheap = svm_push_data_heap (am->vlib_rp);
649
650		found:
651		  /* Kill it, accounting for the memfd header page */
652		  svm_pop_heap (oldheap);
653		  if (munmap ((void *) virtual_base, virtual_size) < 0)
654		    clib_unix_warning ("munmap");
655		  /* Reset the queue-length-address cache */
656		  vec_reset_length (vl_api_queue_cursizes);
657		  oldheap = svm_push_data_heap (am->vlib_rp);
658		}
659	      else
660		{
661		  /* Poison the old registration */
662		  clib_memset (*regpp, 0xF3, sizeof (**regpp));
663		  clib_mem_free (*regpp);
664		}
665	      /* no dangling references, please */
666	      *regpp = 0;
667	    }
668	  else
669	    {
670	      svm_pop_heap (oldheap);
671	      clib_warning ("Duplicate free, client index %d",
672			    regpp - am->vl_clients);
673	      oldheap = svm_push_data_heap (am->vlib_rp);
674	    }
675	}
676
677      svm_client_scan_this_region_nolock (am->vlib_rp);
678
679      vl_msg_pop_heap (oldheap);
680      for (i = 0; i < vec_len (dead_indices); i++)
681	pool_put_index (am->vl_clients, dead_indices[i]);
682    }
683}
684
685static inline int
686void_mem_api_handle_msg_i (api_main_t * am, svm_region_t * vlib_rp,
687			   vlib_main_t * vm, vlib_node_runtime_t * node,
688			   u8 is_private)
689{
690  svm_queue_t *q;
691  uword mp;
692
693  q = ((vl_shmem_hdr_t *) (void *) vlib_rp->user_ctx)->vl_input_queue;
694
695  if (!svm_queue_sub2 (q, (u8 *) & mp))
696    {
697      VL_MSG_API_UNPOISON ((void *) mp);
698      vl_msg_api_handler_with_vm_node (am, vlib_rp, (void *) mp, vm, node,
699				       is_private);
700      return 0;
701    }
702  return -1;
703}
704
705int
706vl_mem_api_handle_msg_main (vlib_main_t * vm, vlib_node_runtime_t * node)
707{
708  api_main_t *am = vlibapi_get_main ();
709  return void_mem_api_handle_msg_i (am, am->vlib_rp, vm, node,
710				    0 /* is_private */ );
711}
712
713int
714vl_mem_api_handle_rpc (vlib_main_t * vm, vlib_node_runtime_t * node)
715{
716  api_main_t *am = vlibapi_get_main ();
717  int i;
718  uword *tmp, mp;
719
720  /*
721   * Swap pending and processing vectors, then process the RPCs
722   * Avoid deadlock conditions by construction.
723   */
724  clib_spinlock_lock_if_init (&vm->pending_rpc_lock);
725  tmp = vm->processing_rpc_requests;
726  vec_reset_length (tmp);
727  vm->processing_rpc_requests = vm->pending_rpc_requests;
728  vm->pending_rpc_requests = tmp;
729  clib_spinlock_unlock_if_init (&vm->pending_rpc_lock);
730
731  /*
732   * RPCs are used to reflect function calls to thread 0
733   * when the underlying code is not thread-safe.
734   *
735   * Grabbing the thread barrier across a set of RPCs
736   * greatly increases efficiency, and avoids
737   * running afoul of the barrier sync holddown timer.
738   * The barrier sync code supports recursive locking.
739   *
740   * We really need to rewrite RPC-based code...
741   */
742  if (PREDICT_TRUE (vec_len (vm->processing_rpc_requests)))
743    {
744      vl_msg_api_barrier_sync ();
745      for (i = 0; i < vec_len (vm->processing_rpc_requests); i++)
746	{
747	  mp = vm->processing_rpc_requests[i];
748	  vl_msg_api_handler_with_vm_node (am, am->vlib_rp, (void *) mp, vm,
749					   node, 0 /* is_private */ );
750	}
751      vl_msg_api_barrier_release ();
752    }
753
754  return 0;
755}
756
757int
758vl_mem_api_handle_msg_private (vlib_main_t * vm, vlib_node_runtime_t * node,
759			       u32 reg_index)
760{
761  api_main_t *am = vlibapi_get_main ();
762  return void_mem_api_handle_msg_i (am, am->vlib_private_rps[reg_index], vm,
763				    node, 1 /* is_private */ );
764}
765
766vl_api_registration_t *
767vl_mem_api_client_index_to_registration (u32 handle)
768{
769  vl_api_registration_t **regpp;
770  vl_api_registration_t *regp;
771  api_main_t *am = vlibapi_get_main ();
772  vl_shmem_hdr_t *shmem_hdr;
773  u32 index;
774
775  index = vl_msg_api_handle_get_index (handle);
776  regpp = am->vl_clients + index;
777
778  if (pool_is_free (am->vl_clients, regpp))
779    {
780      vl_msg_api_increment_missing_client_counter ();
781      return 0;
782    }
783  regp = *regpp;
784
785  shmem_hdr = (vl_shmem_hdr_t *) regp->shmem_hdr;
786  if (!vl_msg_api_handle_is_valid (handle, shmem_hdr->application_restarts))
787    {
788      vl_msg_api_increment_missing_client_counter ();
789      return 0;
790    }
791
792  return (regp);
793}
794
795svm_queue_t *
796vl_api_client_index_to_input_queue (u32 index)
797{
798  vl_api_registration_t *regp;
799  api_main_t *am = vlibapi_get_main ();
800
801  /* Special case: vlib trying to send itself a message */
802  if (index == (u32) ~ 0)
803    return (am->shmem_hdr->vl_input_queue);
804
805  regp = vl_mem_api_client_index_to_registration (index);
806  if (!regp)
807    return 0;
808  return (regp->vl_input_queue);
809}
810
811static clib_error_t *
812setup_memclnt_exit (vlib_main_t * vm)
813{
814  atexit (vl_unmap_shmem);
815  return 0;
816}
817
818VLIB_INIT_FUNCTION (setup_memclnt_exit);
819
820u8 *
821format_api_message_rings (u8 * s, va_list * args)
822{
823  api_main_t *am = va_arg (*args, api_main_t *);
824  vl_shmem_hdr_t *shmem_hdr = va_arg (*args, vl_shmem_hdr_t *);
825  int main_segment = va_arg (*args, int);
826  ring_alloc_t *ap;
827  int i;
828
829  if (shmem_hdr == 0)
830    return format (s, "%8s %8s %8s %8s %8s\n",
831		   "Owner", "Size", "Nitems", "Hits", "Misses");
832
833  ap = shmem_hdr->vl_rings;
834
835  for (i = 0; i < vec_len (shmem_hdr->vl_rings); i++)
836    {
837      s = format (s, "%8s %8d %8d %8d %8d\n",
838		  "vlib", ap->size, ap->nitems, ap->hits, ap->misses);
839      ap++;
840    }
841
842  ap = shmem_hdr->client_rings;
843
844  for (i = 0; i < vec_len (shmem_hdr->client_rings); i++)
845    {
846      s = format (s, "%8s %8d %8d %8d %8d\n",
847		  "clnt", ap->size, ap->nitems, ap->hits, ap->misses);
848      ap++;
849    }
850
851  if (main_segment)
852    {
853      s = format (s, "%d ring miss fallback allocations\n", am->ring_misses);
854      s = format
855	(s,
856	 "%d application restarts, %d reclaimed msgs, %d garbage collects\n",
857	 shmem_hdr->application_restarts, shmem_hdr->restart_reclaims,
858	 shmem_hdr->garbage_collects);
859    }
860  return s;
861}
862
863static clib_error_t *
864vl_api_ring_command (vlib_main_t * vm,
865		     unformat_input_t * input, vlib_cli_command_t * cli_cmd)
866{
867  int i;
868  vl_shmem_hdr_t *shmem_hdr;
869  api_main_t *am = vlibapi_get_main ();
870
871  /* First, dump the primary region rings.. */
872
873  if (am->vlib_primary_rp == 0 || am->vlib_primary_rp->user_ctx == 0)
874    {
875      vlib_cli_output (vm, "Shared memory segment not initialized...\n");
876      return 0;
877    }
878
879  shmem_hdr = (void *) am->vlib_primary_rp->user_ctx;
880
881  vlib_cli_output (vm, "Main API segment rings:");
882
883  vlib_cli_output (vm, "%U", format_api_message_rings, am,
884		   0 /* print header */ , 0 /* notused */ );
885
886  vlib_cli_output (vm, "%U", format_api_message_rings, am,
887		   shmem_hdr, 1 /* main segment */ );
888
889  for (i = 0; i < vec_len (am->vlib_private_rps); i++)
890    {
891      svm_region_t *vlib_rp = am->vlib_private_rps[i];
892      shmem_hdr = (void *) vlib_rp->user_ctx;
893      vl_api_registration_t **regpp;
894      vl_api_registration_t *regp = 0;
895
896      /* For horizontal scaling, add a hash table... */
897      /* *INDENT-OFF* */
898      pool_foreach (regpp, am->vl_clients,
899      ({
900        regp = *regpp;
901        if (regp && regp->vlib_rp == vlib_rp)
902          {
903            vlib_cli_output (vm, "%s segment rings:", regp->name);
904            goto found;
905          }
906      }));
907      vlib_cli_output (vm, "regp %llx not found?", regp);
908      continue;
909      /* *INDENT-ON* */
910    found:
911      vlib_cli_output (vm, "%U", format_api_message_rings, am,
912		       0 /* print header */ , 0 /* notused */ );
913      vlib_cli_output (vm, "%U", format_api_message_rings, am,
914		       shmem_hdr, 0 /* main segment */ );
915    }
916
917  return 0;
918}
919
920/*?
921 * Display binary api message allocation ring statistics
922?*/
923/* *INDENT-OFF* */
924VLIB_CLI_COMMAND (cli_show_api_ring_command, static) =
925{
926  .path = "show api ring-stats",
927  .short_help = "Message ring statistics",
928  .function = vl_api_ring_command,
929};
930/* *INDENT-ON* */
931
932clib_error_t *
933vlibmemory_init (vlib_main_t * vm)
934{
935  api_main_t *am = vlibapi_get_main ();
936  svm_map_region_args_t _a, *a = &_a;
937  u8 *remove_path1, *remove_path2;
938  void vlibsocket_reference (void);
939
940  vlibsocket_reference ();
941
942  /*
943   * By popular request / to avoid support fires, remove any old api segment
944   * files Right Here.
945   */
946  if (am->root_path == 0)
947    {
948      remove_path1 = format (0, "/dev/shm/global_vm%c", 0);
949      remove_path2 = format (0, "/dev/shm/vpe-api%c", 0);
950    }
951  else
952    {
953      remove_path1 = format (0, "/dev/shm/%s-global_vm%c", am->root_path, 0);
954      remove_path2 = format (0, "/dev/shm/%s-vpe-api%c", am->root_path, 0);
955    }
956
957  (void) unlink ((char *) remove_path1);
958  (void) unlink ((char *) remove_path2);
959
960  vec_free (remove_path1);
961  vec_free (remove_path2);
962
963  clib_memset (a, 0, sizeof (*a));
964  a->root_path = am->root_path;
965  a->name = SVM_GLOBAL_REGION_NAME;
966  a->baseva = (am->global_baseva != 0) ?
967    am->global_baseva : +svm_get_global_region_base_va ();
968  a->size = (am->global_size != 0) ? am->global_size : SVM_GLOBAL_REGION_SIZE;
969  a->flags = SVM_FLAGS_NODATA;
970  a->uid = am->api_uid;
971  a->gid = am->api_gid;
972  a->pvt_heap_size =
973    (am->global_pvt_heap_size !=
974     0) ? am->global_pvt_heap_size : SVM_PVT_MHEAP_SIZE;
975
976  svm_region_init_args (a);
977
978  return 0;
979}
980
981void
982vl_set_memory_region_name (const char *name)
983{
984  api_main_t *am = vlibapi_get_main ();
985  am->region_name = name;
986}
987
988/*
989 * fd.io coding-style-patch-verification: ON
990 *
991 * Local Variables:
992 * eval: (c-set-style "gnu")
993 * End:
994 */
995