quic.c revision d1b9e706
1/*
2 * Copyright (c) 2019 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include <sys/socket.h>
17
18#include <vnet/session/application.h>
19#include <vnet/session/transport.h>
20#include <vnet/session/session.h>
21#include <vlib/unix/plugin.h>
22#include <vpp/app/version.h>
23
24#include <vppinfra/lock.h>
25
26#include <quic/quic.h>
27#include <quic/certs.h>
28#include <quic/error.h>
29#include <quic/quic_crypto.h>
30
31#include <quicly/constants.h>
32#include <quicly/defaults.h>
33
34static char *quic_error_strings[] = {
35#define quic_error(n,s) s,
36#include <quic/quic_error.def>
37#undef quic_error
38};
39
40static quic_main_t quic_main;
41static void quic_update_timer (quic_ctx_t * ctx);
42static void quic_check_quic_session_connected (quic_ctx_t * ctx);
43static int quic_reset_connection (u64 udp_session_handle,
44				  quic_rx_packet_ctx_t * pctx);
45static void quic_proto_on_close (u32 ctx_index, u32 thread_index);
46
47static quicly_stream_open_t on_stream_open;
48static quicly_closed_by_peer_t on_closed_by_peer;
49static quicly_now_t quicly_vpp_now_cb;
50
51/* Crypto contexts */
52
53static inline void
54quic_crypto_context_make_key_from_ctx (clib_bihash_kv_24_8_t * kv,
55				       quic_ctx_t * ctx)
56{
57  application_t *app = application_get (ctx->parent_app_id);
58  kv->key[0] = ((u64) ctx->ckpair_index) << 32 | (u64) ctx->crypto_engine;
59  kv->key[1] = app->sm_properties.rx_fifo_size - 1;
60  kv->key[2] = app->sm_properties.tx_fifo_size - 1;
61}
62
63static inline void
64quic_crypto_context_make_key_from_crctx (clib_bihash_kv_24_8_t * kv,
65					 crypto_context_t * crctx)
66{
67  quic_crypto_context_data_t *data =
68    (quic_crypto_context_data_t *) crctx->data;
69  kv->key[0] = ((u64) crctx->ckpair_index) << 32 | (u64) crctx->crypto_engine;
70  kv->key[1] = data->quicly_ctx.transport_params.max_stream_data.bidi_local;
71  kv->key[2] = data->quicly_ctx.transport_params.max_stream_data.bidi_remote;
72}
73
74static void
75quic_crypto_context_free_if_needed (crypto_context_t * crctx, u8 thread_index)
76{
77  quic_main_t *qm = &quic_main;
78  clib_bihash_kv_24_8_t kv;
79  if (crctx->n_subscribers)
80    return;
81  quic_crypto_context_make_key_from_crctx (&kv, crctx);
82  clib_bihash_add_del_24_8 (&qm->wrk_ctx[thread_index].crypto_context_hash,
83			    &kv, 0 /* is_add */ );
84  clib_mem_free (crctx->data);
85  pool_put (qm->wrk_ctx[thread_index].crypto_ctx_pool, crctx);
86}
87
88static int
89quic_app_cert_key_pair_delete_callback (app_cert_key_pair_t * ckpair)
90{
91  quic_main_t *qm = &quic_main;
92  crypto_context_t *crctx;
93  clib_bihash_kv_24_8_t kv;
94  vlib_thread_main_t *vtm = vlib_get_thread_main ();
95  int num_threads = 1 /* main thread */  + vtm->n_threads;
96  int i;
97
98  for (i = 0; i < num_threads; i++)
99    {
100      /* *INDENT-OFF* */
101      pool_foreach (crctx, qm->wrk_ctx[i].crypto_ctx_pool, ({
102	if (crctx->ckpair_index == ckpair->cert_key_index)
103	  {
104	    quic_crypto_context_make_key_from_crctx (&kv, crctx);
105	    clib_bihash_add_del_24_8 (&qm->wrk_ctx[i].crypto_context_hash, &kv, 0 /* is_add */ );
106	  }
107      }));
108      /* *INDENT-ON* */
109    }
110  return 0;
111}
112
113static crypto_context_t *
114quic_crypto_context_alloc (u8 thread_index)
115{
116  quic_main_t *qm = &quic_main;
117  crypto_context_t *crctx;
118  u32 idx;
119
120  pool_get (qm->wrk_ctx[thread_index].crypto_ctx_pool, crctx);
121  clib_memset (crctx, 0, sizeof (*crctx));
122  idx = (crctx - qm->wrk_ctx[thread_index].crypto_ctx_pool);
123  crctx->ctx_index = ((u32) thread_index) << 24 | idx;
124
125  return crctx;
126}
127
128static crypto_context_t *
129quic_crypto_context_get (u32 cr_index, u32 thread_index)
130{
131  quic_main_t *qm = &quic_main;
132  ASSERT (cr_index >> 24 == thread_index);
133  return pool_elt_at_index (qm->wrk_ctx[thread_index].crypto_ctx_pool,
134			    cr_index & 0x00ffffff);
135}
136
137static clib_error_t *
138quic_list_crypto_context_command_fn (vlib_main_t * vm,
139				     unformat_input_t * input,
140				     vlib_cli_command_t * cmd)
141{
142  quic_main_t *qm = &quic_main;
143  crypto_context_t *crctx;
144  vlib_thread_main_t *vtm = vlib_get_thread_main ();
145  int i, num_threads = 1 /* main thread */  + vtm->n_threads;
146  for (i = 0; i < num_threads; i++)
147    {
148      /* *INDENT-OFF* */
149      pool_foreach (crctx, qm->wrk_ctx[i].crypto_ctx_pool, ({
150	vlib_cli_output (vm, "[%d][Q]%U", i, format_crypto_context, crctx);
151      }));
152      /* *INDENT-ON* */
153    }
154  return 0;
155}
156
157static void
158quic_release_crypto_context (u32 crypto_context_index, u8 thread_index)
159{
160  crypto_context_t *crctx;
161  crctx = quic_crypto_context_get (crypto_context_index, thread_index);
162  crctx->n_subscribers--;
163  quic_crypto_context_free_if_needed (crctx, thread_index);
164}
165
166static int
167quic_init_crypto_context (crypto_context_t * crctx, quic_ctx_t * ctx)
168{
169  quic_main_t *qm = &quic_main;
170  quicly_context_t *quicly_ctx;
171  ptls_iovec_t key_vec;
172  app_cert_key_pair_t *ckpair;
173  application_t *app;
174  quic_crypto_context_data_t *data;
175  ptls_context_t *ptls_ctx;
176
177  QUIC_DBG (2, "Init quic crctx %d thread %d", crctx->ctx_index,
178	    ctx->c_thread_index);
179
180  data = clib_mem_alloc (sizeof (*data));
181  /* picotls depends on data being zeroed */
182  clib_memset (data, 0, sizeof (*data));
183  crctx->data = (void *) data;
184  quicly_ctx = &data->quicly_ctx;
185  ptls_ctx = &data->ptls_ctx;
186
187  ptls_ctx->random_bytes = ptls_openssl_random_bytes;
188  ptls_ctx->get_time = &ptls_get_time;
189  ptls_ctx->key_exchanges = ptls_openssl_key_exchanges;
190  ptls_ctx->cipher_suites = qm->quic_ciphers[ctx->crypto_engine];
191  ptls_ctx->certificates.list = NULL;
192  ptls_ctx->certificates.count = 0;
193  ptls_ctx->esni = NULL;
194  ptls_ctx->on_client_hello = NULL;
195  ptls_ctx->emit_certificate = NULL;
196  ptls_ctx->sign_certificate = NULL;
197  ptls_ctx->verify_certificate = NULL;
198  ptls_ctx->ticket_lifetime = 86400;
199  ptls_ctx->max_early_data_size = 8192;
200  ptls_ctx->hkdf_label_prefix__obsolete = NULL;
201  ptls_ctx->require_dhe_on_psk = 1;
202  ptls_ctx->encrypt_ticket = &qm->session_cache.super;
203  clib_memcpy (quicly_ctx, &quicly_spec_context, sizeof (quicly_context_t));
204
205  quicly_ctx->max_packet_size = QUIC_MAX_PACKET_SIZE;
206  quicly_ctx->tls = ptls_ctx;
207  quicly_ctx->stream_open = &on_stream_open;
208  quicly_ctx->closed_by_peer = &on_closed_by_peer;
209  quicly_ctx->now = &quicly_vpp_now_cb;
210  quicly_amend_ptls_context (quicly_ctx->tls);
211
212  quicly_ctx->transport_params.max_data = QUIC_INT_MAX;
213  quicly_ctx->transport_params.max_streams_uni = (uint64_t) 1 << 60;
214  quicly_ctx->transport_params.max_streams_bidi = (uint64_t) 1 << 60;
215  quicly_ctx->transport_params.idle_timeout = qm->connection_timeout;
216
217  app = application_get (ctx->parent_app_id);
218  quicly_ctx->transport_params.max_stream_data.bidi_local =
219    app->sm_properties.rx_fifo_size - 1;
220  quicly_ctx->transport_params.max_stream_data.bidi_remote =
221    app->sm_properties.tx_fifo_size - 1;
222  quicly_ctx->transport_params.max_stream_data.uni = QUIC_INT_MAX;
223
224  if (!app->quic_iv_set)
225    {
226      ptls_openssl_random_bytes (app->quic_iv, QUIC_IV_LEN - 1);
227      app->quic_iv[QUIC_IV_LEN - 1] = 0;
228      app->quic_iv_set = 1;
229    }
230
231  clib_memcpy (data->cid_key, app->quic_iv, QUIC_IV_LEN);
232  key_vec = ptls_iovec_init (data->cid_key, QUIC_IV_LEN);
233  quicly_ctx->cid_encryptor =
234    quicly_new_default_cid_encryptor (&ptls_openssl_bfecb,
235				      &ptls_openssl_aes128ecb,
236				      &ptls_openssl_sha256, key_vec);
237
238  ckpair = app_cert_key_pair_get_if_valid (crctx->ckpair_index);
239  if (!ckpair || !ckpair->key || !ckpair->cert)
240    {
241      QUIC_DBG (1, "Wrong ckpair id %d\n", crctx->ckpair_index);
242      return -1;
243    }
244  if (load_bio_private_key (quicly_ctx->tls, (char *) ckpair->key))
245    {
246      QUIC_DBG (1, "failed to read private key from app configuration\n");
247      return -1;
248    }
249  if (load_bio_certificate_chain (quicly_ctx->tls, (char *) ckpair->cert))
250    {
251      QUIC_DBG (1, "failed to load certificate\n");
252      return -1;
253    }
254  return 0;
255
256}
257
258static int
259quic_acquire_crypto_context (quic_ctx_t * ctx)
260{
261  quic_main_t *qm = &quic_main;
262  crypto_context_t *crctx;
263  clib_bihash_kv_24_8_t kv;
264
265  if (ctx->crypto_engine == CRYPTO_ENGINE_NONE)
266    {
267      QUIC_DBG (2, "No crypto engine specified, using %d",
268		qm->default_crypto_engine);
269      ctx->crypto_engine = qm->default_crypto_engine;
270    }
271  if (!clib_bitmap_get (qm->available_crypto_engines, ctx->crypto_engine))
272    {
273      QUIC_DBG (1, "Quic does not support crypto engine %d",
274		ctx->crypto_engine);
275      return VNET_API_ERROR_MISSING_CERT_KEY;
276    }
277
278  /* Check for exisiting crypto ctx */
279  quic_crypto_context_make_key_from_ctx (&kv, ctx);
280  if (clib_bihash_search_24_8
281      (&qm->wrk_ctx[ctx->c_thread_index].crypto_context_hash, &kv, &kv) == 0)
282    {
283      crctx = quic_crypto_context_get (kv.value, ctx->c_thread_index);
284      QUIC_DBG (2, "Found exisiting crypto context %d", kv.value);
285      ctx->crypto_context_index = kv.value;
286      crctx->n_subscribers++;
287      return 0;
288    }
289
290  crctx = quic_crypto_context_alloc (ctx->c_thread_index);
291  ctx->crypto_context_index = crctx->ctx_index;
292  kv.value = crctx->ctx_index;
293  crctx->crypto_engine = ctx->crypto_engine;
294  crctx->ckpair_index = ctx->ckpair_index;
295  if (quic_init_crypto_context (crctx, ctx))
296    goto error;
297  if (vnet_app_add_cert_key_interest (ctx->ckpair_index, qm->app_index))
298    goto error;
299  crctx->n_subscribers++;
300  clib_bihash_add_del_24_8 (&qm->
301			    wrk_ctx[ctx->c_thread_index].crypto_context_hash,
302			    &kv, 1 /* is_add */ );
303  return 0;
304
305error:
306  quic_crypto_context_free_if_needed (crctx, ctx->c_thread_index);
307  return VNET_API_ERROR_MISSING_CERT_KEY;
308}
309
310/*  Helper functions */
311
312static u32
313quic_ctx_alloc (u32 thread_index)
314{
315  quic_main_t *qm = &quic_main;
316  quic_ctx_t *ctx;
317
318  pool_get (qm->ctx_pool[thread_index], ctx);
319
320  clib_memset (ctx, 0, sizeof (quic_ctx_t));
321  ctx->c_thread_index = thread_index;
322  ctx->timer_handle = QUIC_TIMER_HANDLE_INVALID;
323  QUIC_DBG (3, "Allocated quic_ctx %u on thread %u",
324	    ctx - qm->ctx_pool[thread_index], thread_index);
325  return ctx - qm->ctx_pool[thread_index];
326}
327
328static void
329quic_ctx_free (quic_ctx_t * ctx)
330{
331  QUIC_DBG (2, "Free ctx %u %x", ctx->c_thread_index, ctx->c_c_index);
332  u32 thread_index = ctx->c_thread_index;
333  QUIC_ASSERT (ctx->timer_handle == QUIC_TIMER_HANDLE_INVALID);
334  if (CLIB_DEBUG)
335    clib_memset (ctx, 0xfb, sizeof (*ctx));
336  pool_put (quic_main.ctx_pool[thread_index], ctx);
337}
338
339static quic_ctx_t *
340quic_ctx_get (u32 ctx_index, u32 thread_index)
341{
342  return pool_elt_at_index (quic_main.ctx_pool[thread_index], ctx_index);
343}
344
345static quic_ctx_t *
346quic_ctx_get_if_valid (u32 ctx_index, u32 thread_index)
347{
348  if (pool_is_free_index (quic_main.ctx_pool[thread_index], ctx_index))
349    return 0;
350  return pool_elt_at_index (quic_main.ctx_pool[thread_index], ctx_index);
351}
352
353static quic_ctx_t *
354quic_get_conn_ctx (quicly_conn_t * conn)
355{
356  u64 conn_data;
357  conn_data = (u64) * quicly_get_data (conn);
358  return quic_ctx_get (conn_data & UINT32_MAX, conn_data >> 32);
359}
360
361static void
362quic_store_conn_ctx (quicly_conn_t * conn, quic_ctx_t * ctx)
363{
364  *quicly_get_data (conn) =
365    (void *) (((u64) ctx->c_thread_index) << 32 | (u64) ctx->c_c_index);
366}
367
368static inline int
369quic_ctx_is_stream (quic_ctx_t * ctx)
370{
371  return (ctx->flags & QUIC_F_IS_STREAM);
372}
373
374static inline int
375quic_ctx_is_listener (quic_ctx_t * ctx)
376{
377  return (ctx->flags & QUIC_F_IS_LISTENER);
378}
379
380static inline int
381quic_ctx_is_conn (quic_ctx_t * ctx)
382{
383  return !(quic_ctx_is_listener (ctx) || quic_ctx_is_stream (ctx));
384}
385
386static session_t *
387get_stream_session_from_stream (quicly_stream_t * stream)
388{
389  quic_ctx_t *ctx;
390  quic_stream_data_t *stream_data;
391
392  stream_data = (quic_stream_data_t *) stream->data;
393  ctx = quic_ctx_get (stream_data->ctx_id, stream_data->thread_index);
394  return session_get (ctx->c_s_index, stream_data->thread_index);
395}
396
397static inline void
398quic_make_connection_key (clib_bihash_kv_16_8_t * kv,
399			  const quicly_cid_plaintext_t * id)
400{
401  kv->key[0] = ((u64) id->master_id) << 32 | (u64) id->thread_id;
402  kv->key[1] = id->node_id;
403}
404
405static int
406quic_sendable_packet_count (session_t * udp_session)
407{
408  u32 max_enqueue;
409  u32 packet_size = QUIC_MAX_PACKET_SIZE + SESSION_CONN_HDR_LEN;
410  max_enqueue = svm_fifo_max_enqueue (udp_session->tx_fifo);
411  return clib_min (max_enqueue / packet_size, QUIC_SEND_PACKET_VEC_SIZE);
412}
413
414static quicly_context_t *
415quic_get_quicly_ctx_from_ctx (quic_ctx_t * ctx)
416{
417  crypto_context_t *crctx =
418    quic_crypto_context_get (ctx->crypto_context_index, ctx->c_thread_index);
419  quic_crypto_context_data_t *data =
420    (quic_crypto_context_data_t *) crctx->data;
421  return &data->quicly_ctx;
422}
423
424static quicly_context_t *
425quic_get_quicly_ctx_from_udp (u64 udp_session_handle)
426{
427  session_t *udp_session = session_get_from_handle (udp_session_handle);
428  quic_ctx_t *ctx =
429    quic_ctx_get (udp_session->opaque, udp_session->thread_index);
430  return quic_get_quicly_ctx_from_ctx (ctx);
431}
432
433static inline void
434quic_set_udp_tx_evt (session_t * udp_session)
435{
436  int rv = 0;
437  if (svm_fifo_set_event (udp_session->tx_fifo))
438    rv = session_send_io_evt_to_thread (udp_session->tx_fifo,
439					SESSION_IO_EVT_TX);
440  if (PREDICT_FALSE (rv))
441    clib_warning ("Event enqueue errored %d", rv);
442}
443
444static inline void
445quic_stop_ctx_timer (quic_ctx_t * ctx)
446{
447  tw_timer_wheel_1t_3w_1024sl_ov_t *tw;
448  if (ctx->timer_handle == QUIC_TIMER_HANDLE_INVALID)
449    return;
450  tw = &quic_main.wrk_ctx[ctx->c_thread_index].timer_wheel;
451  tw_timer_stop_1t_3w_1024sl_ov (tw, ctx->timer_handle);
452  ctx->timer_handle = QUIC_TIMER_HANDLE_INVALID;
453  QUIC_DBG (4, "Stopping timer for ctx %u", ctx->c_c_index);
454}
455
456/* QUIC protocol actions */
457
458static void
459quic_ack_rx_data (session_t * stream_session)
460{
461  u32 max_deq;
462  quic_ctx_t *sctx;
463  svm_fifo_t *f;
464  quicly_stream_t *stream;
465  quic_stream_data_t *stream_data;
466
467  sctx = quic_ctx_get (stream_session->connection_index,
468		       stream_session->thread_index);
469  QUIC_ASSERT (quic_ctx_is_stream (sctx));
470  stream = sctx->stream;
471  stream_data = (quic_stream_data_t *) stream->data;
472
473  f = stream_session->rx_fifo;
474  max_deq = svm_fifo_max_dequeue (f);
475
476  QUIC_ASSERT (stream_data->app_rx_data_len >= max_deq);
477  quicly_stream_sync_recvbuf (stream, stream_data->app_rx_data_len - max_deq);
478  QUIC_DBG (3, "Acking %u bytes", stream_data->app_rx_data_len - max_deq);
479  stream_data->app_rx_data_len = max_deq;
480}
481
482static void
483quic_disconnect_transport (quic_ctx_t * ctx)
484{
485  QUIC_DBG (2, "Disconnecting transport 0x%lx", ctx->udp_session_handle);
486  vnet_disconnect_args_t a = {
487    .handle = ctx->udp_session_handle,
488    .app_index = quic_main.app_index,
489  };
490
491  if (vnet_disconnect_session (&a))
492    clib_warning ("UDP session 0x%lx disconnect errored",
493		  ctx->udp_session_handle);
494}
495
496static void
497quic_connection_delete (quic_ctx_t * ctx)
498{
499  clib_bihash_kv_16_8_t kv;
500  quicly_conn_t *conn;
501
502  QUIC_DBG (2, "Deleting connection %u", ctx->c_c_index);
503
504  QUIC_ASSERT (!quic_ctx_is_stream (ctx));
505  quic_stop_ctx_timer (ctx);
506
507  /*  Delete the connection from the connection map */
508  conn = ctx->conn;
509  ctx->conn = NULL;
510  quic_make_connection_key (&kv, quicly_get_master_id (conn));
511  QUIC_DBG (2, "Deleting conn with id %lu %lu from map", kv.key[0],
512	    kv.key[1]);
513  clib_bihash_add_del_16_8 (&quic_main.connection_hash, &kv, 0 /* is_add */ );
514
515  quic_disconnect_transport (ctx);
516
517  if (ctx->conn)
518    quicly_free (ctx->conn);
519  session_transport_delete_notify (&ctx->connection);
520}
521
522void
523quic_increment_counter (u8 evt, u8 val)
524{
525  vlib_main_t *vm = vlib_get_main ();
526  vlib_node_increment_counter (vm, quic_input_node.index, evt, val);
527}
528
529/**
530 * Called when quicly return an error
531 * This function interacts tightly with quic_proto_on_close
532 */
533static void
534quic_connection_closed (quic_ctx_t * ctx)
535{
536  QUIC_DBG (2, "QUIC connection %u/%u closed", ctx->c_thread_index,
537	    ctx->c_c_index);
538
539  /* TODO if connection is not established, just delete the session? */
540  /* Actually should send connect or accept error */
541
542  switch (ctx->conn_state)
543    {
544    case QUIC_CONN_STATE_READY:
545      /* Error on an opened connection (timeout...)
546         This puts the session in closing state, we should receive a notification
547         when the app has closed its session */
548      session_transport_reset_notify (&ctx->connection);
549      /* This ensures we delete the connection when the app confirms the close */
550      ctx->conn_state = QUIC_CONN_STATE_PASSIVE_CLOSING_QUIC_CLOSED;
551      break;
552    case QUIC_CONN_STATE_PASSIVE_CLOSING:
553      ctx->conn_state = QUIC_CONN_STATE_PASSIVE_CLOSING_QUIC_CLOSED;
554      /* quic_proto_on_close will eventually be called when the app confirms the close
555         , we delete the connection at that point */
556      break;
557    case QUIC_CONN_STATE_PASSIVE_CLOSING_APP_CLOSED:
558      /* App already confirmed close, we can delete the connection */
559      quic_connection_delete (ctx);
560      break;
561    case QUIC_CONN_STATE_OPENED:
562    case QUIC_CONN_STATE_HANDSHAKE:
563    case QUIC_CONN_STATE_ACTIVE_CLOSING:
564      quic_connection_delete (ctx);
565      break;
566    default:
567      QUIC_DBG (0, "BUG %d", ctx->conn_state);
568      break;
569    }
570}
571
572static int
573quic_send_datagram (session_t * udp_session, quicly_datagram_t * packet)
574{
575  u32 max_enqueue;
576  session_dgram_hdr_t hdr;
577  u32 len, ret;
578  svm_fifo_t *f;
579  transport_connection_t *tc;
580
581  len = packet->data.len;
582  f = udp_session->tx_fifo;
583  tc = session_get_transport (udp_session);
584  max_enqueue = svm_fifo_max_enqueue (f);
585  if (max_enqueue < SESSION_CONN_HDR_LEN + len)
586    {
587      QUIC_ERR ("Too much data to send, max_enqueue %u, len %u",
588		max_enqueue, len + SESSION_CONN_HDR_LEN);
589      return QUIC_ERROR_FULL_FIFO;
590    }
591
592  /*  Build packet header for fifo */
593  hdr.data_length = len;
594  hdr.data_offset = 0;
595  hdr.is_ip4 = tc->is_ip4;
596  clib_memcpy (&hdr.lcl_ip, &tc->lcl_ip, sizeof (ip46_address_t));
597  hdr.lcl_port = tc->lcl_port;
598
599  /*  Read dest address from quicly-provided sockaddr */
600  if (hdr.is_ip4)
601    {
602      QUIC_ASSERT (packet->dest.sa.sa_family == AF_INET);
603      struct sockaddr_in *sa4 = (struct sockaddr_in *) &packet->dest.sa;
604      hdr.rmt_port = sa4->sin_port;
605      hdr.rmt_ip.ip4.as_u32 = sa4->sin_addr.s_addr;
606    }
607  else
608    {
609      QUIC_ASSERT (packet->dest.sa.sa_family == AF_INET6);
610      struct sockaddr_in6 *sa6 = (struct sockaddr_in6 *) &packet->dest.sa;
611      hdr.rmt_port = sa6->sin6_port;
612      clib_memcpy (&hdr.rmt_ip.ip6, &sa6->sin6_addr, 16);
613    }
614
615  ret = svm_fifo_enqueue (f, sizeof (hdr), (u8 *) & hdr);
616  if (ret != sizeof (hdr))
617    {
618      QUIC_ERR ("Not enough space to enqueue header");
619      return QUIC_ERROR_FULL_FIFO;
620    }
621  ret = svm_fifo_enqueue (f, len, packet->data.base);
622  if (ret != len)
623    {
624      QUIC_ERR ("Not enough space to enqueue payload");
625      return QUIC_ERROR_FULL_FIFO;
626    }
627
628  quic_increment_counter (QUIC_ERROR_TX_PACKETS, 1);
629
630  return 0;
631}
632
633static int
634quic_send_packets (quic_ctx_t * ctx)
635{
636  quicly_datagram_t *packets[QUIC_SEND_PACKET_VEC_SIZE];
637  session_t *udp_session;
638  quicly_conn_t *conn;
639  size_t num_packets, i, max_packets;
640  quicly_packet_allocator_t *pa;
641  int err = 0;
642
643  /* We have sctx, get qctx */
644  if (quic_ctx_is_stream (ctx))
645    ctx = quic_ctx_get (ctx->quic_connection_ctx_id, ctx->c_thread_index);
646
647  QUIC_ASSERT (!quic_ctx_is_stream (ctx));
648
649  udp_session = session_get_from_handle_if_valid (ctx->udp_session_handle);
650  if (!udp_session)
651    goto quicly_error;
652
653  conn = ctx->conn;
654
655  if (!conn)
656    return 0;
657
658  /* TODO : quicly can assert it can send min_packets up to 2 */
659  if (quic_sendable_packet_count (udp_session) < 2)
660    goto stop_sending;
661
662  pa = quic_get_quicly_ctx_from_ctx (ctx)->packet_allocator;
663  do
664    {
665      max_packets = quic_sendable_packet_count (udp_session);
666      if (max_packets < 2)
667	break;
668      num_packets = max_packets;
669      if ((err = quicly_send (conn, packets, &num_packets)))
670	goto quicly_error;
671
672      for (i = 0; i != num_packets; ++i)
673	{
674	  if ((err = quic_send_datagram (udp_session, packets[i])))
675	    goto quicly_error;
676
677	  pa->free_packet (pa, packets[i]);
678	}
679    }
680  while (num_packets > 0 && num_packets == max_packets);
681
682stop_sending:
683  quic_set_udp_tx_evt (udp_session);
684
685  QUIC_DBG (3, "%u[TX] %u[RX]", svm_fifo_max_dequeue (udp_session->tx_fifo),
686	    svm_fifo_max_dequeue (udp_session->rx_fifo));
687  quic_update_timer (ctx);
688  return 0;
689
690quicly_error:
691  if (err && err != QUICLY_ERROR_PACKET_IGNORED
692      && err != QUICLY_ERROR_FREE_CONNECTION)
693    clib_warning ("Quic error '%U'.", quic_format_err, err);
694  quic_connection_closed (ctx);
695  return 1;
696}
697
698/* Quicly callbacks */
699
700static void
701quic_on_stream_destroy (quicly_stream_t * stream, int err)
702{
703  quic_stream_data_t *stream_data = (quic_stream_data_t *) stream->data;
704  quic_ctx_t *sctx = quic_ctx_get (stream_data->ctx_id,
705				   stream_data->thread_index);
706  session_t *stream_session = session_get (sctx->c_s_index,
707					   sctx->c_thread_index);
708  QUIC_DBG (2, "DESTROYED_STREAM: session 0x%lx (%U)",
709	    session_handle (stream_session), quic_format_err, err);
710
711  stream_session->session_state = SESSION_STATE_CLOSED;
712  session_transport_delete_notify (&sctx->connection);
713
714  quic_increment_counter (QUIC_ERROR_CLOSED_STREAM, 1);
715  quic_ctx_free (sctx);
716  clib_mem_free (stream->data);
717}
718
719static int
720quic_on_stop_sending (quicly_stream_t * stream, int err)
721{
722#if QUIC_DEBUG >= 2
723  quic_stream_data_t *stream_data = (quic_stream_data_t *) stream->data;
724  quic_ctx_t *sctx = quic_ctx_get (stream_data->ctx_id,
725				   stream_data->thread_index);
726  session_t *stream_session = session_get (sctx->c_s_index,
727					   sctx->c_thread_index);
728  clib_warning ("(NOT IMPLEMENTD) STOP_SENDING: session 0x%lx (%U)",
729		session_handle (stream_session), quic_format_err, err);
730#endif
731  /* TODO : handle STOP_SENDING */
732  return 0;
733}
734
735static int
736quic_on_receive_reset (quicly_stream_t * stream, int err)
737{
738  quic_stream_data_t *stream_data = (quic_stream_data_t *) stream->data;
739  quic_ctx_t *sctx = quic_ctx_get (stream_data->ctx_id,
740				   stream_data->thread_index);
741#if QUIC_DEBUG >= 2
742  session_t *stream_session = session_get (sctx->c_s_index,
743					   sctx->c_thread_index);
744  clib_warning ("RESET_STREAM: session 0x%lx (%U)",
745		session_handle (stream_session), quic_format_err, err);
746#endif
747  session_transport_closing_notify (&sctx->connection);
748  return 0;
749}
750
751static int
752quic_on_receive (quicly_stream_t * stream, size_t off, const void *src,
753		 size_t len)
754{
755  QUIC_DBG (3, "received data: %lu bytes, offset %lu", len, off);
756  u32 max_enq, rlen, rv;
757  quic_ctx_t *sctx;
758  session_t *stream_session;
759  app_worker_t *app_wrk;
760  svm_fifo_t *f;
761  quic_stream_data_t *stream_data;
762
763  stream_data = (quic_stream_data_t *) stream->data;
764  sctx = quic_ctx_get (stream_data->ctx_id, stream_data->thread_index);
765  stream_session = session_get (sctx->c_s_index, stream_data->thread_index);
766  f = stream_session->rx_fifo;
767
768  max_enq = svm_fifo_max_enqueue_prod (f);
769  QUIC_DBG (3, "Enqueuing %u at off %u in %u space", len, off, max_enq);
770  /* Handle duplicate packet/chunk from quicly */
771  if (off < stream_data->app_rx_data_len)
772    {
773      QUIC_DBG (3, "Session [idx %u, app_wrk %u, thread %u, rx-fifo 0x%llx]: "
774		"DUPLICATE PACKET (max_enq %u, len %u, "
775		"app_rx_data_len %u, off %u, ToBeNQ %u)",
776		stream_session->session_index,
777		stream_session->app_wrk_index,
778		stream_session->thread_index, f,
779		max_enq, len, stream_data->app_rx_data_len, off,
780		off - stream_data->app_rx_data_len + len);
781      return 0;
782    }
783  if (PREDICT_FALSE ((off - stream_data->app_rx_data_len + len) > max_enq))
784    {
785      QUIC_ERR ("Session [idx %u, app_wrk %u, thread %u, rx-fifo 0x%llx]: "
786		"RX FIFO IS FULL (max_enq %u, len %u, "
787		"app_rx_data_len %u, off %u, ToBeNQ %u)",
788		stream_session->session_index,
789		stream_session->app_wrk_index,
790		stream_session->thread_index, f,
791		max_enq, len, stream_data->app_rx_data_len, off,
792		off - stream_data->app_rx_data_len + len);
793      return 1;
794    }
795  if (off == stream_data->app_rx_data_len)
796    {
797      /* Streams live on the same thread so (f, stream_data) should stay consistent */
798      rlen = svm_fifo_enqueue (f, len, (u8 *) src);
799      QUIC_DBG (3, "Session [idx %u, app_wrk %u, ti %u, rx-fifo 0x%llx]: "
800		"Enqueuing %u (rlen %u) at off %u in %u space, ",
801		stream_session->session_index,
802		stream_session->app_wrk_index,
803		stream_session->thread_index, f, len, rlen, off, max_enq);
804      stream_data->app_rx_data_len += rlen;
805      QUIC_ASSERT (rlen >= len);
806      app_wrk = app_worker_get_if_valid (stream_session->app_wrk_index);
807      if (PREDICT_TRUE (app_wrk != 0))
808	{
809	  rv = app_worker_lock_and_send_event (app_wrk, stream_session,
810					       SESSION_IO_EVT_RX);
811	  if (rv)
812	    QUIC_ERR ("Failed to ping app for RX");
813	}
814      quic_ack_rx_data (stream_session);
815    }
816  else
817    {
818      rlen = svm_fifo_enqueue_with_offset (f,
819					   off - stream_data->app_rx_data_len,
820					   len, (u8 *) src);
821      QUIC_ASSERT (rlen == 0);
822    }
823  return 0;
824}
825
826void
827quic_fifo_egress_shift (quicly_stream_t * stream, size_t delta)
828{
829  quic_stream_data_t *stream_data;
830  session_t *stream_session;
831  svm_fifo_t *f;
832  u32 rv;
833
834  stream_data = (quic_stream_data_t *) stream->data;
835  stream_session = get_stream_session_from_stream (stream);
836  f = stream_session->tx_fifo;
837
838  QUIC_ASSERT (stream_data->app_tx_data_len >= delta);
839  stream_data->app_tx_data_len -= delta;
840  rv = svm_fifo_dequeue_drop (f, delta);
841  QUIC_ASSERT (rv == delta);
842
843  rv = quicly_stream_sync_sendbuf (stream, 0);
844  QUIC_ASSERT (!rv);
845}
846
847int
848quic_fifo_egress_emit (quicly_stream_t * stream, size_t off, void *dst,
849		       size_t * len, int *wrote_all)
850{
851  u32 deq_max, first_deq, max_rd_chunk, rem_offset;
852  quic_stream_data_t *stream_data;
853  session_t *stream_session;
854  svm_fifo_t *f;
855
856  stream_data = (quic_stream_data_t *) stream->data;
857  stream_session = get_stream_session_from_stream (stream);
858  f = stream_session->tx_fifo;
859
860  QUIC_DBG (3, "Emitting %u, offset %u", *len, off);
861
862  deq_max = svm_fifo_max_dequeue_cons (f);
863  QUIC_ASSERT (off <= deq_max);
864  if (off + *len < deq_max)
865    {
866      *wrote_all = 0;
867    }
868  else
869    {
870      *wrote_all = 1;
871      *len = deq_max - off;
872    }
873  QUIC_ASSERT (*len > 0);
874
875  if (off + *len > stream_data->app_tx_data_len)
876    stream_data->app_tx_data_len = off + *len;
877
878  /* TODO, use something like : return svm_fifo_peek (f, off, *len, dst); */
879  max_rd_chunk = svm_fifo_max_read_chunk (f);
880
881  first_deq = 0;
882  if (off < max_rd_chunk)
883    {
884      first_deq = clib_min (*len, max_rd_chunk - off);
885      clib_memcpy_fast (dst, svm_fifo_head (f) + off, first_deq);
886    }
887
888  if (max_rd_chunk < off + *len)
889    {
890      rem_offset = max_rd_chunk < off ? off - max_rd_chunk : 0;
891      clib_memcpy_fast (dst + first_deq, f->head_chunk->data + rem_offset,
892			*len - first_deq);
893    }
894
895  return 0;
896}
897
898static const quicly_stream_callbacks_t quic_stream_callbacks = {
899  .on_destroy = quic_on_stream_destroy,
900  .on_send_shift = quic_fifo_egress_shift,
901  .on_send_emit = quic_fifo_egress_emit,
902  .on_send_stop = quic_on_stop_sending,
903  .on_receive = quic_on_receive,
904  .on_receive_reset = quic_on_receive_reset
905};
906
907static int
908quic_on_stream_open (quicly_stream_open_t * self, quicly_stream_t * stream)
909{
910  /* Return code for this function ends either
911   * - in quicly_receive : if not QUICLY_ERROR_PACKET_IGNORED, will close connection
912   * - in quicly_open_stream, returned directly
913   */
914
915  session_t *stream_session, *quic_session;
916  quic_stream_data_t *stream_data;
917  app_worker_t *app_wrk;
918  quic_ctx_t *qctx, *sctx;
919  u32 sctx_id;
920  int rv;
921
922  QUIC_DBG (2, "on_stream_open called");
923  stream->data = clib_mem_alloc (sizeof (quic_stream_data_t));
924  stream->callbacks = &quic_stream_callbacks;
925  /* Notify accept on parent qsession, but only if this is not a locally
926   * initiated stream */
927  if (quicly_stream_is_self_initiated (stream))
928    return 0;
929
930  sctx_id = quic_ctx_alloc (vlib_get_thread_index ());
931  qctx = quic_get_conn_ctx (stream->conn);
932
933  /* Might need to signal that the connection is ready if the first thing the
934   * server does is open a stream */
935  quic_check_quic_session_connected (qctx);
936  /* ctx might be invalidated */
937  qctx = quic_get_conn_ctx (stream->conn);
938
939  stream_session = session_alloc (qctx->c_thread_index);
940  QUIC_DBG (2, "ACCEPTED stream_session 0x%lx ctx %u",
941	    session_handle (stream_session), sctx_id);
942  sctx = quic_ctx_get (sctx_id, qctx->c_thread_index);
943  sctx->parent_app_wrk_id = qctx->parent_app_wrk_id;
944  sctx->parent_app_id = qctx->parent_app_id;
945  sctx->quic_connection_ctx_id = qctx->c_c_index;
946  sctx->c_c_index = sctx_id;
947  sctx->c_s_index = stream_session->session_index;
948  sctx->stream = stream;
949  sctx->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP;
950  sctx->flags |= QUIC_F_IS_STREAM;
951  if (quicly_stream_is_unidirectional (stream->stream_id))
952    stream_session->flags |= SESSION_F_UNIDIRECTIONAL;
953
954  stream_data = (quic_stream_data_t *) stream->data;
955  stream_data->ctx_id = sctx_id;
956  stream_data->thread_index = sctx->c_thread_index;
957  stream_data->app_rx_data_len = 0;
958  stream_data->app_tx_data_len = 0;
959
960  sctx->c_s_index = stream_session->session_index;
961  stream_session->session_state = SESSION_STATE_CREATED;
962  stream_session->app_wrk_index = sctx->parent_app_wrk_id;
963  stream_session->connection_index = sctx->c_c_index;
964  stream_session->session_type =
965    session_type_from_proto_and_ip (TRANSPORT_PROTO_QUIC, qctx->udp_is_ip4);
966  quic_session = session_get (qctx->c_s_index, qctx->c_thread_index);
967  stream_session->listener_handle = listen_session_get_handle (quic_session);
968
969  app_wrk = app_worker_get (stream_session->app_wrk_index);
970  if ((rv = app_worker_init_connected (app_wrk, stream_session)))
971    {
972      QUIC_ERR ("failed to allocate fifos");
973      quicly_reset_stream (stream, QUIC_APP_ALLOCATION_ERROR);
974      return 0;			/* Frame is still valid */
975    }
976  svm_fifo_add_want_deq_ntf (stream_session->rx_fifo,
977			     SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL |
978			     SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY);
979
980  if ((rv = app_worker_accept_notify (app_wrk, stream_session)))
981    {
982      QUIC_ERR ("failed to notify accept worker app");
983      quicly_reset_stream (stream, QUIC_APP_ACCEPT_NOTIFY_ERROR);
984      return 0;			/* Frame is still valid */
985    }
986
987  return 0;
988}
989
990static void
991quic_on_closed_by_peer (quicly_closed_by_peer_t * self, quicly_conn_t * conn,
992			int code, uint64_t frame_type,
993			const char *reason, size_t reason_len)
994{
995  quic_ctx_t *ctx = quic_get_conn_ctx (conn);
996#if QUIC_DEBUG >= 2
997  session_t *quic_session = session_get (ctx->c_s_index, ctx->c_thread_index);
998  clib_warning ("Session 0x%lx closed by peer (%U) %.*s ",
999		session_handle (quic_session), quic_format_err, code,
1000		reason_len, reason);
1001#endif
1002  ctx->conn_state = QUIC_CONN_STATE_PASSIVE_CLOSING;
1003  session_transport_closing_notify (&ctx->connection);
1004}
1005
1006/* Timer handling */
1007
1008static int64_t
1009quic_get_thread_time (u8 thread_index)
1010{
1011  return quic_main.wrk_ctx[thread_index].time_now;
1012}
1013
1014static int64_t
1015quic_get_time (quicly_now_t * self)
1016{
1017  u8 thread_index = vlib_get_thread_index ();
1018  return quic_get_thread_time (thread_index);
1019}
1020
1021static u32
1022quic_set_time_now (u32 thread_index)
1023{
1024  vlib_main_t *vlib_main = vlib_get_main ();
1025  f64 time = vlib_time_now (vlib_main);
1026  quic_main.wrk_ctx[thread_index].time_now = (int64_t) (time * 1000.f);
1027  return quic_main.wrk_ctx[thread_index].time_now;
1028}
1029
1030/* Transport proto callback */
1031static void
1032quic_update_time (f64 now, u8 thread_index)
1033{
1034  tw_timer_wheel_1t_3w_1024sl_ov_t *tw;
1035
1036  tw = &quic_main.wrk_ctx[thread_index].timer_wheel;
1037  quic_set_time_now (thread_index);
1038  tw_timer_expire_timers_1t_3w_1024sl_ov (tw, now);
1039}
1040
1041static void
1042quic_timer_expired (u32 conn_index)
1043{
1044  quic_ctx_t *ctx;
1045  QUIC_DBG (4, "Timer expired for conn %u at %ld", conn_index,
1046	    quic_get_time (NULL));
1047  ctx = quic_ctx_get (conn_index, vlib_get_thread_index ());
1048  ctx->timer_handle = QUIC_TIMER_HANDLE_INVALID;
1049  quic_send_packets (ctx);
1050}
1051
1052static void
1053quic_update_timer (quic_ctx_t * ctx)
1054{
1055  tw_timer_wheel_1t_3w_1024sl_ov_t *tw;
1056  int64_t next_timeout, next_interval;
1057  session_t *quic_session;
1058  int rv;
1059
1060  /*  This timeout is in ms which is the unit of our timer */
1061  next_timeout = quicly_get_first_timeout (ctx->conn);
1062  next_interval = next_timeout - quic_get_time (NULL);
1063
1064  if (next_timeout == 0 || next_interval <= 0)
1065    {
1066      if (ctx->c_s_index == QUIC_SESSION_INVALID)
1067	{
1068	  next_interval = 1;
1069	}
1070      else
1071	{
1072	  quic_session = session_get (ctx->c_s_index, ctx->c_thread_index);
1073	  if (svm_fifo_set_event (quic_session->tx_fifo))
1074	    {
1075	      rv = session_send_io_evt_to_thread_custom (quic_session,
1076							 quic_session->thread_index,
1077							 SESSION_IO_EVT_BUILTIN_TX);
1078	      if (PREDICT_FALSE (rv))
1079		QUIC_ERR ("Failed to enqueue builtin_tx %d", rv);
1080	    }
1081	  return;
1082	}
1083    }
1084
1085  tw = &quic_main.wrk_ctx[vlib_get_thread_index ()].timer_wheel;
1086
1087  QUIC_DBG (4, "Timer set to %ld (int %ld) for ctx %u", next_timeout,
1088	    next_interval, ctx->c_c_index);
1089
1090  if (ctx->timer_handle == QUIC_TIMER_HANDLE_INVALID)
1091    {
1092      if (next_timeout == INT64_MAX)
1093	{
1094	  QUIC_DBG (4, "timer for ctx %u already stopped", ctx->c_c_index);
1095	  return;
1096	}
1097      ctx->timer_handle = tw_timer_start_1t_3w_1024sl_ov (tw, ctx->c_c_index,
1098							  0, next_interval);
1099    }
1100  else
1101    {
1102      if (next_timeout == INT64_MAX)
1103	{
1104	  quic_stop_ctx_timer (ctx);
1105	}
1106      else
1107	tw_timer_update_1t_3w_1024sl_ov (tw, ctx->timer_handle,
1108					 next_interval);
1109    }
1110  return;
1111}
1112
1113static void
1114quic_expired_timers_dispatch (u32 * expired_timers)
1115{
1116  int i;
1117
1118  for (i = 0; i < vec_len (expired_timers); i++)
1119    {
1120      quic_timer_expired (expired_timers[i]);
1121    }
1122}
1123
1124/* Transport proto functions */
1125
1126static int
1127quic_connect_stream (session_t * quic_session, session_endpoint_cfg_t * sep)
1128{
1129  uint64_t quic_session_handle;
1130  session_t *stream_session;
1131  quic_stream_data_t *stream_data;
1132  quicly_stream_t *stream;
1133  quicly_conn_t *conn;
1134  app_worker_t *app_wrk;
1135  quic_ctx_t *qctx, *sctx;
1136  u32 sctx_index;
1137  int rv;
1138
1139  /*  Find base session to which the user want to attach a stream */
1140  quic_session_handle = session_handle (quic_session);
1141  QUIC_DBG (2, "Opening new stream (qsession %u)", quic_session_handle);
1142
1143  if (session_type_transport_proto (quic_session->session_type) !=
1144      TRANSPORT_PROTO_QUIC)
1145    {
1146      QUIC_ERR ("received incompatible session");
1147      return -1;
1148    }
1149
1150  app_wrk = app_worker_get_if_valid (quic_session->app_wrk_index);
1151  if (!app_wrk)
1152    {
1153      QUIC_ERR ("Invalid app worker :(");
1154      return -1;
1155    }
1156
1157  sctx_index = quic_ctx_alloc (quic_session->thread_index);	/*  Allocate before we get pointers */
1158  sctx = quic_ctx_get (sctx_index, quic_session->thread_index);
1159  qctx = quic_ctx_get (quic_session->connection_index,
1160		       quic_session->thread_index);
1161  if (quic_ctx_is_stream (qctx))
1162    {
1163      QUIC_ERR ("session is a stream");
1164      quic_ctx_free (sctx);
1165      return -1;
1166    }
1167
1168  sctx->parent_app_wrk_id = qctx->parent_app_wrk_id;
1169  sctx->parent_app_id = qctx->parent_app_id;
1170  sctx->quic_connection_ctx_id = qctx->c_c_index;
1171  sctx->c_c_index = sctx_index;
1172  sctx->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP;
1173  sctx->flags |= QUIC_F_IS_STREAM;
1174
1175  conn = qctx->conn;
1176
1177  if (!conn || !quicly_connection_is_ready (conn))
1178    return -1;
1179
1180  if ((rv =
1181       quicly_open_stream (conn, &stream,
1182			   sep->flags & SESSION_F_UNIDIRECTIONAL)))
1183    {
1184      QUIC_DBG (2, "Stream open failed with %d", rv);
1185      return -1;
1186    }
1187  quic_increment_counter (QUIC_ERROR_OPENED_STREAM, 1);
1188
1189  sctx->stream = stream;
1190
1191  QUIC_DBG (2, "Opened stream %d, creating session", stream->stream_id);
1192
1193  stream_session = session_alloc (qctx->c_thread_index);
1194  QUIC_DBG (2, "Allocated stream_session 0x%lx ctx %u",
1195	    session_handle (stream_session), sctx_index);
1196  stream_session->app_wrk_index = app_wrk->wrk_index;
1197  stream_session->connection_index = sctx_index;
1198  stream_session->listener_handle = quic_session_handle;
1199  stream_session->session_type =
1200    session_type_from_proto_and_ip (TRANSPORT_PROTO_QUIC, qctx->udp_is_ip4);
1201  if (sep->flags & SESSION_F_UNIDIRECTIONAL)
1202    stream_session->flags |= SESSION_F_UNIDIRECTIONAL;
1203
1204  sctx->c_s_index = stream_session->session_index;
1205  stream_data = (quic_stream_data_t *) stream->data;
1206  stream_data->ctx_id = sctx->c_c_index;
1207  stream_data->thread_index = sctx->c_thread_index;
1208  stream_data->app_rx_data_len = 0;
1209  stream_data->app_tx_data_len = 0;
1210  stream_session->session_state = SESSION_STATE_READY;
1211
1212  /* For now we only reset streams. Cleanup will be triggered by timers */
1213  if (app_worker_init_connected (app_wrk, stream_session))
1214    {
1215      QUIC_ERR ("failed to app_worker_init_connected");
1216      quicly_reset_stream (stream, QUIC_APP_CONNECT_NOTIFY_ERROR);
1217      return app_worker_connect_notify (app_wrk, NULL, sep->opaque);
1218    }
1219
1220  svm_fifo_add_want_deq_ntf (stream_session->rx_fifo,
1221			     SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL |
1222			     SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY);
1223
1224  if (app_worker_connect_notify (app_wrk, stream_session, sep->opaque))
1225    {
1226      QUIC_ERR ("failed to notify app");
1227      quic_increment_counter (QUIC_ERROR_CLOSED_STREAM, 1);
1228      quicly_reset_stream (stream, QUIC_APP_CONNECT_NOTIFY_ERROR);
1229      return -1;
1230    }
1231
1232  return 0;
1233}
1234
1235static int
1236quic_connect_connection (session_endpoint_cfg_t * sep)
1237{
1238  vnet_connect_args_t _cargs, *cargs = &_cargs;
1239  quic_main_t *qm = &quic_main;
1240  quic_ctx_t *ctx;
1241  app_worker_t *app_wrk;
1242  application_t *app;
1243  u32 ctx_index;
1244  u32 thread_index = vlib_get_thread_index ();
1245  int error;
1246
1247  clib_memset (cargs, 0, sizeof (*cargs));
1248  ctx_index = quic_ctx_alloc (thread_index);
1249  ctx = quic_ctx_get (ctx_index, thread_index);
1250  ctx->parent_app_wrk_id = sep->app_wrk_index;
1251  ctx->c_s_index = QUIC_SESSION_INVALID;
1252  ctx->c_c_index = ctx_index;
1253  ctx->udp_is_ip4 = sep->is_ip4;
1254  ctx->timer_handle = QUIC_TIMER_HANDLE_INVALID;
1255  ctx->conn_state = QUIC_CONN_STATE_HANDSHAKE;
1256  ctx->client_opaque = sep->opaque;
1257  ctx->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP;
1258  if (sep->hostname)
1259    ctx->srv_hostname = format (0, "%v", sep->hostname);
1260  else
1261    /*  needed by quic for crypto + determining client / server */
1262    ctx->srv_hostname = format (0, "%U", format_ip46_address,
1263				&sep->ip, sep->is_ip4);
1264  vec_terminate_c_string (ctx->srv_hostname);
1265
1266  clib_memcpy (&cargs->sep, sep, sizeof (session_endpoint_cfg_t));
1267  cargs->sep.transport_proto = TRANSPORT_PROTO_UDPC;
1268  cargs->app_index = qm->app_index;
1269  cargs->api_context = ctx_index;
1270
1271  app_wrk = app_worker_get (sep->app_wrk_index);
1272  app = application_get (app_wrk->app_index);
1273  ctx->parent_app_id = app_wrk->app_index;
1274  cargs->sep_ext.ns_index = app->ns_index;
1275
1276  ctx->crypto_engine = sep->crypto_engine;
1277  ctx->ckpair_index = sep->ckpair_index;
1278  if ((error = quic_acquire_crypto_context (ctx)))
1279    return error;
1280
1281  if ((error = vnet_connect (cargs)))
1282    return error;
1283
1284  return 0;
1285}
1286
1287static int
1288quic_connect (transport_endpoint_cfg_t * tep)
1289{
1290  QUIC_DBG (2, "Called quic_connect");
1291  session_endpoint_cfg_t *sep = (session_endpoint_cfg_t *) tep;
1292  session_t *quic_session;
1293  sep = (session_endpoint_cfg_t *) tep;
1294
1295  quic_session = session_get_from_handle_if_valid (sep->parent_handle);
1296  if (quic_session)
1297    return quic_connect_stream (quic_session, sep);
1298  else
1299    return quic_connect_connection (sep);
1300}
1301
1302static void
1303quic_proto_on_close (u32 ctx_index, u32 thread_index)
1304{
1305  quic_ctx_t *ctx = quic_ctx_get_if_valid (ctx_index, thread_index);
1306  if (!ctx)
1307    return;
1308#if QUIC_DEBUG >= 2
1309  session_t *stream_session = session_get (ctx->c_s_index,
1310					   ctx->c_thread_index);
1311  clib_warning ("Closing session 0x%lx", session_handle (stream_session));
1312#endif
1313  if (quic_ctx_is_stream (ctx))
1314    {
1315      quicly_stream_t *stream = ctx->stream;
1316      if (!quicly_stream_has_send_side (quicly_is_client (stream->conn),
1317					stream->stream_id))
1318	return;
1319      quicly_reset_stream (stream, QUIC_APP_ERROR_CLOSE_NOTIFY);
1320      quic_send_packets (ctx);
1321      return;
1322    }
1323
1324  switch (ctx->conn_state)
1325    {
1326    case QUIC_CONN_STATE_OPENED:
1327    case QUIC_CONN_STATE_HANDSHAKE:
1328    case QUIC_CONN_STATE_READY:
1329      ctx->conn_state = QUIC_CONN_STATE_ACTIVE_CLOSING;
1330      quicly_conn_t *conn = ctx->conn;
1331      /* Start connection closing. Keep sending packets until quicly_send
1332         returns QUICLY_ERROR_FREE_CONNECTION */
1333
1334      quic_increment_counter (QUIC_ERROR_CLOSED_CONNECTION, 1);
1335      quicly_close (conn, QUIC_APP_ERROR_CLOSE_NOTIFY, "Closed by peer");
1336      /* This also causes all streams to be closed (and the cb called) */
1337      quic_send_packets (ctx);
1338      break;
1339    case QUIC_CONN_STATE_PASSIVE_CLOSING:
1340      ctx->conn_state = QUIC_CONN_STATE_PASSIVE_CLOSING_APP_CLOSED;
1341      /* send_packets will eventually return an error, we delete the conn at
1342         that point */
1343      break;
1344    case QUIC_CONN_STATE_PASSIVE_CLOSING_QUIC_CLOSED:
1345      quic_connection_delete (ctx);
1346      break;
1347    case QUIC_CONN_STATE_ACTIVE_CLOSING:
1348      break;
1349    default:
1350      QUIC_ERR ("Trying to close conn in state %d", ctx->conn_state);
1351      break;
1352    }
1353}
1354
1355static u32
1356quic_start_listen (u32 quic_listen_session_index, transport_endpoint_t * tep)
1357{
1358  vnet_listen_args_t _bargs, *args = &_bargs;
1359  quic_main_t *qm = &quic_main;
1360  session_handle_t udp_handle;
1361  session_endpoint_cfg_t *sep;
1362  session_t *udp_listen_session;
1363  app_worker_t *app_wrk;
1364  application_t *app;
1365  quic_ctx_t *lctx;
1366  u32 lctx_index;
1367  app_listener_t *app_listener;
1368  int rv;
1369
1370  sep = (session_endpoint_cfg_t *) tep;
1371  app_wrk = app_worker_get (sep->app_wrk_index);
1372  /* We need to call this because we call app_worker_init_connected in
1373   * quic_accept_stream, which assumes the connect segment manager exists */
1374  app_worker_alloc_connects_segment_manager (app_wrk);
1375  app = application_get (app_wrk->app_index);
1376  QUIC_DBG (2, "Called quic_start_listen for app %d", app_wrk->app_index);
1377
1378  sep->transport_proto = TRANSPORT_PROTO_UDPC;
1379  clib_memset (args, 0, sizeof (*args));
1380  args->app_index = qm->app_index;
1381  args->sep_ext = *sep;
1382  args->sep_ext.ns_index = app->ns_index;
1383  if ((rv = vnet_listen (args)))
1384    return rv;
1385
1386  lctx_index = quic_ctx_alloc (0);
1387  udp_handle = args->handle;
1388  app_listener = app_listener_get_w_handle (udp_handle);
1389  udp_listen_session = app_listener_get_session (app_listener);
1390  udp_listen_session->opaque = lctx_index;
1391
1392  lctx = quic_ctx_get (lctx_index, 0);
1393  lctx->flags |= QUIC_F_IS_LISTENER;
1394
1395  clib_memcpy (&lctx->c_rmt_ip, &args->sep.peer.ip, sizeof (ip46_address_t));
1396  clib_memcpy (&lctx->c_lcl_ip, &args->sep.ip, sizeof (ip46_address_t));
1397  lctx->c_rmt_port = args->sep.peer.port;
1398  lctx->c_lcl_port = args->sep.port;
1399  lctx->c_is_ip4 = args->sep.is_ip4;
1400  lctx->c_fib_index = args->sep.fib_index;
1401  lctx->c_proto = TRANSPORT_PROTO_QUIC;
1402  lctx->parent_app_wrk_id = sep->app_wrk_index;
1403  lctx->parent_app_id = app_wrk->app_index;
1404  lctx->udp_session_handle = udp_handle;
1405  lctx->c_s_index = quic_listen_session_index;
1406  lctx->crypto_engine = sep->crypto_engine;
1407  lctx->ckpair_index = sep->ckpair_index;
1408  if (quic_acquire_crypto_context (lctx))
1409    return -1;
1410
1411  QUIC_DBG (2, "Listening UDP session 0x%lx",
1412	    session_handle (udp_listen_session));
1413  QUIC_DBG (2, "Listening QUIC session 0x%lx", quic_listen_session_index);
1414  return lctx_index;
1415}
1416
1417static u32
1418quic_stop_listen (u32 lctx_index)
1419{
1420  QUIC_DBG (2, "Called quic_stop_listen");
1421  quic_ctx_t *lctx;
1422  lctx = quic_ctx_get (lctx_index, 0);
1423  QUIC_ASSERT (quic_ctx_is_listener (lctx));
1424  vnet_unlisten_args_t a = {
1425    .handle = lctx->udp_session_handle,
1426    .app_index = quic_main.app_index,
1427    .wrk_map_index = 0		/* default wrk */
1428  };
1429  if (vnet_unlisten (&a))
1430    clib_warning ("unlisten errored");
1431
1432  quic_release_crypto_context (lctx->crypto_context_index,
1433			       0 /* thread_index */ );
1434  quic_ctx_free (lctx);
1435  return 0;
1436}
1437
1438static transport_connection_t *
1439quic_connection_get (u32 ctx_index, u32 thread_index)
1440{
1441  quic_ctx_t *ctx;
1442  ctx = quic_ctx_get (ctx_index, thread_index);
1443  return &ctx->connection;
1444}
1445
1446static transport_connection_t *
1447quic_listener_get (u32 listener_index)
1448{
1449  QUIC_DBG (2, "Called quic_listener_get");
1450  quic_ctx_t *ctx;
1451  ctx = quic_ctx_get (listener_index, 0);
1452  return &ctx->connection;
1453}
1454
1455static u8 *
1456format_quic_ctx (u8 * s, va_list * args)
1457{
1458  quic_ctx_t *ctx = va_arg (*args, quic_ctx_t *);
1459  u32 verbose = va_arg (*args, u32);
1460  u8 *str = 0;
1461
1462  if (!ctx)
1463    return s;
1464  str = format (str, "[#%d][Q] ", ctx->c_thread_index);
1465
1466  if (quic_ctx_is_listener (ctx))
1467    str = format (str, "Listener, UDP %ld", ctx->udp_session_handle);
1468  else if (quic_ctx_is_stream (ctx))
1469    str = format (str, "Stream %ld conn %d",
1470		  ctx->stream->stream_id, ctx->quic_connection_ctx_id);
1471  else				/* connection */
1472    str = format (str, "Conn %d UDP %d", ctx->c_c_index,
1473		  ctx->udp_session_handle);
1474
1475  str = format (str, " app %d wrk %d", ctx->parent_app_id,
1476		ctx->parent_app_wrk_id);
1477
1478  if (verbose == 1)
1479    s = format (s, "%-50s%-15d", str, ctx->conn_state);
1480  else
1481    s = format (s, "%s\n", str);
1482  vec_free (str);
1483  return s;
1484}
1485
1486static u8 *
1487format_quic_connection (u8 * s, va_list * args)
1488{
1489  u32 qc_index = va_arg (*args, u32);
1490  u32 thread_index = va_arg (*args, u32);
1491  u32 verbose = va_arg (*args, u32);
1492  quic_ctx_t *ctx = quic_ctx_get (qc_index, thread_index);
1493  s = format (s, "%U", format_quic_ctx, ctx, verbose);
1494  return s;
1495}
1496
1497static u8 *
1498format_quic_half_open (u8 * s, va_list * args)
1499{
1500  u32 qc_index = va_arg (*args, u32);
1501  u32 thread_index = va_arg (*args, u32);
1502  quic_ctx_t *ctx = quic_ctx_get (qc_index, thread_index);
1503  s = format (s, "[#%d][Q] half-open app %u", thread_index,
1504	      ctx->parent_app_id);
1505  return s;
1506}
1507
1508/*  TODO improve */
1509static u8 *
1510format_quic_listener (u8 * s, va_list * args)
1511{
1512  u32 tci = va_arg (*args, u32);
1513  u32 thread_index = va_arg (*args, u32);
1514  u32 verbose = va_arg (*args, u32);
1515  quic_ctx_t *ctx = quic_ctx_get (tci, thread_index);
1516  s = format (s, "%U", format_quic_ctx, ctx, verbose);
1517  return s;
1518}
1519
1520/* Session layer callbacks */
1521
1522static inline void
1523quic_build_sockaddr (struct sockaddr *sa, socklen_t * salen,
1524		     ip46_address_t * addr, u16 port, u8 is_ip4)
1525{
1526  if (is_ip4)
1527    {
1528      struct sockaddr_in *sa4 = (struct sockaddr_in *) sa;
1529      sa4->sin_family = AF_INET;
1530      sa4->sin_port = port;
1531      sa4->sin_addr.s_addr = addr->ip4.as_u32;
1532      *salen = sizeof (struct sockaddr_in);
1533    }
1534  else
1535    {
1536      struct sockaddr_in6 *sa6 = (struct sockaddr_in6 *) sa;
1537      sa6->sin6_family = AF_INET6;
1538      sa6->sin6_port = port;
1539      clib_memcpy (&sa6->sin6_addr, &addr->ip6, 16);
1540      *salen = sizeof (struct sockaddr_in6);
1541    }
1542}
1543
1544static void
1545quic_on_quic_session_connected (quic_ctx_t * ctx)
1546{
1547  session_t *quic_session;
1548  app_worker_t *app_wrk;
1549  u32 ctx_id = ctx->c_c_index;
1550  u32 thread_index = ctx->c_thread_index;
1551  int rv;
1552
1553  quic_session = session_alloc (thread_index);
1554
1555  QUIC_DBG (2, "Allocated quic session 0x%lx", session_handle (quic_session));
1556  ctx->c_s_index = quic_session->session_index;
1557  quic_session->app_wrk_index = ctx->parent_app_wrk_id;
1558  quic_session->connection_index = ctx->c_c_index;
1559  quic_session->listener_handle = SESSION_INVALID_HANDLE;
1560  quic_session->session_type =
1561    session_type_from_proto_and_ip (TRANSPORT_PROTO_QUIC, ctx->udp_is_ip4);
1562
1563  /* If quic session connected fails, immediatly close connection */
1564  app_wrk = app_worker_get (ctx->parent_app_wrk_id);
1565  if (app_worker_init_connected (app_wrk, quic_session))
1566    {
1567      QUIC_ERR ("failed to app_worker_init_connected");
1568      quic_proto_on_close (ctx_id, thread_index);
1569      app_worker_connect_notify (app_wrk, NULL, ctx->client_opaque);
1570      return;
1571    }
1572
1573  quic_session->session_state = SESSION_STATE_CONNECTING;
1574  if ((rv = app_worker_connect_notify (app_wrk, quic_session,
1575				       ctx->client_opaque)))
1576    {
1577      QUIC_ERR ("failed to notify app %d", rv);
1578      quic_proto_on_close (ctx_id, thread_index);
1579      return;
1580    }
1581
1582  /*  If the app opens a stream in its callback it may invalidate ctx */
1583  ctx = quic_ctx_get (ctx_id, thread_index);
1584  /*
1585   * app_worker_connect_notify() might have reallocated pool, reload
1586   * quic_session pointer
1587   */
1588  quic_session = session_get (ctx->c_s_index, thread_index);
1589  quic_session->session_state = SESSION_STATE_LISTENING;
1590}
1591
1592static void
1593quic_check_quic_session_connected (quic_ctx_t * ctx)
1594{
1595  /* Called when we need to trigger quic session connected
1596   * we may call this function on the server side / at
1597   * stream opening */
1598
1599  /* Conn may be set to null if the connection is terminated */
1600  if (!ctx->conn || ctx->conn_state != QUIC_CONN_STATE_HANDSHAKE)
1601    return;
1602  if (!quicly_connection_is_ready (ctx->conn))
1603    return;
1604  ctx->conn_state = QUIC_CONN_STATE_READY;
1605  if (!quicly_is_client (ctx->conn))
1606    return;
1607  quic_on_quic_session_connected (ctx);
1608}
1609
1610static void
1611quic_receive_connection (void *arg)
1612{
1613  u32 new_ctx_id, thread_index = vlib_get_thread_index ();
1614  quic_ctx_t *temp_ctx, *new_ctx;
1615  clib_bihash_kv_16_8_t kv;
1616  quicly_conn_t *conn;
1617  session_t *udp_session;
1618
1619  temp_ctx = arg;
1620  new_ctx_id = quic_ctx_alloc (thread_index);
1621  new_ctx = quic_ctx_get (new_ctx_id, thread_index);
1622
1623  QUIC_DBG (2, "Received conn %u (now %u)", temp_ctx->c_thread_index,
1624	    new_ctx_id);
1625
1626  clib_memcpy (new_ctx, temp_ctx, sizeof (quic_ctx_t));
1627  clib_mem_free (temp_ctx);
1628
1629  new_ctx->c_thread_index = thread_index;
1630  new_ctx->c_c_index = new_ctx_id;
1631  quic_acquire_crypto_context (new_ctx);
1632
1633  conn = new_ctx->conn;
1634  quic_store_conn_ctx (conn, new_ctx);
1635  quic_make_connection_key (&kv, quicly_get_master_id (conn));
1636  kv.value = ((u64) thread_index) << 32 | (u64) new_ctx_id;
1637  QUIC_DBG (2, "Registering conn with id %lu %lu", kv.key[0], kv.key[1]);
1638  clib_bihash_add_del_16_8 (&quic_main.connection_hash, &kv, 1 /* is_add */ );
1639  new_ctx->timer_handle = QUIC_TIMER_HANDLE_INVALID;
1640  quic_update_timer (new_ctx);
1641
1642  /*  Trigger write on this connection if necessary */
1643  udp_session = session_get_from_handle (new_ctx->udp_session_handle);
1644  udp_session->opaque = new_ctx_id;
1645  udp_session->flags &= ~SESSION_F_IS_MIGRATING;
1646  if (svm_fifo_max_dequeue (udp_session->tx_fifo))
1647    quic_set_udp_tx_evt (udp_session);
1648}
1649
1650static void
1651quic_transfer_connection (u32 ctx_index, u32 dest_thread)
1652{
1653  quic_ctx_t *ctx, *temp_ctx;
1654  u32 thread_index = vlib_get_thread_index ();
1655
1656  QUIC_DBG (2, "Transferring conn %u to thread %u", ctx_index, dest_thread);
1657
1658  temp_ctx = clib_mem_alloc (sizeof (quic_ctx_t));
1659  QUIC_ASSERT (temp_ctx != NULL);
1660  ctx = quic_ctx_get (ctx_index, thread_index);
1661
1662  clib_memcpy (temp_ctx, ctx, sizeof (quic_ctx_t));
1663
1664  quic_stop_ctx_timer (ctx);
1665  quic_release_crypto_context (ctx->crypto_context_index, thread_index);
1666  quic_ctx_free (ctx);
1667
1668  /*  Send connection to destination thread */
1669  session_send_rpc_evt_to_thread (dest_thread, quic_receive_connection,
1670				  (void *) temp_ctx);
1671}
1672
1673static int
1674quic_udp_session_connected_callback (u32 quic_app_index, u32 ctx_index,
1675				     session_t * udp_session, u8 is_fail)
1676{
1677  QUIC_DBG (2, "QSession is now connected (id %u)",
1678	    udp_session->session_index);
1679  /* This should always be called before quic_connect returns since UDP always
1680   * connects instantly. */
1681  clib_bihash_kv_16_8_t kv;
1682  struct sockaddr_in6 sa6;
1683  struct sockaddr *sa = (struct sockaddr *) &sa6;
1684  socklen_t salen;
1685  transport_connection_t *tc;
1686  app_worker_t *app_wrk;
1687  quicly_conn_t *conn;
1688  quic_ctx_t *ctx;
1689  u32 thread_index = vlib_get_thread_index ();
1690  int ret;
1691  quicly_context_t *quicly_ctx;
1692
1693
1694  ctx = quic_ctx_get (ctx_index, thread_index);
1695  if (is_fail)
1696    {
1697      u32 api_context;
1698      app_wrk = app_worker_get_if_valid (ctx->parent_app_wrk_id);
1699      if (app_wrk)
1700	{
1701	  api_context = ctx->c_s_index;
1702	  app_worker_connect_notify (app_wrk, 0, api_context);
1703	}
1704      return 0;
1705    }
1706
1707  ctx->c_thread_index = thread_index;
1708  ctx->c_c_index = ctx_index;
1709
1710  QUIC_DBG (2, "Quic connect returned %u. New ctx [%u]%x",
1711	    is_fail, thread_index, (ctx) ? ctx_index : ~0);
1712
1713  ctx->udp_session_handle = session_handle (udp_session);
1714  udp_session->opaque = ctx_index;
1715
1716  /* Init QUIC lib connection
1717   * Generate required sockaddr & salen */
1718  tc = session_get_transport (udp_session);
1719  quic_build_sockaddr (sa, &salen, &tc->rmt_ip, tc->rmt_port, tc->is_ip4);
1720
1721  quicly_ctx = quic_get_quicly_ctx_from_ctx (ctx);
1722  ret = quicly_connect (&ctx->conn, quicly_ctx, (char *) ctx->srv_hostname,
1723			sa, NULL, &quic_main.wrk_ctx[thread_index].next_cid,
1724			ptls_iovec_init (NULL, 0), &quic_main.hs_properties,
1725			NULL);
1726  ++quic_main.wrk_ctx[thread_index].next_cid.master_id;
1727  /*  Save context handle in quicly connection */
1728  quic_store_conn_ctx (ctx->conn, ctx);
1729  assert (ret == 0);
1730
1731  /*  Register connection in connections map */
1732  conn = ctx->conn;
1733  quic_make_connection_key (&kv, quicly_get_master_id (conn));
1734  kv.value = ((u64) thread_index) << 32 | (u64) ctx_index;
1735  QUIC_DBG (2, "Registering conn with id %lu %lu", kv.key[0], kv.key[1]);
1736  clib_bihash_add_del_16_8 (&quic_main.connection_hash, &kv, 1 /* is_add */ );
1737
1738  /*  UDP stack quirk? preemptively transfer connection if that happens */
1739  if (udp_session->thread_index != thread_index)
1740    quic_transfer_connection (ctx_index, udp_session->thread_index);
1741  else
1742    quic_send_packets (ctx);
1743
1744  return ret;
1745}
1746
1747static void
1748quic_udp_session_disconnect_callback (session_t * s)
1749{
1750  clib_warning ("UDP session disconnected???");
1751}
1752
1753static void
1754quic_udp_session_cleanup_callback (session_t * udp_session,
1755				   session_cleanup_ntf_t ntf)
1756{
1757  quic_ctx_t *ctx;
1758
1759  if (ntf != SESSION_CLEANUP_SESSION)
1760    return;
1761
1762  ctx = quic_ctx_get (udp_session->opaque, udp_session->thread_index);
1763  quic_stop_ctx_timer (ctx);
1764  quic_release_crypto_context (ctx->crypto_context_index,
1765			       ctx->c_thread_index);
1766  quic_ctx_free (ctx);
1767}
1768
1769static void
1770quic_udp_session_reset_callback (session_t * s)
1771{
1772  clib_warning ("UDP session reset???");
1773}
1774
1775static void
1776quic_udp_session_migrate_callback (session_t * s, session_handle_t new_sh)
1777{
1778  u32 new_thread = session_thread_from_handle (new_sh);
1779  quic_ctx_t *ctx;
1780
1781  QUIC_DBG (2, "Session %x migrated to %lx", s->session_index, new_sh);
1782  QUIC_ASSERT (vlib_get_thread_index () == s->thread_index);
1783  ctx = quic_ctx_get (s->opaque, s->thread_index);
1784  QUIC_ASSERT (ctx->udp_session_handle == session_handle (s));
1785
1786  ctx->udp_session_handle = new_sh;
1787#if QUIC_DEBUG >= 1
1788  s->opaque = 0xfeedface;
1789#endif
1790  quic_transfer_connection (ctx->c_c_index, new_thread);
1791}
1792
1793int
1794quic_udp_session_accepted_callback (session_t * udp_session)
1795{
1796  /* New UDP connection, try to accept it */
1797  u32 ctx_index;
1798  quic_ctx_t *ctx, *lctx;
1799  session_t *udp_listen_session;
1800  u32 thread_index = vlib_get_thread_index ();
1801
1802  udp_listen_session =
1803    listen_session_get_from_handle (udp_session->listener_handle);
1804
1805  ctx_index = quic_ctx_alloc (thread_index);
1806  ctx = quic_ctx_get (ctx_index, thread_index);
1807  ctx->c_thread_index = udp_session->thread_index;
1808  ctx->c_c_index = ctx_index;
1809  ctx->c_s_index = QUIC_SESSION_INVALID;
1810  ctx->udp_session_handle = session_handle (udp_session);
1811  QUIC_DBG (2, "ACCEPTED UDP 0x%lx", ctx->udp_session_handle);
1812  ctx->listener_ctx_id = udp_listen_session->opaque;
1813  lctx = quic_ctx_get (udp_listen_session->opaque,
1814		       udp_listen_session->thread_index);
1815  ctx->udp_is_ip4 = lctx->c_is_ip4;
1816  ctx->parent_app_id = lctx->parent_app_id;
1817  ctx->parent_app_wrk_id = lctx->parent_app_wrk_id;
1818  ctx->timer_handle = QUIC_TIMER_HANDLE_INVALID;
1819  ctx->conn_state = QUIC_CONN_STATE_OPENED;
1820  ctx->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP;
1821
1822  ctx->crypto_engine = lctx->crypto_engine;
1823  ctx->ckpair_index = lctx->ckpair_index;
1824  quic_acquire_crypto_context (ctx);
1825  udp_session->opaque = ctx_index;
1826
1827  /* TODO timeout to delete these if they never connect */
1828  return 0;
1829}
1830
1831static int
1832quic_add_segment_callback (u32 client_index, u64 seg_handle)
1833{
1834  /* No-op for builtin */
1835  return 0;
1836}
1837
1838static int
1839quic_del_segment_callback (u32 client_index, u64 seg_handle)
1840{
1841  /* No-op for builtin */
1842  return 0;
1843}
1844
1845static int
1846quic_custom_app_rx_callback (transport_connection_t * tc)
1847{
1848  quic_ctx_t *ctx;
1849  session_t *stream_session = session_get (tc->s_index, tc->thread_index);
1850  QUIC_DBG (3, "Received app READ notification");
1851  quic_ack_rx_data (stream_session);
1852  svm_fifo_reset_has_deq_ntf (stream_session->rx_fifo);
1853
1854  /* Need to send packets (acks may never be sent otherwise) */
1855  ctx = quic_ctx_get (stream_session->connection_index,
1856		      stream_session->thread_index);
1857  quic_send_packets (ctx);
1858  return 0;
1859}
1860
1861static int
1862quic_custom_tx_callback (void *s, u32 max_burst_size)
1863{
1864  session_t *stream_session = (session_t *) s;
1865  quic_stream_data_t *stream_data;
1866  quicly_stream_t *stream;
1867  quic_ctx_t *ctx;
1868  u32 max_deq;
1869  int rv;
1870
1871  if (PREDICT_FALSE
1872      (stream_session->session_state >= SESSION_STATE_TRANSPORT_CLOSING))
1873    return 0;
1874  ctx = quic_ctx_get (stream_session->connection_index,
1875		      stream_session->thread_index);
1876  if (PREDICT_FALSE (!quic_ctx_is_stream (ctx)))
1877    {
1878      goto tx_end;		/* Most probably a reschedule */
1879    }
1880
1881  QUIC_DBG (3, "Stream TX event");
1882  quic_ack_rx_data (stream_session);
1883  stream = ctx->stream;
1884  if (!quicly_sendstate_is_open (&stream->sendstate))
1885    {
1886      QUIC_ERR ("Warning: tried to send on closed stream");
1887      return -1;
1888    }
1889
1890  stream_data = (quic_stream_data_t *) stream->data;
1891  max_deq = svm_fifo_max_dequeue (stream_session->tx_fifo);
1892  QUIC_ASSERT (max_deq >= stream_data->app_tx_data_len);
1893  if (max_deq == stream_data->app_tx_data_len)
1894    {
1895      QUIC_DBG (3, "TX but no data %d / %d", max_deq,
1896		stream_data->app_tx_data_len);
1897      return 0;
1898    }
1899  stream_data->app_tx_data_len = max_deq;
1900  rv = quicly_stream_sync_sendbuf (stream, 1);
1901  QUIC_ASSERT (!rv);
1902
1903tx_end:
1904  quic_send_packets (ctx);
1905  return 0;
1906}
1907
1908/*
1909 * Returns 0 if a matching connection is found and is on the right thread.
1910 * Otherwise returns -1.
1911 * If a connection is found, even on the wrong thread, ctx_thread and ctx_index
1912 * will be set.
1913 */
1914static inline int
1915quic_find_packet_ctx (quic_rx_packet_ctx_t * pctx, u32 caller_thread_index)
1916{
1917  clib_bihash_kv_16_8_t kv;
1918  clib_bihash_16_8_t *h;
1919  quic_ctx_t *ctx;
1920  u32 index, thread_id;
1921
1922  h = &quic_main.connection_hash;
1923  quic_make_connection_key (&kv, &pctx->packet.cid.dest.plaintext);
1924  QUIC_DBG (3, "Searching conn with id %lu %lu", kv.key[0], kv.key[1]);
1925
1926  if (clib_bihash_search_16_8 (h, &kv, &kv))
1927    {
1928      QUIC_DBG (3, "connection not found");
1929      return QUIC_PACKET_TYPE_NONE;
1930    }
1931
1932  index = kv.value & UINT32_MAX;
1933  thread_id = kv.value >> 32;
1934  /* Check if this connection belongs to this thread, otherwise
1935   * ask for it to be moved */
1936  if (thread_id != caller_thread_index)
1937    {
1938      QUIC_DBG (2, "Connection is on wrong thread");
1939      /* Cannot make full check with quicly_is_destination... */
1940      pctx->ctx_index = index;
1941      pctx->thread_index = thread_id;
1942      return QUIC_PACKET_TYPE_MIGRATE;
1943    }
1944  ctx = quic_ctx_get (index, vlib_get_thread_index ());
1945  if (!ctx->conn)
1946    {
1947      QUIC_ERR ("ctx has no conn");
1948      return QUIC_PACKET_TYPE_NONE;
1949    }
1950  if (!quicly_is_destination (ctx->conn, NULL, &pctx->sa, &pctx->packet))
1951    return QUIC_PACKET_TYPE_NONE;
1952
1953  QUIC_DBG (3, "Connection found");
1954  pctx->ctx_index = index;
1955  pctx->thread_index = thread_id;
1956  return QUIC_PACKET_TYPE_RECEIVE;
1957}
1958
1959static int
1960quic_accept_connection (u32 ctx_index, quic_rx_packet_ctx_t * pctx)
1961{
1962  u32 thread_index = vlib_get_thread_index ();
1963  quicly_context_t *quicly_ctx;
1964  session_t *quic_session;
1965  clib_bihash_kv_16_8_t kv;
1966  app_worker_t *app_wrk;
1967  quicly_conn_t *conn;
1968  quic_ctx_t *ctx;
1969  quic_ctx_t *lctx;
1970  int rv;
1971
1972  /* new connection, accept and create context if packet is valid
1973   * TODO: check if socket is actually listening? */
1974  ctx = quic_ctx_get (ctx_index, thread_index);
1975  if (ctx->c_s_index != QUIC_SESSION_INVALID)
1976    {
1977      QUIC_DBG (2, "already accepted ctx 0x%x", ctx_index);
1978      return 0;
1979    }
1980
1981  quicly_ctx = quic_get_quicly_ctx_from_ctx (ctx);
1982  if ((rv = quicly_accept (&conn, quicly_ctx, NULL, &pctx->sa,
1983			   &pctx->packet, NULL,
1984			   &quic_main.wrk_ctx[thread_index].next_cid, NULL)))
1985    {
1986      /* Invalid packet, pass */
1987      assert (conn == NULL);
1988      QUIC_ERR ("Accept failed with %U", quic_format_err, rv);
1989      /* TODO: cleanup created quic ctx and UDP session */
1990      return 0;
1991    }
1992  assert (conn != NULL);
1993
1994  ++quic_main.wrk_ctx[thread_index].next_cid.master_id;
1995  /* Save ctx handle in quicly connection */
1996  quic_store_conn_ctx (conn, ctx);
1997  ctx->conn = conn;
1998
1999  quic_session = session_alloc (ctx->c_thread_index);
2000  QUIC_DBG (2, "Allocated quic_session, 0x%lx ctx %u",
2001	    session_handle (quic_session), ctx->c_c_index);
2002  quic_session->session_state = SESSION_STATE_LISTENING;
2003  ctx->c_s_index = quic_session->session_index;
2004
2005  lctx = quic_ctx_get (ctx->listener_ctx_id, 0);
2006
2007  quic_session->app_wrk_index = lctx->parent_app_wrk_id;
2008  quic_session->connection_index = ctx->c_c_index;
2009  quic_session->session_type =
2010    session_type_from_proto_and_ip (TRANSPORT_PROTO_QUIC, ctx->udp_is_ip4);
2011  quic_session->listener_handle = lctx->c_s_index;
2012
2013  /* Register connection in connections map */
2014  quic_make_connection_key (&kv, quicly_get_master_id (conn));
2015  kv.value = ((u64) thread_index) << 32 | (u64) ctx_index;
2016  clib_bihash_add_del_16_8 (&quic_main.connection_hash, &kv, 1 /* is_add */ );
2017  QUIC_DBG (2, "Registering conn with id %lu %lu", kv.key[0], kv.key[1]);
2018
2019  /* If notify fails, reset connection immediatly */
2020  if ((rv = app_worker_init_accepted (quic_session)))
2021    {
2022      QUIC_ERR ("failed to allocate fifos");
2023      quic_proto_on_close (ctx_index, thread_index);
2024      return rv;
2025    }
2026
2027  app_wrk = app_worker_get (quic_session->app_wrk_index);
2028  if ((rv = app_worker_accept_notify (app_wrk, quic_session)))
2029    {
2030      QUIC_ERR ("failed to notify accept worker app");
2031      quic_proto_on_close (ctx_index, thread_index);
2032      return rv;
2033    }
2034
2035  ctx->conn_state = QUIC_CONN_STATE_READY;
2036  pctx->ctx_index = ctx_index;
2037  pctx->thread_index = thread_index;
2038
2039  return 0;
2040}
2041
2042static int
2043quic_reset_connection (u64 udp_session_handle, quic_rx_packet_ctx_t * pctx)
2044{
2045  /* short header packet; potentially a dead connection. No need to check the
2046   * length of the incoming packet, because loop is prevented by authenticating
2047   * the CID (by checking node_id and thread_id). If the peer is also sending a
2048   * reset, then the next CID is highly likely to contain a non-authenticating
2049   * CID, ... */
2050  QUIC_DBG (2, "Sending stateless reset");
2051  int rv;
2052  quicly_datagram_t *dgram;
2053  session_t *udp_session;
2054  quicly_context_t *quicly_ctx;
2055  if (pctx->packet.cid.dest.plaintext.node_id != 0
2056      || pctx->packet.cid.dest.plaintext.thread_id != 0)
2057    return 0;
2058  quicly_ctx = quic_get_quicly_ctx_from_udp (udp_session_handle);
2059  dgram = quicly_send_stateless_reset (quicly_ctx, &pctx->sa, NULL,
2060				       &pctx->packet.cid.dest.plaintext);
2061  if (dgram == NULL)
2062    return 1;
2063  udp_session = session_get_from_handle (udp_session_handle);
2064  rv = quic_send_datagram (udp_session, dgram);
2065  quic_set_udp_tx_evt (udp_session);
2066  return rv;
2067}
2068
2069static int
2070quic_process_one_rx_packet (u64 udp_session_handle, svm_fifo_t * f,
2071			    u32 fifo_offset, quic_rx_packet_ctx_t * pctx)
2072{
2073  size_t plen;
2074  u32 full_len, ret;
2075  u32 thread_index = vlib_get_thread_index ();
2076  u32 cur_deq = svm_fifo_max_dequeue (f) - fifo_offset;
2077  quicly_context_t *quicly_ctx;
2078  int rv;
2079
2080  ret = svm_fifo_peek (f, fifo_offset,
2081		       SESSION_CONN_HDR_LEN, (u8 *) & pctx->ph);
2082  QUIC_ASSERT (ret == SESSION_CONN_HDR_LEN);
2083  QUIC_ASSERT (pctx->ph.data_offset == 0);
2084  full_len = pctx->ph.data_length + SESSION_CONN_HDR_LEN;
2085  if (full_len > cur_deq)
2086    {
2087      QUIC_ERR ("Not enough data in fifo RX");
2088      return 1;
2089    }
2090
2091  /* Quicly can read len bytes from the fifo at offset:
2092   * ph.data_offset + SESSION_CONN_HDR_LEN */
2093  ret = svm_fifo_peek (f, SESSION_CONN_HDR_LEN + fifo_offset,
2094		       pctx->ph.data_length, pctx->data);
2095  if (ret != pctx->ph.data_length)
2096    {
2097      QUIC_ERR ("Not enough data peeked in RX");
2098      return 1;
2099    }
2100
2101  quic_increment_counter (QUIC_ERROR_RX_PACKETS, 1);
2102  quic_build_sockaddr (&pctx->sa, &pctx->salen, &pctx->ph.rmt_ip,
2103		       pctx->ph.rmt_port, pctx->ph.is_ip4);
2104  quicly_ctx = quic_get_quicly_ctx_from_udp (udp_session_handle);
2105  plen = quicly_decode_packet (quicly_ctx, &pctx->packet,
2106			       pctx->data, pctx->ph.data_length);
2107
2108  if (plen == SIZE_MAX)
2109    {
2110      return 1;
2111    }
2112
2113  rv = quic_find_packet_ctx (pctx, thread_index);
2114  if (rv == QUIC_PACKET_TYPE_RECEIVE)
2115    {
2116      pctx->ptype = QUIC_PACKET_TYPE_RECEIVE;
2117      return 0;
2118    }
2119  else if (rv == QUIC_PACKET_TYPE_MIGRATE)
2120    {
2121      pctx->ptype = QUIC_PACKET_TYPE_MIGRATE;
2122      /*  Connection found but on wrong thread, ask move */
2123    }
2124  else if (QUICLY_PACKET_IS_LONG_HEADER (pctx->packet.octets.base[0]))
2125    {
2126      pctx->ptype = QUIC_PACKET_TYPE_ACCEPT;
2127    }
2128  else
2129    {
2130      pctx->ptype = QUIC_PACKET_TYPE_RESET;
2131    }
2132  return 1;
2133}
2134
2135static int
2136quic_udp_session_rx_callback (session_t * udp_session)
2137{
2138  /*  Read data from UDP rx_fifo and pass it to the quicly conn. */
2139  quic_ctx_t *ctx = NULL;
2140  svm_fifo_t *f = udp_session->rx_fifo;
2141  u32 max_deq;
2142  u64 udp_session_handle = session_handle (udp_session);
2143  int rv = 0;
2144  u32 thread_index = vlib_get_thread_index ();
2145  u32 cur_deq, fifo_offset, max_packets, i;
2146
2147  quic_rx_packet_ctx_t packets_ctx[QUIC_RCV_MAX_BATCH_PACKETS];
2148
2149  if (udp_session->flags & SESSION_F_IS_MIGRATING)
2150    {
2151      QUIC_DBG (3, "RX on migrating udp session");
2152      return 0;
2153    }
2154
2155rx_start:
2156  max_deq = svm_fifo_max_dequeue (f);
2157  if (max_deq == 0)
2158    return 0;
2159
2160  fifo_offset = 0;
2161  max_packets = QUIC_RCV_MAX_BATCH_PACKETS;
2162
2163  for (i = 0; i < max_packets; i++)
2164    {
2165      packets_ctx[i].thread_index = UINT32_MAX;
2166      packets_ctx[i].ctx_index = UINT32_MAX;
2167      packets_ctx[i].ptype = QUIC_PACKET_TYPE_DROP;
2168
2169      cur_deq = max_deq - fifo_offset;
2170      if (cur_deq == 0)
2171	{
2172	  max_packets = i + 1;
2173	  break;
2174	}
2175      if (cur_deq < SESSION_CONN_HDR_LEN)
2176	{
2177	  fifo_offset = max_deq;
2178	  max_packets = i + 1;
2179	  QUIC_ERR ("Fifo %d < header size in RX", cur_deq);
2180	  break;
2181	}
2182      rv = quic_process_one_rx_packet (udp_session_handle, f,
2183				       fifo_offset, &packets_ctx[i]);
2184      if (packets_ctx[i].ptype != QUIC_PACKET_TYPE_MIGRATE)
2185	fifo_offset += SESSION_CONN_HDR_LEN + packets_ctx[i].ph.data_length;
2186      if (rv)
2187	{
2188	  max_packets = i + 1;
2189	  break;
2190	}
2191    }
2192
2193  for (i = 0; i < max_packets; i++)
2194    {
2195      switch (packets_ctx[i].ptype)
2196	{
2197	case QUIC_PACKET_TYPE_RECEIVE:
2198	  ctx = quic_ctx_get (packets_ctx[i].ctx_index, thread_index);
2199	  rv = quicly_receive (ctx->conn, NULL, &packets_ctx[i].sa,
2200			       &packets_ctx[i].packet);
2201	  if (rv && rv != QUICLY_ERROR_PACKET_IGNORED)
2202	    {
2203	      QUIC_ERR ("quicly_receive return error %U",
2204			quic_format_err, rv);
2205	    }
2206	  break;
2207	case QUIC_PACKET_TYPE_ACCEPT:
2208	  udp_session = session_get_from_handle (udp_session_handle);
2209	  if ((rv = quic_accept_connection (udp_session->opaque,
2210					    &packets_ctx[i])))
2211	    {
2212	      QUIC_ERR ("quic accept errored with %d", rv);
2213	    }
2214	  break;
2215	case QUIC_PACKET_TYPE_RESET:
2216	  quic_reset_connection (udp_session_handle, &packets_ctx[i]);
2217	  break;
2218	}
2219    }
2220  for (i = 0; i < max_packets; i++)
2221    {
2222      switch (packets_ctx[i].ptype)
2223	{
2224	case QUIC_PACKET_TYPE_RECEIVE:
2225	  ctx = quic_ctx_get (packets_ctx[i].ctx_index,
2226			      packets_ctx[i].thread_index);
2227	  quic_check_quic_session_connected (ctx);
2228	  ctx = quic_ctx_get (packets_ctx[i].ctx_index,
2229			      packets_ctx[i].thread_index);
2230	  break;
2231	case QUIC_PACKET_TYPE_ACCEPT:
2232	  ctx = quic_ctx_get (packets_ctx[i].ctx_index,
2233			      packets_ctx[i].thread_index);
2234	  break;
2235	default:
2236	  continue;
2237	}
2238      quic_send_packets (ctx);
2239    }
2240
2241  udp_session = session_get_from_handle (udp_session_handle);	/*  session alloc might have happened */
2242  f = udp_session->rx_fifo;
2243  svm_fifo_dequeue_drop (f, fifo_offset);
2244
2245  if (svm_fifo_max_dequeue (f))
2246    goto rx_start;
2247
2248  return 0;
2249}
2250
2251always_inline void
2252quic_common_get_transport_endpoint (quic_ctx_t * ctx,
2253				    transport_endpoint_t * tep, u8 is_lcl)
2254{
2255  session_t *udp_session;
2256  if (!quic_ctx_is_stream (ctx))
2257    {
2258      udp_session = session_get_from_handle (ctx->udp_session_handle);
2259      session_get_endpoint (udp_session, tep, is_lcl);
2260    }
2261}
2262
2263static void
2264quic_get_transport_listener_endpoint (u32 listener_index,
2265				      transport_endpoint_t * tep, u8 is_lcl)
2266{
2267  quic_ctx_t *ctx;
2268  app_listener_t *app_listener;
2269  session_t *udp_listen_session;
2270  ctx = quic_ctx_get (listener_index, vlib_get_thread_index ());
2271  if (quic_ctx_is_listener (ctx))
2272    {
2273      app_listener = app_listener_get_w_handle (ctx->udp_session_handle);
2274      udp_listen_session = app_listener_get_session (app_listener);
2275      return session_get_endpoint (udp_listen_session, tep, is_lcl);
2276    }
2277  quic_common_get_transport_endpoint (ctx, tep, is_lcl);
2278}
2279
2280static void
2281quic_get_transport_endpoint (u32 ctx_index, u32 thread_index,
2282			     transport_endpoint_t * tep, u8 is_lcl)
2283{
2284  quic_ctx_t *ctx;
2285  ctx = quic_ctx_get (ctx_index, thread_index);
2286  quic_common_get_transport_endpoint (ctx, tep, is_lcl);
2287}
2288
2289/* *INDENT-OFF* */
2290static session_cb_vft_t quic_app_cb_vft = {
2291  .session_accept_callback = quic_udp_session_accepted_callback,
2292  .session_disconnect_callback = quic_udp_session_disconnect_callback,
2293  .session_connected_callback = quic_udp_session_connected_callback,
2294  .session_reset_callback = quic_udp_session_reset_callback,
2295  .session_migrate_callback = quic_udp_session_migrate_callback,
2296  .add_segment_callback = quic_add_segment_callback,
2297  .del_segment_callback = quic_del_segment_callback,
2298  .builtin_app_rx_callback = quic_udp_session_rx_callback,
2299  .session_cleanup_callback = quic_udp_session_cleanup_callback,
2300  .app_cert_key_pair_delete_callback = quic_app_cert_key_pair_delete_callback,
2301};
2302
2303static const transport_proto_vft_t quic_proto = {
2304  .connect = quic_connect,
2305  .close = quic_proto_on_close,
2306  .start_listen = quic_start_listen,
2307  .stop_listen = quic_stop_listen,
2308  .get_connection = quic_connection_get,
2309  .get_listener = quic_listener_get,
2310  .update_time = quic_update_time,
2311  .app_rx_evt = quic_custom_app_rx_callback,
2312  .custom_tx = quic_custom_tx_callback,
2313  .format_connection = format_quic_connection,
2314  .format_half_open = format_quic_half_open,
2315  .format_listener = format_quic_listener,
2316  .get_transport_endpoint = quic_get_transport_endpoint,
2317  .get_transport_listener_endpoint = quic_get_transport_listener_endpoint,
2318  .transport_options = {
2319    .tx_type = TRANSPORT_TX_INTERNAL,
2320    .service_type = TRANSPORT_SERVICE_APP,
2321  },
2322};
2323/* *INDENT-ON* */
2324
2325static quicly_stream_open_t on_stream_open = { quic_on_stream_open };
2326static quicly_closed_by_peer_t on_closed_by_peer = { quic_on_closed_by_peer };
2327static quicly_now_t quicly_vpp_now_cb = { quic_get_time };
2328
2329static void
2330quic_register_cipher_suite (crypto_engine_type_t type,
2331			    ptls_cipher_suite_t ** ciphers)
2332{
2333  quic_main_t *qm = &quic_main;
2334  vec_validate (qm->quic_ciphers, type);
2335  clib_bitmap_set (qm->available_crypto_engines, type, 1);
2336  qm->quic_ciphers[type] = ciphers;
2337}
2338
2339static void
2340quic_update_fifo_size ()
2341{
2342  quic_main_t *qm = &quic_main;
2343  segment_manager_props_t *seg_mgr_props =
2344    application_get_segment_manager_properties (qm->app_index);
2345
2346  if (!seg_mgr_props)
2347    {
2348      clib_warning
2349	("error while getting segment_manager_props_t, can't update fifo-size");
2350      return;
2351    }
2352
2353  seg_mgr_props->tx_fifo_size = qm->udp_fifo_size;
2354  seg_mgr_props->rx_fifo_size = qm->udp_fifo_size;
2355}
2356
2357static clib_error_t *
2358quic_init (vlib_main_t * vm)
2359{
2360  u32 segment_size = 256 << 20;
2361  vlib_thread_main_t *vtm = vlib_get_thread_main ();
2362  tw_timer_wheel_1t_3w_1024sl_ov_t *tw;
2363  vnet_app_attach_args_t _a, *a = &_a;
2364  u64 options[APP_OPTIONS_N_OPTIONS];
2365  quic_main_t *qm = &quic_main;
2366  u32 num_threads, i;
2367
2368  num_threads = 1 /* main thread */  + vtm->n_threads;
2369
2370  clib_memset (a, 0, sizeof (*a));
2371  clib_memset (options, 0, sizeof (options));
2372
2373  a->session_cb_vft = &quic_app_cb_vft;
2374  a->api_client_index = APP_INVALID_INDEX;
2375  a->options = options;
2376  a->name = format (0, "quic");
2377  a->options[APP_OPTIONS_SEGMENT_SIZE] = segment_size;
2378  a->options[APP_OPTIONS_ADD_SEGMENT_SIZE] = segment_size;
2379  a->options[APP_OPTIONS_RX_FIFO_SIZE] = qm->udp_fifo_size;
2380  a->options[APP_OPTIONS_TX_FIFO_SIZE] = qm->udp_fifo_size;
2381  a->options[APP_OPTIONS_PREALLOC_FIFO_PAIRS] = qm->udp_fifo_prealloc;
2382  a->options[APP_OPTIONS_FLAGS] = APP_OPTIONS_FLAGS_IS_BUILTIN;
2383  a->options[APP_OPTIONS_FLAGS] |= APP_OPTIONS_FLAGS_USE_GLOBAL_SCOPE;
2384  a->options[APP_OPTIONS_FLAGS] |= APP_OPTIONS_FLAGS_IS_TRANSPORT_APP;
2385
2386  if (vnet_application_attach (a))
2387    {
2388      clib_warning ("failed to attach quic app");
2389      return clib_error_return (0, "failed to attach quic app");
2390    }
2391
2392  vec_validate (qm->ctx_pool, num_threads - 1);
2393  vec_validate (qm->wrk_ctx, num_threads - 1);
2394  for (i = 0; i < num_threads; i++)
2395    {
2396      qm->wrk_ctx[i].next_cid.thread_id = i;
2397      tw = &qm->wrk_ctx[i].timer_wheel;
2398      tw_timer_wheel_init_1t_3w_1024sl_ov (tw, quic_expired_timers_dispatch,
2399					   1e-3 /* timer period 1ms */ , ~0);
2400      tw->last_run_time = vlib_time_now (vlib_get_main ());
2401      clib_bihash_init_24_8 (&qm->wrk_ctx[i].crypto_context_hash,
2402			     "quic crypto contexts", 64, 128 << 10);
2403    }
2404
2405  clib_bihash_init_16_8 (&qm->connection_hash, "quic connections", 1024,
2406			 4 << 20);
2407
2408  qm->app_index = a->app_index;
2409  qm->tstamp_ticks_per_clock = vm->clib_time.seconds_per_clock
2410    / QUIC_TSTAMP_RESOLUTION;
2411  qm->session_cache.super.cb = quic_encrypt_ticket_cb;
2412
2413  transport_register_protocol (TRANSPORT_PROTO_QUIC, &quic_proto,
2414			       FIB_PROTOCOL_IP4, ~0);
2415  transport_register_protocol (TRANSPORT_PROTO_QUIC, &quic_proto,
2416			       FIB_PROTOCOL_IP6, ~0);
2417
2418  clib_bitmap_alloc (qm->available_crypto_engines,
2419		     app_crypto_engine_n_types ());
2420  quic_register_cipher_suite (CRYPTO_ENGINE_VPP, quic_crypto_cipher_suites);
2421  quic_register_cipher_suite (CRYPTO_ENGINE_PICOTLS,
2422			      ptls_openssl_cipher_suites);
2423  qm->default_crypto_engine = CRYPTO_ENGINE_PICOTLS;
2424  vec_free (a->name);
2425  return 0;
2426}
2427
2428VLIB_INIT_FUNCTION (quic_init);
2429
2430static clib_error_t *
2431quic_plugin_crypto_command_fn (vlib_main_t * vm,
2432			       unformat_input_t * input,
2433			       vlib_cli_command_t * cmd)
2434{
2435  quic_main_t *qm = &quic_main;
2436  if (unformat_check_input (input) == UNFORMAT_END_OF_INPUT)
2437    return clib_error_return (0, "unknown input '%U'",
2438			      format_unformat_error, input);
2439  if (unformat (input, "vpp"))
2440    qm->default_crypto_engine = CRYPTO_ENGINE_VPP;
2441  else if (unformat (input, "picotls"))
2442    qm->default_crypto_engine = CRYPTO_ENGINE_PICOTLS;
2443  else
2444    return clib_error_return (0, "unknown input '%U'",
2445			      format_unformat_error, input);
2446  return 0;
2447}
2448
2449u64 quic_fifosize = 0;
2450static clib_error_t *
2451quic_plugin_set_fifo_size_command_fn (vlib_main_t * vm,
2452				      unformat_input_t * input,
2453				      vlib_cli_command_t * cmd)
2454{
2455  quic_main_t *qm = &quic_main;
2456  unformat_input_t _line_input, *line_input = &_line_input;
2457  uword tmp;
2458
2459  if (!unformat_user (input, unformat_line_input, line_input))
2460    return 0;
2461
2462  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
2463    {
2464      if (unformat (line_input, "%U", unformat_memory_size, &tmp))
2465	{
2466	  if (tmp >= 0x100000000ULL)
2467	    {
2468	      return clib_error_return
2469		(0, "fifo-size %llu (0x%llx) too large", tmp, tmp);
2470	    }
2471	  qm->udp_fifo_size = tmp;
2472	  quic_update_fifo_size ();
2473	}
2474      else
2475	return clib_error_return (0, "unknown input '%U'",
2476				  format_unformat_error, line_input);
2477    }
2478
2479  return 0;
2480}
2481
2482static inline u64
2483quic_get_counter_value (u32 event_code)
2484{
2485  vlib_node_t *n;
2486  vlib_main_t *vm;
2487  vlib_error_main_t *em;
2488
2489  u32 code, i;
2490  u64 c, sum = 0;
2491  int index = 0;
2492
2493  vm = vlib_get_main ();
2494  em = &vm->error_main;
2495  n = vlib_get_node (vm, quic_input_node.index);
2496  code = event_code;
2497  /* *INDENT-OFF* */
2498  foreach_vlib_main(({
2499    em = &this_vlib_main->error_main;
2500    i = n->error_heap_index + code;
2501    c = em->counters[i];
2502
2503    if (i < vec_len (em->counters_last_clear))
2504       c -= em->counters_last_clear[i];
2505    sum += c;
2506    index++;
2507  }));
2508  /* *INDENT-ON* */
2509  return sum;
2510}
2511
2512static void
2513quic_show_aggregated_stats (vlib_main_t * vm)
2514{
2515  u32 num_workers = vlib_num_workers ();
2516  quic_main_t *qm = &quic_main;
2517  quic_ctx_t *ctx = NULL;
2518  quicly_stats_t st, agg_stats;
2519  u32 i, nconn = 0, nstream = 0;
2520
2521  clib_memset (&agg_stats, 0, sizeof (agg_stats));
2522  for (i = 0; i < num_workers + 1; i++)
2523    {
2524      /* *INDENT-OFF* */
2525      pool_foreach (ctx, qm->ctx_pool[i],
2526      ({
2527	if (quic_ctx_is_conn (ctx) && ctx->conn)
2528	  {
2529	    quicly_get_stats (ctx->conn, &st);
2530	    agg_stats.rtt.smoothed += st.rtt.smoothed;
2531	    agg_stats.rtt.minimum += st.rtt.minimum;
2532	    agg_stats.rtt.variance += st.rtt.variance;
2533	    agg_stats.num_packets.received += st.num_packets.received;
2534	    agg_stats.num_packets.sent += st.num_packets.sent;
2535	    agg_stats.num_packets.lost += st.num_packets.lost;
2536	    agg_stats.num_packets.ack_received += st.num_packets.ack_received;
2537	    agg_stats.num_bytes.received += st.num_bytes.received;
2538	    agg_stats.num_bytes.sent += st.num_bytes.sent;
2539	    nconn++;
2540	  }
2541	else if (quic_ctx_is_stream (ctx))
2542	  nstream++;
2543      }));
2544      /* *INDENT-ON* */
2545    }
2546  vlib_cli_output (vm, "-------- Connections --------");
2547  vlib_cli_output (vm, "Current:         %u", nconn);
2548  vlib_cli_output (vm, "Opened:          %d",
2549		   quic_get_counter_value (QUIC_ERROR_OPENED_CONNECTION));
2550  vlib_cli_output (vm, "Closed:          %d",
2551		   quic_get_counter_value (QUIC_ERROR_CLOSED_CONNECTION));
2552  vlib_cli_output (vm, "---------- Streams ----------");
2553  vlib_cli_output (vm, "Current:         %u", nstream);
2554  vlib_cli_output (vm, "Opened:          %d",
2555		   quic_get_counter_value (QUIC_ERROR_OPENED_STREAM));
2556  vlib_cli_output (vm, "Closed:          %d",
2557		   quic_get_counter_value (QUIC_ERROR_CLOSED_STREAM));
2558  vlib_cli_output (vm, "---------- Packets ----------");
2559  vlib_cli_output (vm, "RX Total:        %d",
2560		   quic_get_counter_value (QUIC_ERROR_RX_PACKETS));
2561  vlib_cli_output (vm, "RX 0RTT:         %d",
2562		   quic_get_counter_value (QUIC_ERROR_ZERO_RTT_RX_PACKETS));
2563  vlib_cli_output (vm, "RX 1RTT:         %d",
2564		   quic_get_counter_value (QUIC_ERROR_ONE_RTT_RX_PACKETS));
2565  vlib_cli_output (vm, "TX Total:        %d",
2566		   quic_get_counter_value (QUIC_ERROR_TX_PACKETS));
2567  vlib_cli_output (vm, "----------- Stats -----------");
2568  vlib_cli_output (vm, "Min      RTT     %f",
2569		   nconn > 0 ? agg_stats.rtt.minimum / nconn : 0);
2570  vlib_cli_output (vm, "Smoothed RTT     %f",
2571		   nconn > 0 ? agg_stats.rtt.smoothed / nconn : 0);
2572  vlib_cli_output (vm, "Variance on RTT  %f",
2573		   nconn > 0 ? agg_stats.rtt.variance / nconn : 0);
2574  vlib_cli_output (vm, "Packets Received %lu",
2575		   agg_stats.num_packets.received);
2576  vlib_cli_output (vm, "Packets Sent     %lu", agg_stats.num_packets.sent);
2577  vlib_cli_output (vm, "Packets Lost     %lu", agg_stats.num_packets.lost);
2578  vlib_cli_output (vm, "Packets Acks     %lu",
2579		   agg_stats.num_packets.ack_received);
2580  vlib_cli_output (vm, "RX bytes         %lu", agg_stats.num_bytes.received);
2581  vlib_cli_output (vm, "TX bytes         %lu", agg_stats.num_bytes.sent);
2582}
2583
2584static u8 *
2585quic_format_quicly_conn_id (u8 * s, va_list * args)
2586{
2587  quicly_cid_plaintext_t *mid = va_arg (*args, quicly_cid_plaintext_t *);
2588  s = format (s, "C%x_%x", mid->master_id, mid->thread_id);
2589  return s;
2590}
2591
2592static u8 *
2593quic_format_quicly_stream_id (u8 * s, va_list * args)
2594{
2595  quicly_stream_t *stream = va_arg (*args, quicly_stream_t *);
2596  s =
2597    format (s, "%U S%lx", quic_format_quicly_conn_id,
2598	    quicly_get_master_id (stream->conn), stream->stream_id);
2599  return s;
2600}
2601
2602static u8 *
2603quic_format_listener_ctx (u8 * s, va_list * args)
2604{
2605  quic_ctx_t *ctx = va_arg (*args, quic_ctx_t *);
2606  s = format (s, "[#%d][%x][Listener]", ctx->c_thread_index, ctx->c_c_index);
2607  return s;
2608}
2609
2610static u8 *
2611quic_format_connection_ctx (u8 * s, va_list * args)
2612{
2613  quic_ctx_t *ctx = va_arg (*args, quic_ctx_t *);
2614  quicly_stats_t quicly_stats;
2615
2616  s = format (s, "[#%d][%x]", ctx->c_thread_index, ctx->c_c_index);
2617
2618  if (!ctx->conn)
2619    {
2620      s = format (s, "- no conn -\n");
2621      return s;
2622    }
2623  s = format (s, "[%U]",
2624	      quic_format_quicly_conn_id, quicly_get_master_id (ctx->conn));
2625  quicly_get_stats (ctx->conn, &quicly_stats);
2626
2627  s = format (s, "[RTT >%3d, ~%3d, V%3d, last %3d]",
2628	      quicly_stats.rtt.minimum, quicly_stats.rtt.smoothed,
2629	      quicly_stats.rtt.variance, quicly_stats.rtt.latest);
2630  s = format (s, " TX:%d RX:%d loss:%d ack:%d",
2631	      quicly_stats.num_packets.sent,
2632	      quicly_stats.num_packets.received,
2633	      quicly_stats.num_packets.lost,
2634	      quicly_stats.num_packets.ack_received);
2635  return s;
2636}
2637
2638static u8 *
2639quic_format_stream_ctx (u8 * s, va_list * args)
2640{
2641  quic_ctx_t *ctx = va_arg (*args, quic_ctx_t *);
2642  session_t *stream_session;
2643  quicly_stream_t *stream = ctx->stream;
2644  u32 txs, rxs;
2645
2646  s = format (s, "[#%d][%x]", ctx->c_thread_index, ctx->c_c_index);
2647  s = format (s, "[%U]", quic_format_quicly_stream_id, stream);
2648
2649  stream_session = session_get_if_valid (ctx->c_s_index, ctx->c_thread_index);
2650  if (!stream_session)
2651    {
2652      s = format (s, "- no session -\n");
2653      return s;
2654    }
2655  txs = svm_fifo_max_dequeue (stream_session->tx_fifo);
2656  rxs = svm_fifo_max_dequeue (stream_session->rx_fifo);
2657  s = format (s, "[rx %d tx %d]\n", rxs, txs);
2658  return s;
2659}
2660
2661static clib_error_t *
2662quic_show_connections_command_fn (vlib_main_t * vm,
2663				  unformat_input_t * input,
2664				  vlib_cli_command_t * cmd)
2665{
2666  unformat_input_t _line_input, *line_input = &_line_input;
2667  u8 show_listeners = 0, show_conn = 0, show_stream = 0;
2668  u32 num_workers = vlib_num_workers ();
2669  quic_main_t *qm = &quic_main;
2670  clib_error_t *error = 0;
2671  quic_ctx_t *ctx = NULL;
2672
2673  session_cli_return_if_not_enabled ();
2674
2675  if (!unformat_user (input, unformat_line_input, line_input))
2676    {
2677      quic_show_aggregated_stats (vm);
2678      return 0;
2679    }
2680
2681  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
2682    {
2683      if (unformat (line_input, "listener"))
2684	show_listeners = 1;
2685      else if (unformat (line_input, "conn"))
2686	show_conn = 1;
2687      else if (unformat (line_input, "stream"))
2688	show_stream = 1;
2689      else
2690	{
2691	  error = clib_error_return (0, "unknown input `%U'",
2692				     format_unformat_error, line_input);
2693	  goto done;
2694	}
2695    }
2696
2697  for (int i = 0; i < num_workers + 1; i++)
2698    {
2699      /* *INDENT-OFF* */
2700      pool_foreach (ctx, qm->ctx_pool[i],
2701      ({
2702        if (quic_ctx_is_stream (ctx) && show_stream)
2703          vlib_cli_output (vm, "%U", quic_format_stream_ctx, ctx);
2704        else if (quic_ctx_is_listener (ctx) && show_listeners)
2705          vlib_cli_output (vm, "%U", quic_format_listener_ctx, ctx);
2706	else if (quic_ctx_is_conn (ctx) && show_conn)
2707          vlib_cli_output (vm, "%U", quic_format_connection_ctx, ctx);
2708      }));
2709      /* *INDENT-ON* */
2710    }
2711
2712done:
2713  unformat_free (line_input);
2714  return error;
2715}
2716
2717/* *INDENT-OFF* */
2718VLIB_CLI_COMMAND (quic_plugin_crypto_command, static) =
2719{
2720  .path = "quic set crypto api",
2721  .short_help = "quic set crypto api [picotls, vpp]",
2722  .function = quic_plugin_crypto_command_fn,
2723};
2724VLIB_CLI_COMMAND(quic_plugin_set_fifo_size_command, static)=
2725{
2726  .path = "quic set fifo-size",
2727  .short_help = "quic set fifo-size N[K|M|G] (default 64K)",
2728  .function = quic_plugin_set_fifo_size_command_fn,
2729};
2730VLIB_CLI_COMMAND(quic_show_ctx_command, static)=
2731{
2732  .path = "show quic",
2733  .short_help = "show quic",
2734  .function = quic_show_connections_command_fn,
2735};
2736VLIB_CLI_COMMAND (quic_list_crypto_context_command, static) =
2737{
2738  .path = "show quic crypto context",
2739  .short_help = "list quic crypto contextes",
2740  .function = quic_list_crypto_context_command_fn,
2741};
2742VLIB_PLUGIN_REGISTER () =
2743{
2744  .version = VPP_BUILD_VER,
2745  .description = "Quic transport protocol",
2746  .default_disabled = 1,
2747};
2748/* *INDENT-ON* */
2749
2750static clib_error_t *
2751quic_config_fn (vlib_main_t * vm, unformat_input_t * input)
2752{
2753  quic_main_t *qm = &quic_main;
2754  uword tmp;
2755  u32 i;
2756
2757  qm->udp_fifo_size = QUIC_DEFAULT_FIFO_SIZE;
2758  qm->udp_fifo_prealloc = 0;
2759  qm->connection_timeout = QUIC_DEFAULT_CONN_TIMEOUT;
2760  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2761    {
2762      if (unformat (input, "fifo-size %U", unformat_memory_size, &tmp))
2763	{
2764	  if (tmp >= 0x100000000ULL)
2765	    {
2766	      return clib_error_return (0,
2767					"fifo-size %llu (0x%llx) too large",
2768					tmp, tmp);
2769	    }
2770	  qm->udp_fifo_size = tmp;
2771	}
2772      else if (unformat (input, "conn-timeout %u", &i))
2773	qm->connection_timeout = i;
2774      else if (unformat (input, "fifo-prealloc %u", &i))
2775	qm->udp_fifo_prealloc = i;
2776      else
2777	return clib_error_return (0, "unknown input '%U'",
2778				  format_unformat_error, input);
2779    }
2780
2781  return 0;
2782}
2783
2784VLIB_EARLY_CONFIG_FUNCTION (quic_config_fn, "quic");
2785
2786static uword
2787quic_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
2788	      vlib_frame_t * frame)
2789{
2790  return 0;
2791}
2792
2793/* *INDENT-OFF* */
2794VLIB_REGISTER_NODE (quic_input_node) =
2795{
2796  .function = quic_node_fn,
2797  .name = "quic-input",
2798  .vector_size = sizeof (u32),
2799  .type = VLIB_NODE_TYPE_INTERNAL,
2800  .n_errors = ARRAY_LEN (quic_error_strings),
2801  .error_strings = quic_error_strings,
2802};
2803/* *INDENT-ON* */
2804
2805/*
2806 * fd.io coding-style-patch-verification: ON
2807 *
2808 * Local Variables:
2809 * eval: (c-set-style "gnu")
2810 * End:
2811 */
2812