1fe750c24SBenoît Ganne/*
2fe750c24SBenoît Ganne *------------------------------------------------------------------
3fe750c24SBenoît Ganne * Copyright (c) 2018 Cisco and/or its affiliates.
4fe750c24SBenoît Ganne * Licensed under the Apache License, Version 2.0 (the "License");
5fe750c24SBenoît Ganne * you may not use this file except in compliance with the License.
6fe750c24SBenoît Ganne * You may obtain a copy of the License at:
7fe750c24SBenoît Ganne *
8fe750c24SBenoît Ganne *     http://www.apache.org/licenses/LICENSE-2.0
9fe750c24SBenoît Ganne *
10fe750c24SBenoît Ganne * Unless required by applicable law or agreed to in writing, software
11fe750c24SBenoît Ganne * distributed under the License is distributed on an "AS IS" BASIS,
12fe750c24SBenoît Ganne * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13fe750c24SBenoît Ganne * See the License for the specific language governing permissions and
14fe750c24SBenoît Ganne * limitations under the License.
15fe750c24SBenoît Ganne *------------------------------------------------------------------
16fe750c24SBenoît Ganne */
17fe750c24SBenoît Ganne
18fe750c24SBenoît Ganne#ifndef _RDMA_H_
19fe750c24SBenoît Ganne#define _RDMA_H_
20fe750c24SBenoît Ganne
21fe750c24SBenoît Ganne#include <infiniband/verbs.h>
22fe750c24SBenoît Ganne#include <vlib/log.h>
23812afe71SBenoît Ganne#include <vlib/pci/pci.h>
24812afe71SBenoît Ganne#include <vnet/interface.h>
25812afe71SBenoît Ganne#include <vnet/ethernet/mac_address.h>
26dd648aacSDamjan Marion#include <rdma/rdma_mlx5dv.h>
27fe750c24SBenoît Ganne
28fe750c24SBenoît Ganne#define foreach_rdma_device_flags \
29f2d5cdbfSBenoît Ganne  _(0, ERROR, "error") \
30f2d5cdbfSBenoît Ganne  _(1, ADMIN_UP, "admin-up") \
31f2d5cdbfSBenoît Ganne  _(2, LINK_UP, "link-up") \
32dd648aacSDamjan Marion  _(3, PROMISC, "promiscuous") \
33dd648aacSDamjan Marion  _(4, MLX5DV, "mlx5dv")
34fe750c24SBenoît Ganne
35fe750c24SBenoît Ganneenum
36fe750c24SBenoît Ganne{
37fe750c24SBenoît Ganne#define _(a, b, c) RDMA_DEVICE_F_##b = (1 << a),
38fe750c24SBenoît Ganne  foreach_rdma_device_flags
39fe750c24SBenoît Ganne#undef _
40fe750c24SBenoît Ganne};
41fe750c24SBenoît Ganne
42aaa65a12SDamjan Marion#ifndef MLX5_ETH_L2_INLINE_HEADER_SIZE
43aaa65a12SDamjan Marion#define MLX5_ETH_L2_INLINE_HEADER_SIZE  18
44aaa65a12SDamjan Marion#endif
45aaa65a12SDamjan Marion
46dc812d9aSBenoît Gannetypedef struct
47dc812d9aSBenoît Ganne{
48dc812d9aSBenoît Ganne  CLIB_ALIGN_MARK (align0, MLX5_SEND_WQE_BB);
49aaa65a12SDamjan Marion  union
50aaa65a12SDamjan Marion  {
51aaa65a12SDamjan Marion    struct mlx5_wqe_ctrl_seg ctrl;
52aaa65a12SDamjan Marion    struct
53aaa65a12SDamjan Marion    {
54aaa65a12SDamjan Marion      u8 opc_mod;
55aaa65a12SDamjan Marion      u8 wqe_index_hi;
56aaa65a12SDamjan Marion      u8 wqe_index_lo;
57aaa65a12SDamjan Marion      u8 opcode;
58aaa65a12SDamjan Marion    };
59aaa65a12SDamjan Marion  };
60dc812d9aSBenoît Ganne  struct mlx5_wqe_eth_seg eseg;
61dc812d9aSBenoît Ganne  struct mlx5_wqe_data_seg dseg;
62dc812d9aSBenoît Ganne} rdma_mlx5_wqe_t;
63dc812d9aSBenoît Ganne#define RDMA_MLX5_WQE_SZ        sizeof(rdma_mlx5_wqe_t)
64dc812d9aSBenoît Ganne#define RDMA_MLX5_WQE_DS        (RDMA_MLX5_WQE_SZ/sizeof(struct mlx5_wqe_data_seg))
66dc812d9aSBenoît Ganne	       RDMA_MLX5_WQE_SZ % sizeof (struct mlx5_wqe_data_seg) == 0,
67dc812d9aSBenoît Ganne	       "bad size");
68dc812d9aSBenoît Ganne
69fe750c24SBenoît Gannetypedef struct
70fe750c24SBenoît Ganne{
71fe750c24SBenoît Ganne  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
72fe750c24SBenoît Ganne  struct ibv_cq *cq;
735763e47bSBenoît Ganne  struct ibv_wq *wq;
74e7e8bf37SBenoît Ganne  u32 *bufs;
75e7e8bf37SBenoît Ganne  u32 size;
76e7e8bf37SBenoît Ganne  u32 head;
77e7e8bf37SBenoît Ganne  u32 tail;
78dd648aacSDamjan Marion  u32 cq_ci;
79dd648aacSDamjan Marion  u16 log2_cq_size;
80dd648aacSDamjan Marion  u16 n_mini_cqes;
81dd648aacSDamjan Marion  u16 n_mini_cqes_left;
82dd648aacSDamjan Marion  u16 last_cqe_flags;
83dd648aacSDamjan Marion  mlx5dv_cqe_t *cqes;
84dd648aacSDamjan Marion  mlx5dv_rwq_t *wqes;
85dd648aacSDamjan Marion  volatile u32 *wq_db;
86dd648aacSDamjan Marion  volatile u32 *cq_db;
87dd648aacSDamjan Marion  u32 cqn;
88dd648aacSDamjan Marion  u32 wqe_cnt;
89dd648aacSDamjan Marion  u32 wq_stride;
90fe750c24SBenoît Ganne} rdma_rxq_t;
91fe750c24SBenoît Ganne
92fe750c24SBenoît Gannetypedef struct
93fe750c24SBenoît Ganne{
94fe750c24SBenoît Ganne  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
95dc812d9aSBenoît Ganne
96dc812d9aSBenoît Ganne  /* following fields are accessed in datapath */
97e7e8bf37SBenoît Ganne  clib_spinlock_t lock;
98dc812d9aSBenoît Ganne
99dc812d9aSBenoît Ganne  union
100dc812d9aSBenoît Ganne  {
101dc812d9aSBenoît Ganne    struct
102dc812d9aSBenoît Ganne    {
103dc812d9aSBenoît Ganne      /* ibverb datapath. Cache of cq, sq below */
104dc812d9aSBenoît Ganne      struct ibv_cq *ibv_cq;
105dc812d9aSBenoît Ganne      struct ibv_qp *ibv_qp;
106dc812d9aSBenoît Ganne    };
107dc812d9aSBenoît Ganne    struct
108dc812d9aSBenoît Ganne    {
109dc812d9aSBenoît Ganne      /* direct verbs datapath */
110dc812d9aSBenoît Ganne      rdma_mlx5_wqe_t *dv_sq_wqes;
111dc812d9aSBenoît Ganne      volatile u32 *dv_sq_dbrec;
112dc812d9aSBenoît Ganne      volatile u64 *dv_sq_db;
113dc812d9aSBenoît Ganne      struct mlx5_cqe64 *dv_cq_cqes;
114dc812d9aSBenoît Ganne      volatile u32 *dv_cq_dbrec;
115dc812d9aSBenoît Ganne    };
116dc812d9aSBenoît Ganne  };
117dc812d9aSBenoît Ganne
118dc812d9aSBenoît Ganne  u32 *bufs;			/* vlib_buffer ring buffer */
119dc812d9aSBenoît Ganne  u16 head;
120dc812d9aSBenoît Ganne  u16 tail;
121dc812d9aSBenoît Ganne  u16 dv_cq_idx;		/* monotonic CQE index (valid only for direct verbs) */
122dc812d9aSBenoît Ganne  u8 bufs_log2sz;		/* log2 vlib_buffer entries */
123dc812d9aSBenoît Ganne  u8 dv_sq_log2sz:4;		/* log2 SQ WQE entries (valid only for direct verbs) */
124dc812d9aSBenoît Ganne  u8 dv_cq_log2sz:4;		/* log2 CQ CQE entries (valid only for direct verbs) */
125dc812d9aSBenoît Ganne    STRUCT_MARK (cacheline1);
126dc812d9aSBenoît Ganne
127dc812d9aSBenoît Ganne  /* WQE template (valid only for direct verbs) */
128dc812d9aSBenoît Ganne  u8 dv_wqe_tmpl[64];
129dc812d9aSBenoît Ganne
130dc812d9aSBenoît Ganne  /* end of 2nd 64-bytes cacheline (or 1st 128-bytes cacheline) */
131dc812d9aSBenoît Ganne    STRUCT_MARK (cacheline2);
132dc812d9aSBenoît Ganne
133dc812d9aSBenoît Ganne  /* fields below are not accessed in datapath */
134fe750c24SBenoît Ganne  struct ibv_cq *cq;
135fe750c24SBenoît Ganne  struct ibv_qp *qp;
136dc812d9aSBenoît Ganne
137fe750c24SBenoît Ganne} rdma_txq_t;
138dc812d9aSBenoît GanneSTATIC_ASSERT_OFFSET_OF (rdma_txq_t, cacheline1, 64);
139dc812d9aSBenoît GanneSTATIC_ASSERT_OFFSET_OF (rdma_txq_t, cacheline2, 128);
140dc812d9aSBenoît Ganne
141dc812d9aSBenoît Ganne#define RDMA_TXQ_DV_INVALID_ID  0xffffffff
142dc812d9aSBenoît Ganne
143dc812d9aSBenoît Ganne#define RDMA_TXQ_BUF_SZ(txq)    (1U << (txq)->bufs_log2sz)
144dc812d9aSBenoît Ganne#define RDMA_TXQ_DV_SQ_SZ(txq)  (1U << (txq)->dv_sq_log2sz)
145dc812d9aSBenoît Ganne#define RDMA_TXQ_DV_CQ_SZ(txq)  (1U << (txq)->dv_cq_log2sz)
146dc812d9aSBenoît Ganne
147dc812d9aSBenoît Ganne#define RDMA_TXQ_USED_SZ(head, tail)            ((u16)((u16)(tail) - (u16)(head)))
148dc812d9aSBenoît Ganne#define RDMA_TXQ_AVAIL_SZ(txq, head, tail)      ((u16)(RDMA_TXQ_BUF_SZ (txq) - RDMA_TXQ_USED_SZ (head, tail)))
149fe750c24SBenoît Ganne
150fe750c24SBenoît Gannetypedef struct
151fe750c24SBenoît Ganne{
152fe750c24SBenoît Ganne  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
153e7e8bf37SBenoît Ganne
154e7e8bf37SBenoît Ganne  /* following fields are accessed in datapath */
155e7e8bf37SBenoît Ganne  rdma_rxq_t *rxqs;
156e7e8bf37SBenoît Ganne  rdma_txq_t *txqs;
157fe750c24SBenoît Ganne  u32 flags;
158fe750c24SBenoît Ganne  u32 per_interface_next_index;
159fe750c24SBenoît Ganne  u32 sw_if_index;
160fe750c24SBenoît Ganne  u32 hw_if_index;
161e7e8bf37SBenoît Ganne  u32 lkey;			/* cache of mr->lkey */
162e7e8bf37SBenoît Ganne  u8 pool;			/* buffer pool index */
163fe750c24SBenoît Ganne
164e7e8bf37SBenoît Ganne  /* fields below are not accessed in datapath */
165e7e8bf37SBenoît Ganne  vlib_pci_device_info_t *pci;
16696d4e533SDamjan Marion  u8 *name;
167e7e8bf37SBenoît Ganne  u8 *linux_ifname;
1685763e47bSBenoît Ganne  mac_address_t hwaddr;
169e7e8bf37SBenoît Ganne  u32 async_event_clib_file_index;
170e7e8bf37SBenoît Ganne  u32 dev_instance;
171fe750c24SBenoît Ganne
172fe750c24SBenoît Ganne  struct ibv_context *ctx;
173fe750c24SBenoît Ganne  struct ibv_pd *pd;
174fe750c24SBenoît Ganne  struct ibv_mr *mr;
1755763e47bSBenoît Ganne  struct ibv_qp *rx_qp;
1765763e47bSBenoît Ganne  struct ibv_rwq_ind_table *rx_rwq_ind_tbl;
177fe750c24SBenoît Ganne  struct ibv_flow *flow_ucast;
178fe750c24SBenoît Ganne  struct ibv_flow *flow_mcast;
179fe750c24SBenoît Ganne
180fe750c24SBenoît Ganne  clib_error_t *error;
181fe750c24SBenoît Ganne} rdma_device_t;
182fe750c24SBenoît Ganne
183fe750c24SBenoît Gannetypedef struct
184fe750c24SBenoît Ganne{
185a3188febSDamjan Marion  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
186dd648aacSDamjan Marion  union
187dd648aacSDamjan Marion  {
188dd648aacSDamjan Marion    u16 cqe_flags[VLIB_FRAME_SIZE];
189dd648aacSDamjan Marion    u16x8 cqe_flags8[VLIB_FRAME_SIZE / 8];
190dd648aacSDamjan Marion    u16x16 cqe_flags16[VLIB_FRAME_SIZE / 16];
191dd648aacSDamjan Marion  };
192a3188febSDamjan Marion  vlib_buffer_t buffer_template;
193a3188febSDamjan Marion} rdma_per_thread_data_t;
194a3188febSDamjan Marion
195a3188febSDamjan Mariontypedef struct
196a3188febSDamjan Marion{
197a3188febSDamjan Marion  rdma_per_thread_data_t *per_thread_data;
198fe750c24SBenoît Ganne  rdma_device_t *devices;
199fe750c24SBenoît Ganne  vlib_log_class_t log_class;
200812afe71SBenoît Ganne  u16 msg_id_base;
201fe750c24SBenoît Ganne} rdma_main_t;
202fe750c24SBenoît Ganne
203fe750c24SBenoît Ganneextern rdma_main_t rdma_main;
204fe750c24SBenoît Ganne
205d8c1ef92SBenoît Gannetypedef enum
206d8c1ef92SBenoît Ganne{
207d8c1ef92SBenoît Ganne  RDMA_MODE_AUTO = 0,
208d8c1ef92SBenoît Ganne  RDMA_MODE_IBV,
209d8c1ef92SBenoît Ganne  RDMA_MODE_DV,
210d8c1ef92SBenoît Ganne} rdma_mode_t;
211d8c1ef92SBenoît Ganne
212fe750c24SBenoît Gannetypedef struct
213fe750c24SBenoît Ganne{
214fe750c24SBenoît Ganne  u8 *ifname;
21596d4e533SDamjan Marion  u8 *name;
2165763e47bSBenoît Ganne  u32 rxq_size;
2175763e47bSBenoît Ganne  u32 txq_size;
2185763e47bSBenoît Ganne  u32 rxq_num;
219d8c1ef92SBenoît Ganne  rdma_mode_t mode;
220fe750c24SBenoît Ganne
221fe750c24SBenoît Ganne  /* return */
222fe750c24SBenoît Ganne  int rv;
223fe750c24SBenoît Ganne  u32 sw_if_index;
224fe750c24SBenoît Ganne  clib_error_t *error;
225fe750c24SBenoît Ganne} rdma_create_if_args_t;
226fe750c24SBenoît Ganne
227fe750c24SBenoît Gannevoid rdma_create_if (vlib_main_t * vm, rdma_create_if_args_t * args);
228fe750c24SBenoît Gannevoid rdma_delete_if (vlib_main_t * vm, rdma_device_t * rd);
229fe750c24SBenoît Ganne
230fe750c24SBenoît Ganneextern vlib_node_registration_t rdma_input_node;
231fe750c24SBenoît Ganneextern vnet_device_class_t rdma_device_class;
232fe750c24SBenoît Ganne
233fe750c24SBenoît Ganneformat_function_t format_rdma_device;
234fe750c24SBenoît Ganneformat_function_t format_rdma_device_name;
235fe750c24SBenoît Ganneformat_function_t format_rdma_input_trace;
236dd648aacSDamjan Marionformat_function_t format_rdma_rxq;
237812afe71SBenoît Ganneunformat_function_t unformat_rdma_create_if_args;
238fe750c24SBenoît Ganne
239fe750c24SBenoît Gannetypedef struct
240fe750c24SBenoît Ganne{
241fe750c24SBenoît Ganne  u32 next_index;
242fe750c24SBenoît Ganne  u32 hw_if_index;
243dd648aacSDamjan Marion  u16 cqe_flags;
244fe750c24SBenoît Ganne} rdma_input_trace_t;
245fe750c24SBenoît Ganne
246dc812d9aSBenoît Ganne#define foreach_rdma_tx_func_error \
247dc812d9aSBenoît Ganne_(SEGMENT_SIZE_EXCEEDED, "segment size exceeded") \
248dc812d9aSBenoît Ganne_(NO_FREE_SLOTS, "no free tx slots") \
249dc812d9aSBenoît Ganne_(SUBMISSION, "tx submission errors") \
250dc812d9aSBenoît Ganne_(COMPLETION, "tx completion errors")
251fe750c24SBenoît Ganne
252fe750c24SBenoît Gannetypedef enum
253fe750c24SBenoît Ganne{
254fe750c24SBenoît Ganne#define _(f,s) RDMA_TX_ERROR_##f,
255fe750c24SBenoît Ganne  foreach_rdma_tx_func_error
256fe750c24SBenoît Ganne#undef _
257fe750c24SBenoît Ganne    RDMA_TX_N_ERROR,
258fe750c24SBenoît Ganne} rdma_tx_func_error_t;
259fe750c24SBenoît Ganne
260dc812d9aSBenoît Ganne#endif /* _RDMA_H_ */
261fe750c24SBenoît Ganne
262fe750c24SBenoît Ganne/*
263fe750c24SBenoît Ganne * fd.io coding-style-patch-verification: ON
264fe750c24SBenoît Ganne *
265fe750c24SBenoît Ganne * Local Variables:
266fe750c24SBenoît Ganne * eval: (c-set-style "gnu")
267fe750c24SBenoît Ganne * End:
268fe750c24SBenoît Ganne */