1/*
2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include <vnet/vnet.h>
17#include <vnet/devices/devices.h>
18#include <vnet/feature/feature.h>
19#include <vnet/ip/ip.h>
20#include <vnet/ethernet/ethernet.h>
21
22vnet_device_main_t vnet_device_main;
23
24static uword
25device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
26		 vlib_frame_t * frame)
27{
28  return 0;
29}
30
31/* *INDENT-OFF* */
32VLIB_REGISTER_NODE (device_input_node) = {
33  .function = device_input_fn,
34  .name = "device-input",
35  .runtime_data_bytes = sizeof (vnet_device_input_runtime_t),
36  .type = VLIB_NODE_TYPE_INPUT,
37  .state = VLIB_NODE_STATE_DISABLED,
38  .n_next_nodes = VNET_DEVICE_INPUT_N_NEXT_NODES,
39  .next_nodes = VNET_DEVICE_INPUT_NEXT_NODES,
40};
41
42/* Table defines how much we need to advance current data pointer
43   in the buffer if we shortcut to l3 nodes */
44
45const u32 __attribute__((aligned (CLIB_CACHE_LINE_BYTES)))
46device_input_next_node_advance[((VNET_DEVICE_INPUT_N_NEXT_NODES /
47				CLIB_CACHE_LINE_BYTES) +1) * CLIB_CACHE_LINE_BYTES] =
48{
49      [VNET_DEVICE_INPUT_NEXT_IP4_INPUT] = sizeof (ethernet_header_t),
50      [VNET_DEVICE_INPUT_NEXT_IP4_NCS_INPUT] = sizeof (ethernet_header_t),
51      [VNET_DEVICE_INPUT_NEXT_IP6_INPUT] = sizeof (ethernet_header_t),
52      [VNET_DEVICE_INPUT_NEXT_MPLS_INPUT] = sizeof (ethernet_header_t),
53};
54
55const u32 __attribute__((aligned (CLIB_CACHE_LINE_BYTES)))
56device_input_next_node_flags[((VNET_DEVICE_INPUT_N_NEXT_NODES /
57				CLIB_CACHE_LINE_BYTES) +1) * CLIB_CACHE_LINE_BYTES] =
58{
59      [VNET_DEVICE_INPUT_NEXT_IP4_INPUT] = VNET_BUFFER_F_L3_HDR_OFFSET_VALID,
60      [VNET_DEVICE_INPUT_NEXT_IP4_NCS_INPUT] = VNET_BUFFER_F_L3_HDR_OFFSET_VALID,
61      [VNET_DEVICE_INPUT_NEXT_IP6_INPUT] = VNET_BUFFER_F_L3_HDR_OFFSET_VALID,
62      [VNET_DEVICE_INPUT_NEXT_MPLS_INPUT] = VNET_BUFFER_F_L3_HDR_OFFSET_VALID,
63};
64
65VNET_FEATURE_ARC_INIT (device_input, static) =
66{
67  .arc_name  = "device-input",
68  .start_nodes = VNET_FEATURES ("device-input"),
69  .last_in_arc = "ethernet-input",
70  .arc_index_ptr = &feature_main.device_input_feature_arc_index,
71};
72
73VNET_FEATURE_INIT (l2_patch, static) = {
74  .arc_name = "device-input",
75  .node_name = "l2-patch",
76  .runs_before = VNET_FEATURES ("ethernet-input"),
77};
78
79VNET_FEATURE_INIT (worker_handoff, static) = {
80  .arc_name = "device-input",
81  .node_name = "worker-handoff",
82  .runs_before = VNET_FEATURES ("ethernet-input"),
83};
84
85VNET_FEATURE_INIT (span_input, static) = {
86  .arc_name = "device-input",
87  .node_name = "span-input",
88  .runs_before = VNET_FEATURES ("ethernet-input"),
89};
90
91VNET_FEATURE_INIT (p2p_ethernet_node, static) = {
92  .arc_name = "device-input",
93  .node_name = "p2p-ethernet-input",
94  .runs_before = VNET_FEATURES ("ethernet-input"),
95};
96
97VNET_FEATURE_INIT (ethernet_input, static) = {
98  .arc_name = "device-input",
99  .node_name = "ethernet-input",
100  .runs_before = 0, /* not before any other features */
101};
102/* *INDENT-ON* */
103
104static int
105vnet_device_queue_sort (void *a1, void *a2)
106{
107  vnet_device_and_queue_t *dq1 = a1;
108  vnet_device_and_queue_t *dq2 = a2;
109
110  if (dq1->dev_instance > dq2->dev_instance)
111    return 1;
112  else if (dq1->dev_instance < dq2->dev_instance)
113    return -1;
114  else if (dq1->queue_id > dq2->queue_id)
115    return 1;
116  else if (dq1->queue_id < dq2->queue_id)
117    return -1;
118  else
119    return 0;
120}
121
122static void
123vnet_device_queue_update (vnet_main_t * vnm, vnet_device_input_runtime_t * rt)
124{
125  vnet_device_and_queue_t *dq;
126  vnet_hw_interface_t *hw;
127
128  vec_sort_with_function (rt->devices_and_queues, vnet_device_queue_sort);
129
130  vec_foreach (dq, rt->devices_and_queues)
131  {
132    hw = vnet_get_hw_interface (vnm, dq->hw_if_index);
133    vec_validate (hw->dq_runtime_index_by_queue, dq->queue_id);
134    hw->dq_runtime_index_by_queue[dq->queue_id] = dq - rt->devices_and_queues;
135  }
136}
137
138void
139vnet_hw_interface_assign_rx_thread (vnet_main_t * vnm, u32 hw_if_index,
140				    u16 queue_id, uword thread_index)
141{
142  vnet_device_main_t *vdm = &vnet_device_main;
143  vlib_main_t *vm, *vm0;
144  vnet_device_input_runtime_t *rt;
145  vnet_device_and_queue_t *dq;
146  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
147
148  ASSERT (hw->input_node_index > 0);
149
150  if (vdm->first_worker_thread_index == 0)
151    thread_index = 0;
152
153  if (thread_index != 0 &&
154      (thread_index < vdm->first_worker_thread_index ||
155       thread_index > vdm->last_worker_thread_index))
156    {
157      thread_index = vdm->next_worker_thread_index++;
158      if (vdm->next_worker_thread_index > vdm->last_worker_thread_index)
159	vdm->next_worker_thread_index = vdm->first_worker_thread_index;
160    }
161
162  vm = vlib_mains[thread_index];
163  vm0 = vlib_get_main ();
164
165  vlib_worker_thread_barrier_sync (vm0);
166
167  rt = vlib_node_get_runtime_data (vm, hw->input_node_index);
168
169  vec_add2 (rt->devices_and_queues, dq, 1);
170  dq->hw_if_index = hw_if_index;
171  dq->dev_instance = hw->dev_instance;
172  dq->queue_id = queue_id;
173  dq->mode = VNET_HW_INTERFACE_RX_MODE_POLLING;
174  rt->enabled_node_state = VLIB_NODE_STATE_POLLING;
175
176  vnet_device_queue_update (vnm, rt);
177  vec_validate (hw->input_node_thread_index_by_queue, queue_id);
178  vec_validate (hw->rx_mode_by_queue, queue_id);
179  hw->input_node_thread_index_by_queue[queue_id] = thread_index;
180  hw->rx_mode_by_queue[queue_id] = VNET_HW_INTERFACE_RX_MODE_POLLING;
181
182  vlib_worker_thread_barrier_release (vm0);
183
184  vlib_node_set_state (vm, hw->input_node_index, rt->enabled_node_state);
185}
186
187int
188vnet_hw_interface_unassign_rx_thread (vnet_main_t * vnm, u32 hw_if_index,
189				      u16 queue_id)
190{
191  vlib_main_t *vm, *vm0;
192  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
193  vnet_device_input_runtime_t *rt;
194  vnet_device_and_queue_t *dq;
195  uword old_thread_index;
196  vnet_hw_interface_rx_mode mode;
197
198  if (hw->input_node_thread_index_by_queue == 0)
199    return VNET_API_ERROR_INVALID_INTERFACE;
200
201  if (vec_len (hw->input_node_thread_index_by_queue) < queue_id + 1)
202    return VNET_API_ERROR_INVALID_INTERFACE;
203
204  old_thread_index = hw->input_node_thread_index_by_queue[queue_id];
205
206  vm = vlib_mains[old_thread_index];
207
208  rt = vlib_node_get_runtime_data (vm, hw->input_node_index);
209
210  vec_foreach (dq, rt->devices_and_queues)
211    if (dq->hw_if_index == hw_if_index && dq->queue_id == queue_id)
212    {
213      mode = dq->mode;
214      goto delete;
215    }
216
217  return VNET_API_ERROR_INVALID_INTERFACE;
218
219delete:
220
221  vm0 = vlib_get_main ();
222  vlib_worker_thread_barrier_sync (vm0);
223  vec_del1 (rt->devices_and_queues, dq - rt->devices_and_queues);
224  vnet_device_queue_update (vnm, rt);
225  hw->rx_mode_by_queue[queue_id] = VNET_HW_INTERFACE_RX_MODE_UNKNOWN;
226  vlib_worker_thread_barrier_release (vm0);
227
228  if (vec_len (rt->devices_and_queues) == 0)
229    vlib_node_set_state (vm, hw->input_node_index, VLIB_NODE_STATE_DISABLED);
230  else if (mode == VNET_HW_INTERFACE_RX_MODE_POLLING)
231    {
232      /*
233       * if the deleted interface is polling, we may need to set the node state
234       * to interrupt if there is no more polling interface for this device's
235       * corresponding thread. This is because mixed interfaces
236       * (polling and interrupt), assigned to the same thread, set the
237       * thread to polling prior to the deletion.
238       */
239      vec_foreach (dq, rt->devices_and_queues)
240      {
241	if (dq->mode == VNET_HW_INTERFACE_RX_MODE_POLLING)
242	  return 0;
243      }
244      rt->enabled_node_state = VLIB_NODE_STATE_INTERRUPT;
245      vlib_node_set_state (vm, hw->input_node_index, rt->enabled_node_state);
246    }
247
248  return 0;
249}
250
251
252int
253vnet_hw_interface_set_rx_mode (vnet_main_t * vnm, u32 hw_if_index,
254			       u16 queue_id, vnet_hw_interface_rx_mode mode)
255{
256  vlib_main_t *vm;
257  uword thread_index;
258  vnet_device_and_queue_t *dq;
259  vlib_node_state_t enabled_node_state;
260  ASSERT (mode < VNET_HW_INTERFACE_NUM_RX_MODES);
261  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
262  vnet_device_input_runtime_t *rt;
263  int is_polling = 0;
264
265  if (mode == VNET_HW_INTERFACE_RX_MODE_DEFAULT)
266    mode = hw->default_rx_mode;
267
268  if (hw->input_node_thread_index_by_queue == 0 || hw->rx_mode_by_queue == 0)
269    return VNET_API_ERROR_INVALID_INTERFACE;
270
271  if (hw->rx_mode_by_queue[queue_id] == mode)
272    return 0;
273
274  if (mode != VNET_HW_INTERFACE_RX_MODE_POLLING &&
275      (hw->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE) == 0)
276    return VNET_API_ERROR_UNSUPPORTED;
277
278  if ((vec_len (hw->input_node_thread_index_by_queue) < queue_id + 1) ||
279      (vec_len (hw->rx_mode_by_queue) < queue_id + 1))
280    return VNET_API_ERROR_INVALID_QUEUE;
281
282  hw->rx_mode_by_queue[queue_id] = mode;
283  thread_index = hw->input_node_thread_index_by_queue[queue_id];
284  vm = vlib_mains[thread_index];
285
286  rt = vlib_node_get_runtime_data (vm, hw->input_node_index);
287
288  vec_foreach (dq, rt->devices_and_queues)
289  {
290    if (dq->hw_if_index == hw_if_index && dq->queue_id == queue_id)
291      dq->mode = mode;
292    if (dq->mode == VNET_HW_INTERFACE_RX_MODE_POLLING)
293      is_polling = 1;
294  }
295
296  if (is_polling)
297    enabled_node_state = VLIB_NODE_STATE_POLLING;
298  else
299    enabled_node_state = VLIB_NODE_STATE_INTERRUPT;
300
301  if (rt->enabled_node_state != enabled_node_state)
302    {
303      rt->enabled_node_state = enabled_node_state;
304      if (vlib_node_get_state (vm, hw->input_node_index) !=
305	  VLIB_NODE_STATE_DISABLED)
306	vlib_node_set_state (vm, hw->input_node_index, enabled_node_state);
307    }
308
309  return 0;
310}
311
312int
313vnet_hw_interface_get_rx_mode (vnet_main_t * vnm, u32 hw_if_index,
314			       u16 queue_id, vnet_hw_interface_rx_mode * mode)
315{
316  vlib_main_t *vm;
317  uword thread_index;
318  vnet_device_and_queue_t *dq;
319  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
320  vnet_device_input_runtime_t *rt;
321
322  if (hw->input_node_thread_index_by_queue == 0)
323    return VNET_API_ERROR_INVALID_INTERFACE;
324
325  if ((vec_len (hw->input_node_thread_index_by_queue) < queue_id + 1) ||
326      (vec_len (hw->rx_mode_by_queue) < queue_id + 1))
327    return VNET_API_ERROR_INVALID_QUEUE;
328
329  thread_index = hw->input_node_thread_index_by_queue[queue_id];
330  vm = vlib_mains[thread_index];
331
332  rt = vlib_node_get_runtime_data (vm, hw->input_node_index);
333
334  vec_foreach (dq, rt->devices_and_queues)
335    if (dq->hw_if_index == hw_if_index && dq->queue_id == queue_id)
336    {
337      *mode = dq->mode;
338      return 0;
339    }
340
341  return VNET_API_ERROR_INVALID_INTERFACE;
342}
343
344
345
346static clib_error_t *
347vnet_device_init (vlib_main_t * vm)
348{
349  vnet_device_main_t *vdm = &vnet_device_main;
350  vlib_thread_main_t *tm = vlib_get_thread_main ();
351  vlib_thread_registration_t *tr;
352  uword *p;
353
354  vec_validate_aligned (vdm->workers, tm->n_vlib_mains - 1,
355			CLIB_CACHE_LINE_BYTES);
356
357  p = hash_get_mem (tm->thread_registrations_by_name, "workers");
358  tr = p ? (vlib_thread_registration_t *) p[0] : 0;
359  if (tr && tr->count > 0)
360    {
361      vdm->first_worker_thread_index = tr->first_index;
362      vdm->next_worker_thread_index = tr->first_index;
363      vdm->last_worker_thread_index = tr->first_index + tr->count - 1;
364    }
365  return 0;
366}
367
368VLIB_INIT_FUNCTION (vnet_device_init);
369
370/*
371 * fd.io coding-style-patch-verification: ON
372 *
373 * Local Variables:
374 * eval: (c-set-style "gnu")
375 * End:
376 */
377