output.c revision 2985e0af
1/*
2 *------------------------------------------------------------------
3 * Copyright (c) 2018 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
16 */
17
18#include <vlib/vlib.h>
19#include <vlib/unix/unix.h>
20#include <vlib/pci/pci.h>
21#include <vnet/ethernet/ethernet.h>
22#include <vnet/devices/devices.h>
23
24#include <vmxnet3/vmxnet3.h>
25
26static_always_inline void
27vmxnet3_tx_comp_ring_advance_next (vmxnet3_txq_t * txq)
28{
29  vmxnet3_tx_comp_ring *comp_ring = &txq->tx_comp_ring;
30
31  comp_ring->next++;
32  if (PREDICT_FALSE (comp_ring->next == txq->size))
33    {
34      comp_ring->next = 0;
35      comp_ring->gen ^= VMXNET3_TXCF_GEN;
36    }
37}
38
39static_always_inline void
40vmxnet3_tx_ring_advance_produce (vmxnet3_txq_t * txq)
41{
42  txq->tx_ring.produce++;
43  if (PREDICT_FALSE (txq->tx_ring.produce == txq->size))
44    {
45      txq->tx_ring.produce = 0;
46      txq->tx_ring.gen ^= VMXNET3_TXF_GEN;
47    }
48}
49
50static_always_inline void
51vmxnet3_tx_ring_advance_consume (vmxnet3_txq_t * txq)
52{
53  txq->tx_ring.consume++;
54  txq->tx_ring.consume &= txq->size - 1;
55}
56
57static_always_inline void
58vmxnet3_txq_release (vlib_main_t * vm, vmxnet3_device_t * vd,
59		     vmxnet3_txq_t * txq)
60{
61  vmxnet3_tx_comp *tx_comp;
62  vmxnet3_tx_comp_ring *comp_ring;
63
64  comp_ring = &txq->tx_comp_ring;
65  tx_comp = &txq->tx_comp[comp_ring->next];
66
67  while ((tx_comp->flags & VMXNET3_TXCF_GEN) == comp_ring->gen)
68    {
69      u16 eop_idx = tx_comp->index & VMXNET3_TXC_INDEX;
70      u32 bi0 = txq->tx_ring.bufs[txq->tx_ring.consume];
71
72      vlib_buffer_free_one (vm, bi0);
73      while (txq->tx_ring.consume != eop_idx)
74	{
75	  vmxnet3_tx_ring_advance_consume (txq);
76	}
77      vmxnet3_tx_ring_advance_consume (txq);
78
79      vmxnet3_tx_comp_ring_advance_next (txq);
80      tx_comp = &txq->tx_comp[comp_ring->next];
81    }
82}
83
84static_always_inline u16
85vmxnet3_tx_ring_space_left (vmxnet3_txq_t * txq)
86{
87  u16 count;
88
89  count = (txq->tx_ring.consume - txq->tx_ring.produce - 1);
90  /* Wrapped? */
91  if (txq->tx_ring.produce >= txq->tx_ring.consume)
92    count += txq->size;
93  return count;
94}
95
96VNET_DEVICE_CLASS_TX_FN (vmxnet3_device_class) (vlib_main_t * vm,
97						vlib_node_runtime_t * node,
98						vlib_frame_t * frame)
99{
100  vmxnet3_main_t *vmxm = &vmxnet3_main;
101  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
102  vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, rd->dev_instance);
103  u32 *buffers = vlib_frame_vector_args (frame);
104  u32 bi0;
105  vlib_buffer_t *b0;
106  vmxnet3_tx_desc *txd = 0;
107  u32 desc_idx, generation, first_idx;
108  u16 space_left;
109  u16 n_left = frame->n_vectors;
110  vmxnet3_txq_t *txq;
111  u16 qid = vm->thread_index % vd->num_tx_queues, produce;
112
113  if (PREDICT_FALSE (!(vd->flags & VMXNET3_DEVICE_F_LINK_UP)))
114    {
115      vlib_buffer_free (vm, buffers, n_left);
116      vlib_error_count (vm, node->node_index, VMXNET3_TX_ERROR_LINK_DOWN,
117			n_left);
118      return (0);
119    }
120
121  txq = vec_elt_at_index (vd->txqs, qid);
122  clib_spinlock_lock_if_init (&txq->lock);
123
124  vmxnet3_txq_release (vm, vd, txq);
125
126  produce = txq->tx_ring.produce;
127  while (PREDICT_TRUE (n_left))
128    {
129      u16 space_needed = 1, i;
130      u32 gso_size = 0;
131      vlib_buffer_t *b;
132      u32 hdr_len = 0;
133
134      bi0 = buffers[0];
135      b0 = vlib_get_buffer (vm, bi0);
136      b = b0;
137
138      space_left = vmxnet3_tx_ring_space_left (txq);
139      while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
140	{
141	  u32 next_buffer = b->next_buffer;
142
143	  b = vlib_get_buffer (vm, next_buffer);
144	  space_needed++;
145	}
146      if (PREDICT_FALSE (space_left < space_needed))
147	{
148	  vmxnet3_txq_release (vm, vd, txq);
149	  space_left = vmxnet3_tx_ring_space_left (txq);
150
151	  if (PREDICT_FALSE (space_left < space_needed))
152	    {
153	      vlib_buffer_free_one (vm, bi0);
154	      vlib_error_count (vm, node->node_index,
155				VMXNET3_TX_ERROR_NO_FREE_SLOTS, 1);
156	      buffers++;
157	      n_left--;
158	      /*
159	       * Drop this packet. But we may have enough room for the next
160	       * packet
161	       */
162	      continue;
163	    }
164	}
165
166      /*
167       * Toggle the generation bit for SOP fragment to avoid device starts
168       * reading incomplete packet
169       */
170      generation = txq->tx_ring.gen ^ VMXNET3_TXF_GEN;
171      first_idx = txq->tx_ring.produce;
172      for (i = 0; i < space_needed; i++)
173	{
174	  b0 = vlib_get_buffer (vm, bi0);
175	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
176
177	  desc_idx = txq->tx_ring.produce;
178
179	  vmxnet3_tx_ring_advance_produce (txq);
180	  txq->tx_ring.bufs[desc_idx] = bi0;
181
182	  txd = &txq->tx_desc[desc_idx];
183
184	  txd->address = vlib_buffer_get_current_pa (vm, b0);
185
186	  txd->flags[0] = generation | b0->current_length;
187	  txd->flags[1] = 0;
188	  if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_GSO))
189	    {
190	      /*
191	       * We should not be getting GSO outbound traffic unless it is
192	       * lro is enable
193	       */
194	      ASSERT (vd->gso_enable == 1);
195	      gso_size = vnet_buffer2 (b0)->gso_size;
196	      hdr_len = vnet_buffer (b0)->l4_hdr_offset +
197		sizeof (ethernet_header_t);
198	    }
199
200	  generation = txq->tx_ring.gen;
201	  bi0 = b0->next_buffer;
202	}
203      if (PREDICT_FALSE (gso_size != 0))
204	{
205	  txd->flags[1] = hdr_len;
206	  txd->flags[1] |= VMXNET3_TXF_OM (VMXNET3_OM_TSO);
207	  txd->flags[0] |= VMXNET3_TXF_MSSCOF (gso_size);
208	}
209      txd->flags[1] |= VMXNET3_TXF_CQ | VMXNET3_TXF_EOP;
210      asm volatile ("":::"memory");
211      /*
212       * Now toggle back the generation bit for the first segment.
213       * Device can start reading the packet
214       */
215      txq->tx_desc[first_idx].flags[0] ^= VMXNET3_TXF_GEN;
216
217      buffers++;
218      n_left--;
219    }
220
221  if (PREDICT_TRUE (produce != txq->tx_ring.produce))
222    vmxnet3_reg_write_inline (vd, 0, txq->reg_txprod, txq->tx_ring.produce);
223
224  clib_spinlock_unlock_if_init (&txq->lock);
225
226  return (frame->n_vectors - n_left);
227}
228
229/*
230 * fd.io coding-style-patch-verification: ON
231 *
232 * Local Variables:
233 * eval: (c-set-style "gnu")
234 * End:
235 */
236