misc.h revision aa97dd1c
1/*
2 * Copyright (c) 2016  Intel Corporation.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#ifndef _MISC_H_
17#define _MISC_H_
18
19#ifdef __cplusplus
20extern "C" {
21#endif
22
23static inline int
24xmm_cmp(const rte_xmm_t *da, const rte_xmm_t *sa)
25{
26	uint64_t ret;
27
28	ret = (sa->u64[0] ^ da->u64[0]) |
29		(sa->u64[1] ^ da->u64[1]);
30
31	return (ret != 0);
32}
33
34static inline int
35ymm_cmp(const _ymm_t *da, const _ymm_t *sa)
36{
37	uint64_t ret;
38
39	ret = (sa->u64[0] ^ da->u64[0]) |
40		(sa->u64[1] ^ da->u64[1]) |
41		(sa->u64[2] ^ da->u64[2]) |
42		(sa->u64[3] ^ da->u64[3]);
43
44	return (ret != 0);
45}
46
47static inline int
48ymm_mask_cmp(const _ymm_t *da, const _ymm_t *sa, const _ymm_t *sm)
49{
50	uint64_t ret;
51
52	ret = ((da->u64[0] & sm->u64[0]) ^ sa->u64[0]) |
53		((da->u64[1] & sm->u64[1]) ^ sa->u64[1]) |
54		((da->u64[2] & sm->u64[2]) ^ sa->u64[2]) |
55		((da->u64[3] & sm->u64[3]) ^ sa->u64[3]);
56
57	return (ret != 0);
58}
59
60/*
61 * Setup tx_offload field inside mbuf using raw 64-bit field.
62 * Consider to move it into DPDK librte_mbuf.
63 */
64static inline uint64_t
65_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso,
66	uint64_t ol3, uint64_t ol2)
67{
68	return il2 | il3 << 7 | il4 << 16 | tso << 24 | ol3 << 40 | ol2 << 49;
69}
70
71/*
72 * Given the value of mbuf's tx_offload, calculate L4 payload offset.
73 */
74static inline uint32_t
75_tx_offload_l4_offset(uint64_t ofl)
76{
77	uint32_t l2, l3, l4;
78
79	l2 = ofl & 0x7f;
80	l3 = ofl >> 7 & 0x1ff;
81	l4 = ofl >> 16 & UINT8_MAX;
82
83	return l2 + l3 + l4;
84}
85
86/*
87 * Routines to calculate L3/L4 checksums in SW.
88 * Pretty similar to ones from DPDK librte_net/rte_ip.h,
89 * but provide better performance (at least for tested configurations),
90 * and extended functionality.
91 * Consider to move them into DPDK librte_net/rte_ip.h.
92 */
93
94/* make compiler to generate: add %r1, %r2; adc $0, %r1. */
95#define CKSUM_ADD_CARRY(s, v)	do {       \
96	(s) += (v);                        \
97	(s) = ((s) < (v)) ? (s) + 1 : (s); \
98} while (0)
99
100/**
101 * Process the non-complemented checksum of a buffer.
102 * Similar  to rte_raw_cksum(), but provide better performance
103 * (at least on IA platforms).
104 * @param buf
105 *   Pointer to the buffer.
106 * @param size
107 *   Length of the buffer.
108 * @return
109 *   The non-complemented checksum.
110 */
111static inline uint16_t
112__raw_cksum(const uint8_t *buf, uint32_t size)
113{
114	uint64_t s, sum;
115	uint32_t i, n;
116	uint32_t dw1, dw2;
117	uint16_t w1, w2;
118	const uint64_t *b;
119
120	b = (const uint64_t *)buf;
121	n = size / sizeof(*b);
122	sum = 0;
123
124	/* main loop, consume 8 bytes per iteration. */
125	for (i = 0; i != n; i++) {
126		s = b[i];
127		CKSUM_ADD_CARRY(sum, s);
128	}
129
130	/* consume the remainder. */
131	n = size % sizeof(*b);
132	if (n != 0) {
133		/* position of the of last 8 bytes of data. */
134		b = (const uint64_t *)((uintptr_t)(b + i) + n - sizeof(*b));
135		/* calculate shift amount. */
136		n = (sizeof(*b) - n) * CHAR_BIT;
137		s = b[0] >> n;
138		CKSUM_ADD_CARRY(sum, s);
139	}
140
141	/* reduce to 16 bits */
142	dw1 = sum;
143	dw2 = sum >> 32;
144	CKSUM_ADD_CARRY(dw1, dw2);
145	w1 = dw1;
146	w2 = dw1 >> 16;
147	CKSUM_ADD_CARRY(w1, w2);
148	return w1;
149}
150
151/**
152 * Process UDP or TCP checksum over possibly multi-segmented packet.
153 * @param mb
154 *   The pointer to the mbuf with the packet.
155 * @param l4_ofs
156 *   Offset to the beginning of the L4 header (should be in first segment).
157 * @param cksum
158 *   Already pre-calculated pseudo-header checksum value.
159 * @return
160 *   The complemented checksum.
161 */
162static inline uint32_t
163__udptcp_mbuf_cksum(const struct rte_mbuf *mb, uint16_t l4_ofs,
164	uint32_t cksum)
165{
166	uint32_t dlen, i, plen;
167	const struct rte_mbuf *ms;
168	const void *data;
169
170	plen = rte_pktmbuf_pkt_len(mb);
171	ms = mb;
172
173	for (i = l4_ofs; i < plen && ms != NULL; i += dlen) {
174		data = rte_pktmbuf_mtod_offset(ms, const void *, l4_ofs);
175		dlen = rte_pktmbuf_data_len(ms) - l4_ofs;
176		cksum += __raw_cksum(data, dlen);
177		ms = ms->next;
178		l4_ofs = 0;
179	}
180
181	cksum = ((cksum & 0xffff0000) >> 16) + (cksum & 0xffff);
182	cksum = (~cksum) & 0xffff;
183	if (cksum == 0)
184		cksum = 0xffff;
185
186	return cksum;
187}
188
189/**
190 * Process the pseudo-header checksum of an IPv4 header.
191 *
192 * Depending on the ol_flags, the pseudo-header checksum expected by the
193 * drivers is not the same. For instance, when TSO is enabled, the IP
194 * payload length must not be included in the packet.
195 *
196 * When ol_flags is 0, it computes the standard pseudo-header checksum.
197 *
198 * @param ipv4_hdr
199 *   The pointer to the contiguous IPv4 header.
200 * @param ipv4_len
201 *   Length of the IPv4 header.
202 * @param ol_flags
203 *   The ol_flags of the associated mbuf.
204 * @return
205 *   The non-complemented checksum to set in the L4 header.
206 */
207static inline uint16_t
208_ipv4x_phdr_cksum(const struct ipv4_hdr *ipv4_hdr, size_t ipv4h_len,
209	uint64_t ol_flags)
210{
211	uint32_t s0, s1;
212
213	s0 = ipv4_hdr->src_addr;
214	s1 = ipv4_hdr->dst_addr;
215	CKSUM_ADD_CARRY(s0, s1);
216
217	if (ol_flags & PKT_TX_TCP_SEG)
218		s1 = 0;
219	else
220		s1 = rte_cpu_to_be_16(
221			(uint16_t)(rte_be_to_cpu_16(ipv4_hdr->total_length) -
222			ipv4h_len));
223
224	s1 += rte_cpu_to_be_16(ipv4_hdr->next_proto_id);
225	CKSUM_ADD_CARRY(s0, s1);
226
227	return __rte_raw_cksum_reduce(s0);
228}
229
230/**
231 * Process the IPv4 UDP or TCP checksum.
232 *
233 * @param mb
234 *   The pointer to the IPv4 packet.
235 * @param l4_ofs
236 *   Offset to the beginning of the L4 header (should be in first segment).
237 * @param ipv4_hdr
238 *   The pointer to the contiguous IPv4 header.
239 * @return
240 *   The complemented checksum to set in the IP packet.
241 */
242static inline int
243_ipv4_udptcp_mbuf_cksum(const struct rte_mbuf *mb, uint16_t l4_ofs,
244	const struct ipv4_hdr *ipv4_hdr)
245{
246	uint32_t cksum;
247
248	cksum = _ipv4x_phdr_cksum(ipv4_hdr, mb->l3_len, 0);
249	cksum = __udptcp_mbuf_cksum(mb, l4_ofs, cksum);
250
251	return cksum;
252}
253
254/**
255 * Process the IPv6 UDP or TCP checksum.
256 *
257 * @param mb
258 *   The pointer to the IPv6 packet.
259 * @param l4_ofs
260 *   Offset to the beginning of the L4 header (should be in first segment).
261 * @param ipv6_hdr
262 *   The pointer to the contiguous IPv6 header.
263 * @return
264 *   The complemented checksum to set in the IP packet.
265 */
266static inline int
267_ipv6_udptcp_mbuf_cksum(const struct rte_mbuf *mb, uint16_t l4_ofs,
268	const struct ipv6_hdr *ipv6_hdr)
269{
270	uint32_t cksum;
271
272	cksum = rte_ipv6_phdr_cksum(ipv6_hdr, 0);
273	cksum = __udptcp_mbuf_cksum(mb, l4_ofs, cksum);
274
275	return cksum;
276}
277
278static inline uint16_t
279_ipv4x_cksum(const void *iph, size_t len)
280{
281	uint16_t cksum;
282
283	cksum = __raw_cksum(iph, len);
284	return (cksum == 0xffff) ? cksum : ~cksum;
285}
286
287static inline int
288check_pkt_csum(const struct rte_mbuf *m, uint64_t ol_flags, uint32_t type,
289	uint32_t proto)
290{
291	const struct ipv4_hdr *l3h4;
292	const struct ipv6_hdr *l3h6;
293	const struct udp_hdr *l4h;
294	int32_t ret;
295	uint16_t csum;
296
297	ret = 0;
298	l3h4 = rte_pktmbuf_mtod_offset(m, const struct ipv4_hdr *, m->l2_len);
299	l3h6 = rte_pktmbuf_mtod_offset(m, const struct ipv6_hdr *, m->l2_len);
300
301	if ((ol_flags & PKT_RX_IP_CKSUM_BAD) != 0) {
302		csum = _ipv4x_cksum(l3h4, m->l3_len);
303		ret = (csum != UINT16_MAX);
304	}
305
306	if (ret == 0 && (ol_flags & PKT_RX_L4_CKSUM_BAD) != 0) {
307
308		/*
309		 * for IPv4 it is allowed to have zero UDP cksum,
310		 * for IPv6 valid UDP cksum is mandatory.
311		 */
312		if (type == TLE_V4) {
313			l4h = (const struct udp_hdr *)((uintptr_t)l3h4 +
314				m->l3_len);
315			csum = (proto == IPPROTO_UDP && l4h->dgram_cksum == 0) ?
316				UINT16_MAX : _ipv4_udptcp_mbuf_cksum(m,
317				m->l2_len + m->l3_len, l3h4);
318		} else
319			csum = _ipv6_udptcp_mbuf_cksum(m,
320				m->l2_len + m->l3_len, l3h6);
321
322		ret = (csum != UINT16_MAX);
323	}
324
325	return ret;
326}
327
328/*
329 * Analog of read-write locks, very much in favour of read side.
330 * Assumes, that there are no more then INT32_MAX concurrent readers.
331 * Consider to move into DPDK librte_eal.
332 */
333
334static inline int
335rwl_try_acquire(rte_atomic32_t *p)
336{
337	return rte_atomic32_add_return(p, 1);
338}
339
340static inline void
341rwl_release(rte_atomic32_t *p)
342{
343	rte_atomic32_sub(p, 1);
344}
345
346static inline int
347rwl_acquire(rte_atomic32_t *p)
348{
349	int32_t rc;
350
351	rc = rwl_try_acquire(p);
352	if (rc < 0)
353		rwl_release(p);
354	return rc;
355}
356
357static inline void
358rwl_down(rte_atomic32_t *p)
359{
360	 while (rte_atomic32_cmpset((volatile uint32_t *)p, 0, INT32_MIN) == 0)
361		rte_pause();
362}
363
364static inline void
365rwl_up(rte_atomic32_t *p)
366{
367	rte_atomic32_sub(p, INT32_MIN);
368}
369
370/* exclude NULLs from the final list of packets. */
371static inline uint32_t
372compress_pkt_list(struct rte_mbuf *pkt[], uint32_t nb_pkt, uint32_t nb_zero)
373{
374	uint32_t i, j, k, l;
375
376	for (j = nb_pkt; nb_zero != 0 && j-- != 0; ) {
377
378		/* found a hole. */
379		if (pkt[j] == NULL) {
380
381			/* find how big is it. */
382			for (i = j; i-- != 0 && pkt[i] == NULL; )
383				;
384			/* fill the hole. */
385			for (k = j + 1, l = i + 1; k != nb_pkt; k++, l++)
386				pkt[l] = pkt[k];
387
388			nb_pkt -= j - i;
389			nb_zero -= j - i;
390			j = i + 1;
391		}
392	}
393
394	return nb_pkt;
395}
396
397/* empty ring and free queued mbufs */
398static inline void
399empty_mbuf_ring(struct rte_ring *r)
400{
401	uint32_t i, n;
402	struct rte_mbuf *mb[MAX_PKT_BURST];
403
404	do {
405		n = rte_ring_dequeue_burst(r, (void **)mb, RTE_DIM(mb));
406		for (i = 0; i != n; i++)
407			rte_pktmbuf_free(mb[i]);
408	} while (n != 0);
409}
410
411#ifdef __cplusplus
412}
413#endif
414
415#endif /* _MISC_H_ */
416