1/*
2 * Copyright (c) 2016-2017  Intel Corporation.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#ifndef _MISC_H_
17#define _MISC_H_
18
19#include <tle_dpdk_wrapper.h>
20
21#ifdef __cplusplus
22extern "C" {
23#endif
24
25static inline int
26xmm_cmp(const rte_xmm_t *da, const rte_xmm_t *sa)
27{
28	uint64_t ret;
29
30	ret = (sa->u64[0] ^ da->u64[0]) |
31		(sa->u64[1] ^ da->u64[1]);
32
33	return (ret != 0);
34}
35
36static inline int
37ymm_cmp(const _ymm_t *da, const _ymm_t *sa)
38{
39	uint64_t ret;
40
41	ret = (sa->u64[0] ^ da->u64[0]) |
42		(sa->u64[1] ^ da->u64[1]) |
43		(sa->u64[2] ^ da->u64[2]) |
44		(sa->u64[3] ^ da->u64[3]);
45
46	return (ret != 0);
47}
48
49static inline int
50ymm_mask_cmp(const _ymm_t *da, const _ymm_t *sa, const _ymm_t *sm)
51{
52	uint64_t ret;
53
54	ret = ((da->u64[0] & sm->u64[0]) ^ sa->u64[0]) |
55		((da->u64[1] & sm->u64[1]) ^ sa->u64[1]) |
56		((da->u64[2] & sm->u64[2]) ^ sa->u64[2]) |
57		((da->u64[3] & sm->u64[3]) ^ sa->u64[3]);
58
59	return (ret != 0);
60}
61
62/*
63 * Setup tx_offload field inside mbuf using raw 64-bit field.
64 * Consider to move it into DPDK librte_mbuf.
65 */
66static inline uint64_t
67_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso,
68	uint64_t ol3, uint64_t ol2)
69{
70	return il2 | il3 << 7 | il4 << 16 | tso << 24 | ol3 << 40 | ol2 << 49;
71}
72
73/*
74 * Given the value of mbuf's tx_offload, calculate L4 payload offset.
75 */
76static inline uint32_t
77_tx_offload_l4_offset(uint64_t ofl)
78{
79	uint32_t l2, l3, l4;
80
81	l2 = ofl & 0x7f;
82	l3 = ofl >> 7 & 0x1ff;
83	l4 = ofl >> 16 & UINT8_MAX;
84
85	return l2 + l3 + l4;
86}
87
88/*
89 * Routines to calculate L3/L4 checksums in SW.
90 * Pretty similar to ones from DPDK librte_net/rte_ip.h,
91 * but provide better performance (at least for tested configurations),
92 * and extended functionality.
93 * Consider to move them into DPDK librte_net/rte_ip.h.
94 */
95
96/* make compiler to generate: add %r1, %r2; adc $0, %r1. */
97#define CKSUM_ADD_CARRY(s, v)	do {       \
98	(s) += (v);                        \
99	(s) = ((s) < (v)) ? (s) + 1 : (s); \
100} while (0)
101
102/**
103 * Process the non-complemented checksum of a buffer.
104 * Similar  to rte_raw_cksum(), but provide better performance
105 * (at least on IA platforms).
106 * @param buf
107 *   Pointer to the buffer.
108 * @param size
109 *   Length of the buffer.
110 * @return
111 *   The non-complemented checksum.
112 */
113static inline uint16_t
114__raw_cksum(const uint8_t *buf, uint32_t size)
115{
116	uint64_t s, sum;
117	uint32_t i, n;
118	uint32_t dw1, dw2;
119	uint16_t w1, w2;
120	const uint64_t *b;
121
122	b = (const uint64_t *)buf;
123	n = size / sizeof(*b);
124	sum = 0;
125
126	/* main loop, consume 8 bytes per iteration. */
127	for (i = 0; i != n; i++) {
128		s = b[i];
129		CKSUM_ADD_CARRY(sum, s);
130	}
131
132	/* consume the remainder. */
133	n = size % sizeof(*b);
134	if (n != 0) {
135		/* position of the of last 8 bytes of data. */
136		b = (const uint64_t *)((uintptr_t)(b + i) + n - sizeof(*b));
137		/* calculate shift amount. */
138		n = (sizeof(*b) - n) * CHAR_BIT;
139		s = b[0] >> n;
140		CKSUM_ADD_CARRY(sum, s);
141	}
142
143	/* reduce to 16 bits */
144	dw1 = sum;
145	dw2 = sum >> 32;
146	CKSUM_ADD_CARRY(dw1, dw2);
147	w1 = dw1;
148	w2 = dw1 >> 16;
149	CKSUM_ADD_CARRY(w1, w2);
150	return w1;
151}
152
153/**
154 * Process UDP or TCP checksum over possibly multi-segmented packet.
155 * @param mb
156 *   The pointer to the mbuf with the packet.
157 * @param l4_ofs
158 *   Offset to the beginning of the L4 header (should be in first segment).
159 * @param cksum
160 *   Already pre-calculated pseudo-header checksum value.
161 * @return
162 *   The complemented checksum.
163 */
164static inline uint32_t
165__udptcp_mbuf_cksum(const struct rte_mbuf *mb, uint16_t l4_ofs,
166	uint32_t cksum)
167{
168	uint32_t dlen, i, plen;
169	const struct rte_mbuf *ms;
170	const void *data;
171
172	plen = rte_pktmbuf_pkt_len(mb);
173	ms = mb;
174
175	for (i = l4_ofs; i < plen && ms != NULL; i += dlen) {
176		data = rte_pktmbuf_mtod_offset(ms, const void *, l4_ofs);
177		dlen = rte_pktmbuf_data_len(ms) - l4_ofs;
178		cksum += __raw_cksum(data, dlen);
179		ms = ms->next;
180		l4_ofs = 0;
181	}
182
183	cksum = ((cksum & 0xffff0000) >> 16) + (cksum & 0xffff);
184	cksum = (~cksum) & 0xffff;
185	if (cksum == 0)
186		cksum = 0xffff;
187
188	return cksum;
189}
190
191/**
192 * Process the pseudo-header checksum of an IPv4 header.
193 *
194 * Depending on the ol_flags, the pseudo-header checksum expected by the
195 * drivers is not the same. For instance, when TSO is enabled, the IP
196 * payload length must not be included in the packet.
197 *
198 * When ol_flags is 0, it computes the standard pseudo-header checksum.
199 *
200 * @param ipv4_hdr
201 *   The pointer to the contiguous IPv4 header.
202 * @param ipv4_len
203 *   Length of the IPv4 header.
204 * @param ol_flags
205 *   The ol_flags of the associated mbuf.
206 * @return
207 *   The non-complemented checksum to set in the L4 header.
208 */
209static inline uint16_t
210_ipv4x_phdr_cksum(const struct ipv4_hdr *ipv4_hdr, size_t ipv4h_len,
211	uint64_t ol_flags)
212{
213	uint32_t s0, s1;
214
215	s0 = ipv4_hdr->src_addr;
216	s1 = ipv4_hdr->dst_addr;
217	CKSUM_ADD_CARRY(s0, s1);
218
219	if (ol_flags & PKT_TX_TCP_SEG)
220		s1 = 0;
221	else
222		s1 = rte_cpu_to_be_16(
223			(uint16_t)(rte_be_to_cpu_16(ipv4_hdr->total_length) -
224			ipv4h_len));
225
226	s1 += rte_cpu_to_be_16(ipv4_hdr->next_proto_id);
227	CKSUM_ADD_CARRY(s0, s1);
228
229	return __rte_raw_cksum_reduce(s0);
230}
231
232/**
233 * Process the IPv4 UDP or TCP checksum.
234 *
235 * @param mb
236 *   The pointer to the IPv4 packet.
237 * @param l4_ofs
238 *   Offset to the beginning of the L4 header (should be in first segment).
239 * @param ipv4_hdr
240 *   The pointer to the contiguous IPv4 header.
241 * @return
242 *   The complemented checksum to set in the IP packet.
243 */
244static inline int
245_ipv4_udptcp_mbuf_cksum(const struct rte_mbuf *mb, uint16_t l4_ofs,
246	const struct ipv4_hdr *ipv4_hdr)
247{
248	uint32_t cksum;
249
250	cksum = _ipv4x_phdr_cksum(ipv4_hdr, mb->l3_len, 0);
251	cksum = __udptcp_mbuf_cksum(mb, l4_ofs, cksum);
252
253	return cksum;
254}
255
256/**
257 * Process the IPv6 UDP or TCP checksum.
258 *
259 * @param mb
260 *   The pointer to the IPv6 packet.
261 * @param l4_ofs
262 *   Offset to the beginning of the L4 header (should be in first segment).
263 * @param ipv6_hdr
264 *   The pointer to the contiguous IPv6 header.
265 * @return
266 *   The complemented checksum to set in the IP packet.
267 */
268static inline int
269_ipv6_udptcp_mbuf_cksum(const struct rte_mbuf *mb, uint16_t l4_ofs,
270	const struct ipv6_hdr *ipv6_hdr)
271{
272	uint32_t cksum;
273
274	cksum = rte_ipv6_phdr_cksum(ipv6_hdr, 0);
275	cksum = __udptcp_mbuf_cksum(mb, l4_ofs, cksum);
276
277	return cksum;
278}
279
280static inline uint16_t
281_ipv4x_cksum(const void *iph, size_t len)
282{
283	uint16_t cksum;
284
285	cksum = __raw_cksum(iph, len);
286	return (cksum == 0xffff) ? cksum : ~cksum;
287}
288
289/*
290 * helper function to check csum.
291 */
292static inline int
293check_pkt_csum(const struct rte_mbuf *m, uint64_t ol_flags, uint32_t type,
294	uint32_t proto)
295{
296	const struct ipv4_hdr *l3h4;
297	const struct ipv6_hdr *l3h6;
298	const struct udp_hdr *l4h;
299	uint64_t fl3, fl4;
300	uint16_t csum;
301	int32_t ret;
302
303	fl4 = ol_flags & PKT_RX_L4_CKSUM_MASK;
304	fl3 = (type == TLE_V4) ?
305		(ol_flags & PKT_RX_IP_CKSUM_MASK) : PKT_RX_IP_CKSUM_GOOD;
306
307	/* case 0: both ip and l4 cksum is verified or data is valid */
308	if ((fl3 | fl4) == (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD))
309		return 0;
310
311	/* case 1: either ip or l4 cksum bad */
312	if (fl3 == PKT_RX_IP_CKSUM_BAD || fl4 == PKT_RX_L4_CKSUM_BAD)
313		return 1;
314
315	/* case 2: either ip or l4 or both cksum is unknown */
316	l3h4 = rte_pktmbuf_mtod_offset(m, const struct ipv4_hdr *, m->l2_len);
317	l3h6 = rte_pktmbuf_mtod_offset(m, const struct ipv6_hdr *, m->l2_len);
318
319	ret = 0;
320	if (fl3 == PKT_RX_IP_CKSUM_UNKNOWN && l3h4->hdr_checksum != 0) {
321		csum = _ipv4x_cksum(l3h4, m->l3_len);
322		ret = (csum != UINT16_MAX);
323	}
324
325	if (ret == 0 && fl4 == PKT_RX_L4_CKSUM_UNKNOWN) {
326
327		/*
328		 * for IPv4 it is allowed to have zero UDP cksum,
329		 * for IPv6 valid UDP cksum is mandatory.
330		 */
331		if (type == TLE_V4) {
332			l4h = (const struct udp_hdr *)((uintptr_t)l3h4 +
333				m->l3_len);
334			csum = (proto == IPPROTO_UDP && l4h->dgram_cksum == 0) ?
335				UINT16_MAX : _ipv4_udptcp_mbuf_cksum(m,
336				m->l2_len + m->l3_len, l3h4);
337		} else
338			csum = _ipv6_udptcp_mbuf_cksum(m,
339				m->l2_len + m->l3_len, l3h6);
340
341		ret = (csum != UINT16_MAX);
342	}
343
344	return ret;
345}
346
347/*
348 * Analog of read-write locks, very much in favour of read side.
349 * Assumes, that there are no more then INT32_MAX concurrent readers.
350 * Consider to move into DPDK librte_eal.
351 */
352
353static inline int
354rwl_try_acquire(rte_atomic32_t *p)
355{
356	return rte_atomic32_add_return(p, 1);
357}
358
359static inline void
360rwl_release(rte_atomic32_t *p)
361{
362	rte_atomic32_sub(p, 1);
363}
364
365static inline int
366rwl_acquire(rte_atomic32_t *p)
367{
368	int32_t rc;
369
370	rc = rwl_try_acquire(p);
371	if (rc < 0)
372		rwl_release(p);
373	return rc;
374}
375
376static inline void
377rwl_down(rte_atomic32_t *p)
378{
379	 while (rte_atomic32_cmpset((volatile uint32_t *)p, 0, INT32_MIN) == 0)
380		rte_pause();
381}
382
383static inline void
384rwl_up(rte_atomic32_t *p)
385{
386	rte_atomic32_sub(p, INT32_MIN);
387}
388
389/* exclude NULLs from the final list of packets. */
390static inline uint32_t
391compress_pkt_list(struct rte_mbuf *pkt[], uint32_t nb_pkt, uint32_t nb_zero)
392{
393	uint32_t i, j, k, l;
394
395	for (j = nb_pkt; nb_zero != 0 && j-- != 0; ) {
396
397		/* found a hole. */
398		if (pkt[j] == NULL) {
399
400			/* find how big is it. */
401			for (i = j; i-- != 0 && pkt[i] == NULL; )
402				;
403			/* fill the hole. */
404			for (k = j + 1, l = i + 1; k != nb_pkt; k++, l++)
405				pkt[l] = pkt[k];
406
407			nb_pkt -= j - i;
408			nb_zero -= j - i;
409			j = i + 1;
410		}
411	}
412
413	return nb_pkt;
414}
415
416static inline void
417free_mbufs(struct rte_mbuf *mb[], uint32_t num)
418{
419	uint32_t i;
420
421	for (i = 0; i != num; i++)
422		rte_pktmbuf_free(mb[i]);
423}
424
425/* empty ring and free queued mbufs */
426static inline void
427empty_mbuf_ring(struct rte_ring *r)
428{
429	uint32_t n;
430	struct rte_mbuf *mb[MAX_PKT_BURST];
431
432	do {
433		n = _rte_ring_dequeue_burst(r, (void **)mb, RTE_DIM(mb));
434		free_mbufs(mb, n);
435	} while (n != 0);
436}
437
438static inline uint32_t
439_mbus_to_iovec(struct iovec *iv, struct rte_mbuf *mb[], uint32_t num)
440{
441	uint32_t i, ns;
442	uint32_t len, slen, tlen;
443	struct rte_mbuf *m, *next;
444	const void *src;
445
446	for (i = 0; i != num; i++) {
447
448		m = mb[i];
449		tlen = 0;
450		ns = 0;
451
452		do {
453			slen = m->data_len;
454			src = rte_pktmbuf_mtod(m, const void *);
455			len = RTE_MIN(iv->iov_len - tlen, slen);
456			rte_memcpy((uint8_t *)iv->iov_base + tlen, src, len);
457			slen -= len;
458			tlen += len;
459			if (slen != 0)
460				break;
461			ns++;
462			next = m->next;
463			rte_pktmbuf_free_seg(m);
464 			m = next;
465		 } while (m != NULL);
466
467		iv->iov_base = (uint8_t *)iv->iov_base + tlen;
468		iv->iov_len -= tlen;
469
470		/* partly consumed mbuf */
471		if (m != NULL) {
472			m->pkt_len = mb[i]->pkt_len - tlen;
473			m->data_len = slen;
474			m->data_off += len;
475			m->nb_segs = mb[i]->nb_segs - ns;
476			mb[i] = m;
477			break;
478		}
479	}
480
481	return i;
482}
483
484static inline uint32_t
485_iovec_to_mbsegs(struct iovec *iv, uint32_t seglen, struct rte_mbuf *mb[],
486	uint32_t num)
487{
488	uint32_t i;
489	uint32_t len, slen, tlen;
490	struct rte_mbuf *m;
491	void *dst;
492
493	tlen = 0;
494	for (i = 0; i != num; i++) {
495
496		m = mb[i];
497		slen = rte_pktmbuf_tailroom(m);
498		slen = RTE_MIN(slen, seglen - m->data_len);
499		len = RTE_MIN(iv->iov_len - tlen, slen);
500		dst = rte_pktmbuf_append(m, len);
501		rte_memcpy(dst, (uint8_t *)iv->iov_base + tlen, len);
502		tlen += len;
503		if (len != slen)
504			break;
505	}
506
507	iv->iov_base = (uint8_t *)iv->iov_base + tlen;
508	iv->iov_len -= tlen;
509
510	return i;
511}
512
513/**
514 * Remove len bytes at the beginning of an mbuf.
515 *
516 * It's an enhancement version of rte_pktmbuf_abj which not support
517 * adjusting length greater than the length of the first segment.
518 *
519 * Returns a pointer to the new mbuf. If the
520 * length is greater than the total length of the mbuf, then the
521 * function will fail and return NULL, without modifying the mbuf.
522 *
523 * @param m
524 *   The packet mbuf.
525 * @param len
526 *   The amount of data to remove (in bytes).
527 * @return
528 *   A pointer to the new start of the data.
529 */
530static inline struct rte_mbuf *
531_rte_pktmbuf_adj(struct rte_mbuf *m, uint32_t len)
532{
533	struct rte_mbuf *next;
534	uint32_t remain, plen;
535	uint16_t segs;
536
537	if (unlikely(len > m->pkt_len))
538		return NULL;
539
540	plen = m->pkt_len;
541	remain = len;
542	segs = m->nb_segs;
543	/* don't free last segment */
544	while (remain >= m->data_len && m->next) {
545		next = m->next;
546		remain -= m->data_len;
547		segs--;
548		rte_pktmbuf_free_seg(m);
549		m = next;
550	}
551
552	if (remain) {
553		m->data_len = (uint16_t)(m->data_len - remain);
554		m->data_off = (uint16_t)(m->data_off + remain);
555	}
556
557	m->pkt_len = plen - len;
558	m->nb_segs = segs;
559	return m;
560}
561
562/**
563 * Remove len bytes of data at the end of the mbuf.
564 *
565 * It's an enhancement version of rte_pktmbuf_trim, which not support
566 * removing length greater than the length of the last segment.
567 *
568 * @param m
569 *   The packet mbuf.
570 * @param len
571 *   The amount of data to remove (in bytes).
572 * @return
573 *   - 0: On success.
574 *   - -1: On error.
575 */
576static inline int
577_rte_pktmbuf_trim(struct rte_mbuf *m, uint32_t len)
578{
579	struct rte_mbuf *last, *next, *tmp;
580	uint32_t remain;
581	uint16_t segs;
582
583	if (unlikely(len > m->pkt_len))
584		return -1;
585
586	tmp = m;
587	/* find the last segment will remain after trim */
588	remain = m->pkt_len - len;
589	while (remain > tmp->data_len) {
590		remain -= tmp->data_len;
591		tmp = tmp->next;
592	}
593
594	/* trim the remained last segment */
595	tmp->data_len = remain;
596
597	/* remove trimmed segments */
598	segs = m->nb_segs;
599	last = tmp;
600	for (tmp = tmp->next; tmp != NULL; tmp = next) {
601		next = tmp->next;
602		rte_pktmbuf_free_seg(tmp);
603		segs--;
604	}
605
606	last->next = NULL;
607	m->pkt_len -= len;
608	m->nb_segs = segs;
609
610	return 0;
611}
612
613#ifdef __cplusplus
614}
615#endif
616
617#endif /* _MISC_H_ */
618