1/*-
2 *   BSD LICENSE
3 *
4 *   Copyright(c) 2015 Cavium networks. All rights reserved.
5 *   All rights reserved.
6 *
7 *   Redistribution and use in source and binary forms, with or without
8 *   modification, are permitted provided that the following conditions
9 *   are met:
10 *
11 *     * Redistributions of source code must retain the above copyright
12 *       notice, this list of conditions and the following disclaimer.
13 *     * Redistributions in binary form must reproduce the above copyright
14 *       notice, this list of conditions and the following disclaimer in
15 *       the documentation and/or other materials provided with the
16 *       distribution.
17 *     * Neither the name of Cavium networks nor the names of its
18 *       contributors may be used to endorse or promote products derived
19 *       from this software without specific prior written permission.
20 *
21 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#ifndef _RTE_CRC_ARM64_H_
35#define _RTE_CRC_ARM64_H_
36
37/**
38 * @file
39 *
40 * RTE CRC arm64 Hash
41 */
42
43#ifdef __cplusplus
44extern "C" {
45#endif
46
47#include <stdint.h>
48#include <rte_cpuflags.h>
49#include <rte_branch_prediction.h>
50#include <rte_common.h>
51
52static inline uint32_t
53crc32c_arm64_u8(uint8_t data, uint32_t init_val)
54{
55	asm(".arch armv8-a+crc");
56	__asm__ volatile(
57			"crc32cb %w[crc], %w[crc], %w[value]"
58			: [crc] "+r" (init_val)
59			: [value] "r" (data));
60	return init_val;
61}
62
63static inline uint32_t
64crc32c_arm64_u16(uint16_t data, uint32_t init_val)
65{
66	asm(".arch armv8-a+crc");
67	__asm__ volatile(
68			"crc32ch %w[crc], %w[crc], %w[value]"
69			: [crc] "+r" (init_val)
70			: [value] "r" (data));
71	return init_val;
72}
73
74static inline uint32_t
75crc32c_arm64_u32(uint32_t data, uint32_t init_val)
76{
77	asm(".arch armv8-a+crc");
78	__asm__ volatile(
79			"crc32cw %w[crc], %w[crc], %w[value]"
80			: [crc] "+r" (init_val)
81			: [value] "r" (data));
82	return init_val;
83}
84
85static inline uint32_t
86crc32c_arm64_u64(uint64_t data, uint32_t init_val)
87{
88	asm(".arch armv8-a+crc");
89	__asm__ volatile(
90			"crc32cx %w[crc], %w[crc], %x[value]"
91			: [crc] "+r" (init_val)
92			: [value] "r" (data));
93	return init_val;
94}
95
96/**
97 * Allow or disallow use of arm64 SIMD instrinsics for CRC32 hash
98 * calculation.
99 *
100 * @param alg
101 *   An OR of following flags:
102 *   - (CRC32_SW) Don't use arm64 crc intrinsics
103 *   - (CRC32_ARM64) Use ARMv8 CRC intrinsic if available
104 *
105 */
106static inline void
107rte_hash_crc_set_alg(uint8_t alg)
108{
109	switch (alg) {
110	case CRC32_ARM64:
111		if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_CRC32))
112			alg = CRC32_SW;
113	case CRC32_SW:
114		crc32_alg = alg;
115	default:
116		break;
117	}
118}
119
120/* Setting the best available algorithm */
121static inline void __attribute__((constructor))
122rte_hash_crc_init_alg(void)
123{
124	rte_hash_crc_set_alg(CRC32_ARM64);
125}
126
127/**
128 * Use single crc32 instruction to perform a hash on a 1 byte value.
129 * Fall back to software crc32 implementation in case arm64 crc intrinsics is
130 * not supported
131 *
132 * @param data
133 *   Data to perform hash on.
134 * @param init_val
135 *   Value to initialise hash generator.
136 * @return
137 *   32bit calculated hash value.
138 */
139static inline uint32_t
140rte_hash_crc_1byte(uint8_t data, uint32_t init_val)
141{
142	if (likely(crc32_alg & CRC32_ARM64))
143		return crc32c_arm64_u8(data, init_val);
144
145	return crc32c_1byte(data, init_val);
146}
147
148/**
149 * Use single crc32 instruction to perform a hash on a 2 bytes value.
150 * Fall back to software crc32 implementation in case arm64 crc intrinsics is
151 * not supported
152 *
153 * @param data
154 *   Data to perform hash on.
155 * @param init_val
156 *   Value to initialise hash generator.
157 * @return
158 *   32bit calculated hash value.
159 */
160static inline uint32_t
161rte_hash_crc_2byte(uint16_t data, uint32_t init_val)
162{
163	if (likely(crc32_alg & CRC32_ARM64))
164		return crc32c_arm64_u16(data, init_val);
165
166	return crc32c_2bytes(data, init_val);
167}
168
169/**
170 * Use single crc32 instruction to perform a hash on a 4 byte value.
171 * Fall back to software crc32 implementation in case arm64 crc intrinsics is
172 * not supported
173 *
174 * @param data
175 *   Data to perform hash on.
176 * @param init_val
177 *   Value to initialise hash generator.
178 * @return
179 *   32bit calculated hash value.
180 */
181static inline uint32_t
182rte_hash_crc_4byte(uint32_t data, uint32_t init_val)
183{
184	if (likely(crc32_alg & CRC32_ARM64))
185		return crc32c_arm64_u32(data, init_val);
186
187	return crc32c_1word(data, init_val);
188}
189
190/**
191 * Use single crc32 instruction to perform a hash on a 8 byte value.
192 * Fall back to software crc32 implementation in case arm64 crc intrinsics is
193 * not supported
194 *
195 * @param data
196 *   Data to perform hash on.
197 * @param init_val
198 *   Value to initialise hash generator.
199 * @return
200 *   32bit calculated hash value.
201 */
202static inline uint32_t
203rte_hash_crc_8byte(uint64_t data, uint32_t init_val)
204{
205	if (likely(crc32_alg == CRC32_ARM64))
206		return crc32c_arm64_u64(data, init_val);
207
208	return crc32c_2words(data, init_val);
209}
210
211#ifdef __cplusplus
212}
213#endif
214
215#endif /* _RTE_CRC_ARM64_H_ */
216