18b25d1adSChristian Ehrhardt/*-
28b25d1adSChristian Ehrhardt *   BSD LICENSE
38b25d1adSChristian Ehrhardt *
48b25d1adSChristian Ehrhardt *   Copyright(c) 2016 Intel Corporation. All rights reserved.
58b25d1adSChristian Ehrhardt *   All rights reserved.
68b25d1adSChristian Ehrhardt *
78b25d1adSChristian Ehrhardt *   Redistribution and use in source and binary forms, with or without
88b25d1adSChristian Ehrhardt *   modification, are permitted provided that the following conditions
98b25d1adSChristian Ehrhardt *   are met:
108b25d1adSChristian Ehrhardt *
118b25d1adSChristian Ehrhardt *     * Redistributions of source code must retain the above copyright
128b25d1adSChristian Ehrhardt *	 notice, this list of conditions and the following disclaimer.
138b25d1adSChristian Ehrhardt *     * Redistributions in binary form must reproduce the above copyright
148b25d1adSChristian Ehrhardt *	 notice, this list of conditions and the following disclaimer in
158b25d1adSChristian Ehrhardt *	 the documentation and/or other materials provided with the
168b25d1adSChristian Ehrhardt *	 distribution.
178b25d1adSChristian Ehrhardt *     * Neither the name of Intel Corporation nor the names of its
188b25d1adSChristian Ehrhardt *	 contributors may be used to endorse or promote products derived
198b25d1adSChristian Ehrhardt *	 from this software without specific prior written permission.
208b25d1adSChristian Ehrhardt *
218b25d1adSChristian Ehrhardt *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
228b25d1adSChristian Ehrhardt *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
238b25d1adSChristian Ehrhardt *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
248b25d1adSChristian Ehrhardt *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
258b25d1adSChristian Ehrhardt *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
268b25d1adSChristian Ehrhardt *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
278b25d1adSChristian Ehrhardt *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
288b25d1adSChristian Ehrhardt *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
298b25d1adSChristian Ehrhardt *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
308b25d1adSChristian Ehrhardt *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
318b25d1adSChristian Ehrhardt *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
328b25d1adSChristian Ehrhardt */
338b25d1adSChristian Ehrhardt#include <inttypes.h>
348b25d1adSChristian Ehrhardt#include <locale.h>
358b25d1adSChristian Ehrhardt
368b25d1adSChristian Ehrhardt#include <rte_cycles.h>
378b25d1adSChristian Ehrhardt#include <rte_hash.h>
388b25d1adSChristian Ehrhardt#include <rte_hash_crc.h>
398b25d1adSChristian Ehrhardt#include <rte_launch.h>
408b25d1adSChristian Ehrhardt#include <rte_malloc.h>
418b25d1adSChristian Ehrhardt#include <rte_random.h>
428b25d1adSChristian Ehrhardt#include <rte_spinlock.h>
438b25d1adSChristian Ehrhardt
448b25d1adSChristian Ehrhardt#include "test.h"
458b25d1adSChristian Ehrhardt
468b25d1adSChristian Ehrhardt/*
478b25d1adSChristian Ehrhardt * Check condition and return an error if true. Assumes that "handle" is the
488b25d1adSChristian Ehrhardt * name of the hash structure pointer to be freed.
498b25d1adSChristian Ehrhardt */
508b25d1adSChristian Ehrhardt#define RETURN_IF_ERROR(cond, str, ...) do {                            \
518b25d1adSChristian Ehrhardt	if (cond) {                                                     \
528b25d1adSChristian Ehrhardt		printf("ERROR line %d: " str "\n", __LINE__,            \
538b25d1adSChristian Ehrhardt							##__VA_ARGS__);	\
548b25d1adSChristian Ehrhardt		if (handle)                                             \
558b25d1adSChristian Ehrhardt			rte_hash_free(handle);                          \
568b25d1adSChristian Ehrhardt		return -1;                                              \
578b25d1adSChristian Ehrhardt	}                                                               \
588b25d1adSChristian Ehrhardt} while (0)
598b25d1adSChristian Ehrhardt
608b25d1adSChristian Ehrhardt#define RTE_APP_TEST_HASH_MULTIWRITER_FAILED 0
618b25d1adSChristian Ehrhardt
628b25d1adSChristian Ehrhardtstruct {
638b25d1adSChristian Ehrhardt	uint32_t *keys;
648b25d1adSChristian Ehrhardt	uint32_t *found;
658b25d1adSChristian Ehrhardt	uint32_t nb_tsx_insertion;
668b25d1adSChristian Ehrhardt	struct rte_hash *h;
678b25d1adSChristian Ehrhardt} tbl_multiwriter_test_params;
688b25d1adSChristian Ehrhardt
698b25d1adSChristian Ehrhardtconst uint32_t nb_entries = 16*1024*1024;
708b25d1adSChristian Ehrhardtconst uint32_t nb_total_tsx_insertion = 15*1024*1024;
718b25d1adSChristian Ehrhardtuint32_t rounded_nb_total_tsx_insertion;
728b25d1adSChristian Ehrhardt
738b25d1adSChristian Ehrhardtstatic rte_atomic64_t gcycles;
748b25d1adSChristian Ehrhardtstatic rte_atomic64_t ginsertions;
758b25d1adSChristian Ehrhardt
768b25d1adSChristian Ehrhardtstatic int use_htm;
778b25d1adSChristian Ehrhardt
788b25d1adSChristian Ehrhardtstatic int
798e6d9d11SLuca Boccassitest_hash_multiwriter_worker(void *arg)
808b25d1adSChristian Ehrhardt{
818b25d1adSChristian Ehrhardt	uint64_t i, offset;
828e6d9d11SLuca Boccassi	uint16_t pos_core;
838b25d1adSChristian Ehrhardt	uint32_t lcore_id = rte_lcore_id();
848b25d1adSChristian Ehrhardt	uint64_t begin, cycles;
858e6d9d11SLuca Boccassi	uint16_t *enabled_core_ids = (uint16_t *)arg;
868b25d1adSChristian Ehrhardt
878e6d9d11SLuca Boccassi	for (pos_core = 0; pos_core < rte_lcore_count(); pos_core++) {
888e6d9d11SLuca Boccassi		if (enabled_core_ids[pos_core] == lcore_id)
898e6d9d11SLuca Boccassi			break;
908e6d9d11SLuca Boccassi	}
918e6d9d11SLuca Boccassi
928e6d9d11SLuca Boccassi	/*
938e6d9d11SLuca Boccassi	 * Calculate offset for entries based on the position of the
948e6d9d11SLuca Boccassi	 * logical core, from the master core (not counting not enabled cores)
958e6d9d11SLuca Boccassi	 */
968e6d9d11SLuca Boccassi	offset = pos_core * tbl_multiwriter_test_params.nb_tsx_insertion;
978b25d1adSChristian Ehrhardt
988b25d1adSChristian Ehrhardt	printf("Core #%d inserting %d: %'"PRId64" - %'"PRId64"\n",
998b25d1adSChristian Ehrhardt	       lcore_id, tbl_multiwriter_test_params.nb_tsx_insertion,
1008e6d9d11SLuca Boccassi	       offset,
1018e6d9d11SLuca Boccassi	       offset + tbl_multiwriter_test_params.nb_tsx_insertion - 1);
1028b25d1adSChristian Ehrhardt
1038b25d1adSChristian Ehrhardt	begin = rte_rdtsc_precise();
1048b25d1adSChristian Ehrhardt
1058b25d1adSChristian Ehrhardt	for (i = offset;
1068b25d1adSChristian Ehrhardt	     i < offset + tbl_multiwriter_test_params.nb_tsx_insertion;
1078b25d1adSChristian Ehrhardt	     i++) {
1088b25d1adSChristian Ehrhardt		if (rte_hash_add_key(tbl_multiwriter_test_params.h,
1098b25d1adSChristian Ehrhardt				     tbl_multiwriter_test_params.keys + i) < 0)
1108b25d1adSChristian Ehrhardt			break;
1118b25d1adSChristian Ehrhardt	}
1128b25d1adSChristian Ehrhardt
1138b25d1adSChristian Ehrhardt	cycles = rte_rdtsc_precise() - begin;
1148b25d1adSChristian Ehrhardt	rte_atomic64_add(&gcycles, cycles);
1158b25d1adSChristian Ehrhardt	rte_atomic64_add(&ginsertions, i - offset);
1168b25d1adSChristian Ehrhardt
1178b25d1adSChristian Ehrhardt	for (; i < offset + tbl_multiwriter_test_params.nb_tsx_insertion; i++)
1188b25d1adSChristian Ehrhardt		tbl_multiwriter_test_params.keys[i]
1198b25d1adSChristian Ehrhardt			= RTE_APP_TEST_HASH_MULTIWRITER_FAILED;
1208b25d1adSChristian Ehrhardt
1218b25d1adSChristian Ehrhardt	return 0;
1228b25d1adSChristian Ehrhardt}
1238b25d1adSChristian Ehrhardt
1248b25d1adSChristian Ehrhardt
1258b25d1adSChristian Ehrhardtstatic int
1268b25d1adSChristian Ehrhardttest_hash_multiwriter(void)
1278b25d1adSChristian Ehrhardt{
1288b25d1adSChristian Ehrhardt	unsigned int i, rounded_nb_total_tsx_insertion;
1298b25d1adSChristian Ehrhardt	static unsigned calledCount = 1;
1308e6d9d11SLuca Boccassi	uint16_t enabled_core_ids[RTE_MAX_LCORE];
1318e6d9d11SLuca Boccassi	uint16_t core_id;
1328b25d1adSChristian Ehrhardt
1338b25d1adSChristian Ehrhardt	uint32_t *keys;
1348b25d1adSChristian Ehrhardt	uint32_t *found;
1358b25d1adSChristian Ehrhardt
1368b25d1adSChristian Ehrhardt	struct rte_hash_parameters hash_params = {
1378b25d1adSChristian Ehrhardt		.entries = nb_entries,
1388b25d1adSChristian Ehrhardt		.key_len = sizeof(uint32_t),
1398b25d1adSChristian Ehrhardt		.hash_func = rte_hash_crc,
1408b25d1adSChristian Ehrhardt		.hash_func_init_val = 0,
1418b25d1adSChristian Ehrhardt		.socket_id = rte_socket_id(),
1428b25d1adSChristian Ehrhardt	};
1438b25d1adSChristian Ehrhardt	if (use_htm)
1448b25d1adSChristian Ehrhardt		hash_params.extra_flag =
1458b25d1adSChristian Ehrhardt			RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT
1468b25d1adSChristian Ehrhardt				| RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
1478b25d1adSChristian Ehrhardt	else
1488b25d1adSChristian Ehrhardt		hash_params.extra_flag =
1498b25d1adSChristian Ehrhardt			RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
1508b25d1adSChristian Ehrhardt
1518b25d1adSChristian Ehrhardt	struct rte_hash *handle;
1528b25d1adSChristian Ehrhardt	char name[RTE_HASH_NAMESIZE];
1538b25d1adSChristian Ehrhardt
1548b25d1adSChristian Ehrhardt	const void *next_key;
1558b25d1adSChristian Ehrhardt	void *next_data;
1568b25d1adSChristian Ehrhardt	uint32_t iter = 0;
1578b25d1adSChristian Ehrhardt
1588b25d1adSChristian Ehrhardt	uint32_t duplicated_keys = 0;
1598b25d1adSChristian Ehrhardt	uint32_t lost_keys = 0;
1608b25d1adSChristian Ehrhardt
1618b25d1adSChristian Ehrhardt	snprintf(name, 32, "test%u", calledCount++);
1628b25d1adSChristian Ehrhardt	hash_params.name = name;
1638b25d1adSChristian Ehrhardt
1648b25d1adSChristian Ehrhardt	handle = rte_hash_create(&hash_params);
1658b25d1adSChristian Ehrhardt	RETURN_IF_ERROR(handle == NULL, "hash creation failed");
1668b25d1adSChristian Ehrhardt
1678b25d1adSChristian Ehrhardt	tbl_multiwriter_test_params.h = handle;
1688b25d1adSChristian Ehrhardt	tbl_multiwriter_test_params.nb_tsx_insertion =
1698b25d1adSChristian Ehrhardt		nb_total_tsx_insertion / rte_lcore_count();
1708b25d1adSChristian Ehrhardt
1718b25d1adSChristian Ehrhardt	rounded_nb_total_tsx_insertion = (nb_total_tsx_insertion /
1728b25d1adSChristian Ehrhardt		tbl_multiwriter_test_params.nb_tsx_insertion)
1738b25d1adSChristian Ehrhardt		* tbl_multiwriter_test_params.nb_tsx_insertion;
1748b25d1adSChristian Ehrhardt
1758b25d1adSChristian Ehrhardt	rte_srand(rte_rdtsc());
1768b25d1adSChristian Ehrhardt
1778b25d1adSChristian Ehrhardt	keys = rte_malloc(NULL, sizeof(uint32_t) * nb_entries, 0);
1788b25d1adSChristian Ehrhardt
1798b25d1adSChristian Ehrhardt	if (keys == NULL) {
1808b25d1adSChristian Ehrhardt		printf("RTE_MALLOC failed\n");
1818b25d1adSChristian Ehrhardt		goto err1;
1828b25d1adSChristian Ehrhardt	}
1838b25d1adSChristian Ehrhardt
1848e6d9d11SLuca Boccassi	for (i = 0; i < nb_entries; i++)
1858e6d9d11SLuca Boccassi		keys[i] = i;
1868e6d9d11SLuca Boccassi
1878e6d9d11SLuca Boccassi	tbl_multiwriter_test_params.keys = keys;
1888e6d9d11SLuca Boccassi
1898b25d1adSChristian Ehrhardt	found = rte_zmalloc(NULL, sizeof(uint32_t) * nb_entries, 0);
1908b25d1adSChristian Ehrhardt	if (found == NULL) {
1918b25d1adSChristian Ehrhardt		printf("RTE_ZMALLOC failed\n");
1928b25d1adSChristian Ehrhardt		goto err2;
1938b25d1adSChristian Ehrhardt	}
1948b25d1adSChristian Ehrhardt
1958b25d1adSChristian Ehrhardt	tbl_multiwriter_test_params.found = found;
1968b25d1adSChristian Ehrhardt
1978b25d1adSChristian Ehrhardt	rte_atomic64_init(&gcycles);
1988b25d1adSChristian Ehrhardt	rte_atomic64_clear(&gcycles);
1998b25d1adSChristian Ehrhardt
2008b25d1adSChristian Ehrhardt	rte_atomic64_init(&ginsertions);
2018b25d1adSChristian Ehrhardt	rte_atomic64_clear(&ginsertions);
2028b25d1adSChristian Ehrhardt
2038e6d9d11SLuca Boccassi	/* Get list of enabled cores */
2048e6d9d11SLuca Boccassi	i = 0;
2058e6d9d11SLuca Boccassi	for (core_id = 0; core_id < RTE_MAX_LCORE; core_id++) {
2068e6d9d11SLuca Boccassi		if (i == rte_lcore_count())
2078e6d9d11SLuca Boccassi			break;
2088e6d9d11SLuca Boccassi
2098e6d9d11SLuca Boccassi		if (rte_lcore_is_enabled(core_id)) {
2108e6d9d11SLuca Boccassi			enabled_core_ids[i] = core_id;
2118e6d9d11SLuca Boccassi			i++;
2128e6d9d11SLuca Boccassi		}
2138e6d9d11SLuca Boccassi	}
2148e6d9d11SLuca Boccassi
2158e6d9d11SLuca Boccassi	if (i != rte_lcore_count()) {
2168e6d9d11SLuca Boccassi		printf("Number of enabled cores in list is different from "
2178e6d9d11SLuca Boccassi				"number given by rte_lcore_count()\n");
2188e6d9d11SLuca Boccassi		goto err3;
2198e6d9d11SLuca Boccassi	}
2208e6d9d11SLuca Boccassi
2218b25d1adSChristian Ehrhardt	/* Fire all threads. */
2228b25d1adSChristian Ehrhardt	rte_eal_mp_remote_launch(test_hash_multiwriter_worker,
2238e6d9d11SLuca Boccassi				 enabled_core_ids, CALL_MASTER);
2248b25d1adSChristian Ehrhardt	rte_eal_mp_wait_lcore();
2258b25d1adSChristian Ehrhardt
2268b25d1adSChristian Ehrhardt	while (rte_hash_iterate(handle, &next_key, &next_data, &iter) >= 0) {
2278b25d1adSChristian Ehrhardt		/* Search for the key in the list of keys added .*/
2288b25d1adSChristian Ehrhardt		i = *(const uint32_t *)next_key;
2298b25d1adSChristian Ehrhardt		tbl_multiwriter_test_params.found[i]++;
2308b25d1adSChristian Ehrhardt	}
2318b25d1adSChristian Ehrhardt
2328b25d1adSChristian Ehrhardt	for (i = 0; i < rounded_nb_total_tsx_insertion; i++) {
2338b25d1adSChristian Ehrhardt		if (tbl_multiwriter_test_params.keys[i]
2348b25d1adSChristian Ehrhardt		    != RTE_APP_TEST_HASH_MULTIWRITER_FAILED) {
2358b25d1adSChristian Ehrhardt			if (tbl_multiwriter_test_params.found[i] > 1) {
2368b25d1adSChristian Ehrhardt				duplicated_keys++;
2378b25d1adSChristian Ehrhardt				break;
2388b25d1adSChristian Ehrhardt			}
2398b25d1adSChristian Ehrhardt			if (tbl_multiwriter_test_params.found[i] == 0) {
2408b25d1adSChristian Ehrhardt				lost_keys++;
2418b25d1adSChristian Ehrhardt				printf("key %d is lost\n", i);
2428b25d1adSChristian Ehrhardt				break;
2438b25d1adSChristian Ehrhardt			}
2448b25d1adSChristian Ehrhardt		}
2458b25d1adSChristian Ehrhardt	}
2468b25d1adSChristian Ehrhardt
2478b25d1adSChristian Ehrhardt	if (duplicated_keys > 0) {
2488b25d1adSChristian Ehrhardt		printf("%d key duplicated\n", duplicated_keys);
2498b25d1adSChristian Ehrhardt		goto err3;
2508b25d1adSChristian Ehrhardt	}
2518b25d1adSChristian Ehrhardt
2528b25d1adSChristian Ehrhardt	if (lost_keys > 0) {
2538b25d1adSChristian Ehrhardt		printf("%d key lost\n", lost_keys);
2548b25d1adSChristian Ehrhardt		goto err3;
2558b25d1adSChristian Ehrhardt	}
2568b25d1adSChristian Ehrhardt
2578b25d1adSChristian Ehrhardt	printf("No key corrupted during multiwriter insertion.\n");
2588b25d1adSChristian Ehrhardt
2598b25d1adSChristian Ehrhardt	unsigned long long int cycles_per_insertion =
2608b25d1adSChristian Ehrhardt		rte_atomic64_read(&gcycles)/
2618b25d1adSChristian Ehrhardt		rte_atomic64_read(&ginsertions);
2628b25d1adSChristian Ehrhardt
2638b25d1adSChristian Ehrhardt	printf(" cycles per insertion: %llu\n", cycles_per_insertion);
2648b25d1adSChristian Ehrhardt
2658b25d1adSChristian Ehrhardt	rte_free(tbl_multiwriter_test_params.found);
2668b25d1adSChristian Ehrhardt	rte_free(tbl_multiwriter_test_params.keys);
2678b25d1adSChristian Ehrhardt	rte_hash_free(handle);
2688b25d1adSChristian Ehrhardt	return 0;
2698b25d1adSChristian Ehrhardt
2708b25d1adSChristian Ehrhardterr3:
2718b25d1adSChristian Ehrhardt	rte_free(tbl_multiwriter_test_params.found);
2728b25d1adSChristian Ehrhardterr2:
2738b25d1adSChristian Ehrhardt	rte_free(tbl_multiwriter_test_params.keys);
2748b25d1adSChristian Ehrhardterr1:
2758b25d1adSChristian Ehrhardt	rte_hash_free(handle);
2768b25d1adSChristian Ehrhardt	return -1;
2778b25d1adSChristian Ehrhardt}
2788b25d1adSChristian Ehrhardt
2798b25d1adSChristian Ehrhardtstatic int
2808b25d1adSChristian Ehrhardttest_hash_multiwriter_main(void)
2818b25d1adSChristian Ehrhardt{
2828b25d1adSChristian Ehrhardt	if (rte_lcore_count() == 1) {
2838b25d1adSChristian Ehrhardt		printf("More than one lcore is required to do multiwriter test\n");
2848b25d1adSChristian Ehrhardt		return 0;
2858b25d1adSChristian Ehrhardt	}
2868b25d1adSChristian Ehrhardt
2878b25d1adSChristian Ehrhardt
2888b25d1adSChristian Ehrhardt	setlocale(LC_NUMERIC, "");
2898b25d1adSChristian Ehrhardt
2908b25d1adSChristian Ehrhardt
2918b25d1adSChristian Ehrhardt	if (!rte_tm_supported()) {
2928b25d1adSChristian Ehrhardt		printf("Hardware transactional memory (lock elision) "
2938b25d1adSChristian Ehrhardt			"is NOT supported\n");
2948b25d1adSChristian Ehrhardt	} else {
2958b25d1adSChristian Ehrhardt		printf("Hardware transactional memory (lock elision) "
2968b25d1adSChristian Ehrhardt			"is supported\n");
2978b25d1adSChristian Ehrhardt
2988b25d1adSChristian Ehrhardt		printf("Test multi-writer with Hardware transactional memory\n");
2998b25d1adSChristian Ehrhardt
3008b25d1adSChristian Ehrhardt		use_htm = 1;
30132e04ea0SChristian Ehrhardt		if (test_hash_multiwriter() < 0)
30232e04ea0SChristian Ehrhardt			return -1;
3038b25d1adSChristian Ehrhardt	}
3048b25d1adSChristian Ehrhardt
3058b25d1adSChristian Ehrhardt	printf("Test multi-writer without Hardware transactional memory\n");
3068b25d1adSChristian Ehrhardt	use_htm = 0;
30732e04ea0SChristian Ehrhardt	if (test_hash_multiwriter() < 0)
30832e04ea0SChristian Ehrhardt		return -1;
3098b25d1adSChristian Ehrhardt
31032e04ea0SChristian Ehrhardt	return 0;
3118b25d1adSChristian Ehrhardt}
3128b25d1adSChristian Ehrhardt
3135d4e5dcdSRicardo SalvetiREGISTER_TEST_COMMAND(hash_multiwriter_autotest, test_hash_multiwriter_main);
314