1703faabfSKonstantin Ananyev/*
2703faabfSKonstantin Ananyev * Copyright (c) 2016  Intel Corporation.
3703faabfSKonstantin Ananyev * Licensed under the Apache License, Version 2.0 (the "License");
4703faabfSKonstantin Ananyev * you may not use this file except in compliance with the License.
5703faabfSKonstantin Ananyev * You may obtain a copy of the License at:
6703faabfSKonstantin Ananyev *
7703faabfSKonstantin Ananyev *     http://www.apache.org/licenses/LICENSE-2.0
8703faabfSKonstantin Ananyev *
9703faabfSKonstantin Ananyev * Unless required by applicable law or agreed to in writing, software
10703faabfSKonstantin Ananyev * distributed under the License is distributed on an "AS IS" BASIS,
11703faabfSKonstantin Ananyev * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12703faabfSKonstantin Ananyev * See the License for the specific language governing permissions and
13703faabfSKonstantin Ananyev * limitations under the License.
14703faabfSKonstantin Ananyev */
15703faabfSKonstantin Ananyev
16703faabfSKonstantin Ananyev#include <string.h>
17703faabfSKonstantin Ananyev#include <stdarg.h>
18703faabfSKonstantin Ananyev#include <stdio.h>
19703faabfSKonstantin Ananyev#include <stdlib.h>
20703faabfSKonstantin Ananyev#include <stdint.h>
21703faabfSKonstantin Ananyev#include <inttypes.h>
22703faabfSKonstantin Ananyev#include <errno.h>
23703faabfSKonstantin Ananyev#include <unistd.h>
24703faabfSKonstantin Ananyev
25703faabfSKonstantin Ananyev#include <rte_common.h>
26703faabfSKonstantin Ananyev#include <rte_log.h>
27703faabfSKonstantin Ananyev#include <rte_errno.h>
28703faabfSKonstantin Ananyev#include <rte_launch.h>
29703faabfSKonstantin Ananyev#include <rte_cycles.h>
30703faabfSKonstantin Ananyev#include <rte_eal.h>
31703faabfSKonstantin Ananyev#include <rte_ring.h>
32703faabfSKonstantin Ananyev#include <rte_per_lcore.h>
33703faabfSKonstantin Ananyev#include <rte_lcore.h>
34703faabfSKonstantin Ananyev#include <rte_random.h>
35703faabfSKonstantin Ananyev#include <rte_hexdump.h>
36703faabfSKonstantin Ananyev#include <rte_malloc.h>
37703faabfSKonstantin Ananyev
38703faabfSKonstantin Ananyev#include <tle_memtank.h>
39703faabfSKonstantin Ananyev
40703faabfSKonstantin Ananyevstruct memstat {
41703faabfSKonstantin Ananyev	struct {
42703faabfSKonstantin Ananyev		rte_atomic64_t nb_call;
43703faabfSKonstantin Ananyev		rte_atomic64_t nb_fail;
44703faabfSKonstantin Ananyev		rte_atomic64_t sz;
45703faabfSKonstantin Ananyev	} alloc;
46703faabfSKonstantin Ananyev	struct {
47703faabfSKonstantin Ananyev		rte_atomic64_t nb_call;
48703faabfSKonstantin Ananyev		rte_atomic64_t nb_fail;
49703faabfSKonstantin Ananyev	} free;
50703faabfSKonstantin Ananyev	uint64_t nb_alloc_obj;
51703faabfSKonstantin Ananyev};
52703faabfSKonstantin Ananyev
53703faabfSKonstantin Ananyevstruct memtank_stat {
54703faabfSKonstantin Ananyev	uint64_t nb_cycle;
55703faabfSKonstantin Ananyev	struct {
56703faabfSKonstantin Ananyev		uint64_t nb_call;
57703faabfSKonstantin Ananyev		uint64_t nb_req;
58703faabfSKonstantin Ananyev		uint64_t nb_alloc;
59703faabfSKonstantin Ananyev		uint64_t nb_cycle;
60db273c22SKonstantin Ananyev		uint64_t max_cycle;
61db273c22SKonstantin Ananyev		uint64_t min_cycle;
62703faabfSKonstantin Ananyev	} alloc;
63703faabfSKonstantin Ananyev	struct {
64703faabfSKonstantin Ananyev		uint64_t nb_call;
65703faabfSKonstantin Ananyev		uint64_t nb_free;
66703faabfSKonstantin Ananyev		uint64_t nb_cycle;
67db273c22SKonstantin Ananyev		uint64_t max_cycle;
68db273c22SKonstantin Ananyev		uint64_t min_cycle;
69703faabfSKonstantin Ananyev	} free;
70703faabfSKonstantin Ananyev	struct {
71703faabfSKonstantin Ananyev		uint64_t nb_call;
72703faabfSKonstantin Ananyev		uint64_t nb_chunk;
73703faabfSKonstantin Ananyev		uint64_t nb_cycle;
74db273c22SKonstantin Ananyev		uint64_t max_cycle;
75db273c22SKonstantin Ananyev		uint64_t min_cycle;
76703faabfSKonstantin Ananyev	} grow;
77703faabfSKonstantin Ananyev	struct {
78703faabfSKonstantin Ananyev		uint64_t nb_call;
79703faabfSKonstantin Ananyev		uint64_t nb_chunk;
80703faabfSKonstantin Ananyev		uint64_t nb_cycle;
81db273c22SKonstantin Ananyev		uint64_t max_cycle;
82db273c22SKonstantin Ananyev		uint64_t min_cycle;
83703faabfSKonstantin Ananyev	} shrink;
84703faabfSKonstantin Ananyev};
85703faabfSKonstantin Ananyev
86703faabfSKonstantin Ananyevstruct master_args {
87703faabfSKonstantin Ananyev	uint64_t run_cycles;
88703faabfSKonstantin Ananyev	uint32_t delay_us;
89703faabfSKonstantin Ananyev	uint32_t flags;
90703faabfSKonstantin Ananyev};
91703faabfSKonstantin Ananyev
92703faabfSKonstantin Ananyevstruct worker_args {
93703faabfSKonstantin Ananyev	uint32_t max_obj;
94703faabfSKonstantin Ananyev	uint32_t obj_size;
95703faabfSKonstantin Ananyev	uint32_t alloc_flags;
96703faabfSKonstantin Ananyev	uint32_t free_flags;
97db273c22SKonstantin Ananyev	struct rte_ring *rng;
98703faabfSKonstantin Ananyev};
99703faabfSKonstantin Ananyev
100703faabfSKonstantin Ananyevstruct memtank_arg {
101703faabfSKonstantin Ananyev	struct tle_memtank *mt;
102703faabfSKonstantin Ananyev	union {
103703faabfSKonstantin Ananyev		struct master_args master;
104703faabfSKonstantin Ananyev		struct worker_args worker;
105703faabfSKonstantin Ananyev	};
106703faabfSKonstantin Ananyev	struct memtank_stat stats;
107db273c22SKonstantin Ananyev} __rte_cache_aligned;
108703faabfSKonstantin Ananyev
109703faabfSKonstantin Ananyev#define BULK_NUM	32
110703faabfSKonstantin Ananyev
111703faabfSKonstantin Ananyev#define	OBJ_SZ_MIN	1
112703faabfSKonstantin Ananyev#define	OBJ_SZ_MAX	0x100000
113703faabfSKonstantin Ananyev#define	OBJ_SZ_DEF	(4 * RTE_CACHE_LINE_SIZE + 1)
114703faabfSKonstantin Ananyev
115703faabfSKonstantin Ananyev#define TEST_TIME	10
116db273c22SKonstantin Ananyev#define CLEANUP_TIME	3
117703faabfSKonstantin Ananyev
118703faabfSKonstantin Ananyev#define FREE_THRSH_MIN	0
119703faabfSKonstantin Ananyev#define FREE_THRSH_MAX	100
120703faabfSKonstantin Ananyev
121703faabfSKonstantin Ananyevenum {
122703faabfSKonstantin Ananyev	WRK_CMD_STOP,
123703faabfSKonstantin Ananyev	WRK_CMD_RUN,
124703faabfSKonstantin Ananyev};
125703faabfSKonstantin Ananyev
126703faabfSKonstantin Ananyevenum {
127703faabfSKonstantin Ananyev	MASTER_FLAG_GROW = 1,
128703faabfSKonstantin Ananyev	MASTER_FLAG_SHRINK = 2,
129703faabfSKonstantin Ananyev};
130703faabfSKonstantin Ananyev
131703faabfSKonstantin Ananyevenum {
132703faabfSKonstantin Ananyev	MEM_FUNC_SYS,
133703faabfSKonstantin Ananyev	MEM_FUNC_RTE,
134703faabfSKonstantin Ananyev};
135703faabfSKonstantin Ananyev
136703faabfSKonstantin Ananyevstatic uint32_t wrk_cmd __rte_cache_aligned;
137703faabfSKonstantin Ananyev
138703faabfSKonstantin Ananyevstatic struct tle_memtank_prm mtnk_prm = {
139703faabfSKonstantin Ananyev	.min_free = 4 * BULK_NUM,
140703faabfSKonstantin Ananyev	.max_free = 32 * BULK_NUM,
141703faabfSKonstantin Ananyev	.obj_size = OBJ_SZ_DEF,
142703faabfSKonstantin Ananyev	.obj_align = RTE_CACHE_LINE_SIZE,
143703faabfSKonstantin Ananyev	.nb_obj_chunk = BULK_NUM,
144703faabfSKonstantin Ananyev	.flags = TLE_MTANK_OBJ_DBG,
145703faabfSKonstantin Ananyev};
146703faabfSKonstantin Ananyev
147703faabfSKonstantin Ananyevstatic struct {
148703faabfSKonstantin Ananyev	uint32_t run_time;       /* test run-time in seconds */
149703faabfSKonstantin Ananyev	uint32_t wrk_max_obj;    /* max alloced objects per worker */
150703faabfSKonstantin Ananyev	uint32_t wrk_free_thrsh; /* wrk free thresh % (0-100) */
151703faabfSKonstantin Ananyev	int32_t mem_func;        /* memory subsystem to use for alloc/free */
152703faabfSKonstantin Ananyev} global_cfg = {
153703faabfSKonstantin Ananyev	.run_time = TEST_TIME,
154703faabfSKonstantin Ananyev	.wrk_max_obj = 2 * BULK_NUM,
155703faabfSKonstantin Ananyev	.wrk_free_thrsh = FREE_THRSH_MIN,
156703faabfSKonstantin Ananyev	.mem_func = MEM_FUNC_SYS,
157703faabfSKonstantin Ananyev};
158703faabfSKonstantin Ananyev
159703faabfSKonstantin Ananyevstatic void *
160703faabfSKonstantin Ananyevalloc_func(size_t sz)
161703faabfSKonstantin Ananyev{
162703faabfSKonstantin Ananyev	switch (global_cfg.mem_func) {
163703faabfSKonstantin Ananyev	case MEM_FUNC_SYS:
164703faabfSKonstantin Ananyev		return malloc(sz);
165703faabfSKonstantin Ananyev	case MEM_FUNC_RTE:
166703faabfSKonstantin Ananyev		return rte_malloc(NULL, sz, 0);
167703faabfSKonstantin Ananyev	}
168703faabfSKonstantin Ananyev
169703faabfSKonstantin Ananyev	return NULL;
170703faabfSKonstantin Ananyev}
171703faabfSKonstantin Ananyev
172703faabfSKonstantin Ananyevstatic void
173703faabfSKonstantin Ananyevfree_func(void *p)
174703faabfSKonstantin Ananyev{
175703faabfSKonstantin Ananyev	switch (global_cfg.mem_func) {
176703faabfSKonstantin Ananyev	case MEM_FUNC_SYS:
177703faabfSKonstantin Ananyev		return free(p);
178703faabfSKonstantin Ananyev	case MEM_FUNC_RTE:
179703faabfSKonstantin Ananyev		return rte_free(p);
180703faabfSKonstantin Ananyev	}
181703faabfSKonstantin Ananyev}
182703faabfSKonstantin Ananyev
183703faabfSKonstantin Ananyevstatic void *
184703faabfSKonstantin Ananyevtest_alloc1(size_t sz, void *p)
185703faabfSKonstantin Ananyev{
186703faabfSKonstantin Ananyev	struct memstat *ms;
187703faabfSKonstantin Ananyev	void *buf;
188703faabfSKonstantin Ananyev
189703faabfSKonstantin Ananyev	ms = p;
190703faabfSKonstantin Ananyev	buf = alloc_func(sz);
191703faabfSKonstantin Ananyev	rte_atomic64_inc(&ms->alloc.nb_call);
192703faabfSKonstantin Ananyev	if (buf != NULL) {
193703faabfSKonstantin Ananyev		memset(buf, 0, sz);
194703faabfSKonstantin Ananyev		rte_atomic64_add(&ms->alloc.sz, sz);
195703faabfSKonstantin Ananyev	} else
196703faabfSKonstantin Ananyev		rte_atomic64_inc(&ms->alloc.nb_fail);
197703faabfSKonstantin Ananyev
198703faabfSKonstantin Ananyev	return buf;
199703faabfSKonstantin Ananyev}
200703faabfSKonstantin Ananyev
201703faabfSKonstantin Ananyevstatic void
202703faabfSKonstantin Ananyevtest_free1(void *buf, void *p)
203703faabfSKonstantin Ananyev{
204703faabfSKonstantin Ananyev	struct memstat *ms;
205703faabfSKonstantin Ananyev
206703faabfSKonstantin Ananyev	ms = p;
207703faabfSKonstantin Ananyev
208703faabfSKonstantin Ananyev	free_func(buf);
209703faabfSKonstantin Ananyev	rte_atomic64_inc(&ms->free.nb_call);
210703faabfSKonstantin Ananyev	if (buf == NULL)
211703faabfSKonstantin Ananyev		rte_atomic64_inc(&ms->free.nb_fail);
212703faabfSKonstantin Ananyev}
213703faabfSKonstantin Ananyev
214703faabfSKonstantin Ananyevstatic void
215703faabfSKonstantin Ananyevmemstat_dump(FILE *f, struct memstat *ms)
216703faabfSKonstantin Ananyev{
217703faabfSKonstantin Ananyev
218703faabfSKonstantin Ananyev	uint64_t alloc_sz, nb_alloc;
219703faabfSKonstantin Ananyev	long double muc, mut;
220703faabfSKonstantin Ananyev
221703faabfSKonstantin Ananyev	nb_alloc = rte_atomic64_read(&ms->alloc.nb_call) -
222703faabfSKonstantin Ananyev		rte_atomic64_read(&ms->alloc.nb_fail);
223703faabfSKonstantin Ananyev	alloc_sz = rte_atomic64_read(&ms->alloc.sz) / nb_alloc;
224703faabfSKonstantin Ananyev	nb_alloc -= rte_atomic64_read(&ms->free.nb_call) -
225703faabfSKonstantin Ananyev		rte_atomic64_read(&ms->free.nb_fail);
226703faabfSKonstantin Ananyev	alloc_sz *= nb_alloc;
227703faabfSKonstantin Ananyev	mut = (alloc_sz == 0) ? 1 :
228703faabfSKonstantin Ananyev		(long double)ms->nb_alloc_obj * mtnk_prm.obj_size / alloc_sz;
229703faabfSKonstantin Ananyev	muc = (alloc_sz == 0) ? 1 :
230703faabfSKonstantin Ananyev		(long double)(ms->nb_alloc_obj + mtnk_prm.max_free) *
231703faabfSKonstantin Ananyev		mtnk_prm.obj_size / alloc_sz;
232703faabfSKonstantin Ananyev
233703faabfSKonstantin Ananyev	fprintf(f, "%s(%p)={\n", __func__, ms);
234703faabfSKonstantin Ananyev	fprintf(f, "\talloc={\n");
235