1/*
2 * Copyright (c) 2016  Intel Corporation.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include <string.h>
17#include <stdarg.h>
18#include <stdio.h>
19#include <stdlib.h>
20#include <stdint.h>
21#include <inttypes.h>
22#include <errno.h>
23#include <unistd.h>
24
25#include <rte_common.h>
26#include <rte_log.h>
27#include <rte_errno.h>
28#include <rte_launch.h>
29#include <rte_cycles.h>
30#include <rte_eal.h>
31#include <rte_ring.h>
32#include <rte_per_lcore.h>
33#include <rte_lcore.h>
34#include <rte_random.h>
35#include <rte_hexdump.h>
36#include <rte_malloc.h>
37
38#include <tle_memtank.h>
39
40struct memstat {
41	struct {
42		rte_atomic64_t nb_call;
43		rte_atomic64_t nb_fail;
44		rte_atomic64_t sz;
45	} alloc;
46	struct {
47		rte_atomic64_t nb_call;
48		rte_atomic64_t nb_fail;
49	} free;
50	uint64_t nb_alloc_obj;
51};
52
53struct memtank_stat {
54	uint64_t nb_cycle;
55	struct {
56		uint64_t nb_call;
57		uint64_t nb_req;
58		uint64_t nb_alloc;
59		uint64_t nb_cycle;
60		uint64_t max_cycle;
61		uint64_t min_cycle;
62	} alloc;
63	struct {
64		uint64_t nb_call;
65		uint64_t nb_free;
66		uint64_t nb_cycle;
67		uint64_t max_cycle;
68		uint64_t min_cycle;
69	} free;
70	struct {
71		uint64_t nb_call;
72		uint64_t nb_chunk;
73		uint64_t nb_cycle;
74		uint64_t max_cycle;
75		uint64_t min_cycle;
76	} grow;
77	struct {
78		uint64_t nb_call;
79		uint64_t nb_chunk;
80		uint64_t nb_cycle;
81		uint64_t max_cycle;
82		uint64_t min_cycle;
83	} shrink;
84};
85
86struct master_args {
87	uint64_t run_cycles;
88	uint32_t delay_us;
89	uint32_t flags;
90};
91
92struct worker_args {
93	uint32_t max_obj;
94	uint32_t obj_size;
95	uint32_t alloc_flags;
96	uint32_t free_flags;
97	struct rte_ring *rng;
98};
99
100struct memtank_arg {
101	struct tle_memtank *mt;
102	union {
103		struct master_args master;
104		struct worker_args worker;
105	};
106	struct memtank_stat stats;
107} __rte_cache_aligned;
108
109#define BULK_NUM	32
110
111#define	OBJ_SZ_MIN	1
112#define	OBJ_SZ_MAX	0x100000
113#define	OBJ_SZ_DEF	(4 * RTE_CACHE_LINE_SIZE + 1)
114
115#define TEST_TIME	10
116#define CLEANUP_TIME	3
117
118#define FREE_THRSH_MIN	0
119#define FREE_THRSH_MAX	100
120
121enum {
122	WRK_CMD_STOP,
123	WRK_CMD_RUN,
124};
125
126enum {
127	MASTER_FLAG_GROW = 1,
128	MASTER_FLAG_SHRINK = 2,
129};
130
131enum {
132	MEM_FUNC_SYS,
133	MEM_FUNC_RTE,
134};
135
136static uint32_t wrk_cmd __rte_cache_aligned;
137
138static struct tle_memtank_prm mtnk_prm = {
139	.min_free = 4 * BULK_NUM,
140	.max_free = 32 * BULK_NUM,
141	.obj_size = OBJ_SZ_DEF,
142	.obj_align = RTE_CACHE_LINE_SIZE,
143	.nb_obj_chunk = BULK_NUM,
144	.flags = TLE_MTANK_OBJ_DBG,
145};
146
147static struct {
148	uint32_t run_time;       /* test run-time in seconds */
149	uint32_t wrk_max_obj;    /* max alloced objects per worker */
150	uint32_t wrk_free_thrsh; /* wrk free thresh % (0-100) */
151	int32_t mem_func;        /* memory subsystem to use for alloc/free */
152} global_cfg = {
153	.run_time = TEST_TIME,
154	.wrk_max_obj = 2 * BULK_NUM,
155	.wrk_free_thrsh = FREE_THRSH_MIN,
156	.mem_func = MEM_FUNC_SYS,
157};
158
159static void *
160alloc_func(size_t sz)
161{
162	switch (global_cfg.mem_func) {
163	case MEM_FUNC_SYS:
164		return malloc(sz);
165	case MEM_FUNC_RTE:
166		return rte_malloc(NULL, sz, 0);
167	}
168
169	return NULL;
170}
171
172static void
173free_func(void *p)
174{
175	switch (global_cfg.mem_func) {
176	case MEM_FUNC_SYS:
177		return free(p);
178	case MEM_FUNC_RTE:
179		return rte_free(p);
180	}
181}
182
183static void *
184test_alloc1(size_t sz, void *p)
185{
186	struct memstat *ms;
187	void *buf;
188
189	ms = p;
190	buf = alloc_func(sz);
191	rte_atomic64_inc(&ms->alloc.nb_call);
192	if (buf != NULL) {
193		memset(buf, 0, sz);
194		rte_atomic64_add(&ms->alloc.sz, sz);
195	} else
196		rte_atomic64_inc(&ms->alloc.nb_fail);
197
198	return buf;
199}
200
201static void
202test_free1(void *buf, void *p)
203{
204	struct memstat *ms;
205
206	ms = p;
207
208	free_func(buf);
209	rte_atomic64_inc(&ms->free.nb_call);
210	if (buf == NULL)
211		rte_atomic64_inc(&ms->free.nb_fail);
212}
213
214static void
215memstat_dump(FILE *f, struct memstat *ms)
216{
217
218	uint64_t alloc_sz, nb_alloc;
219	long double muc, mut;
220
221	nb_alloc = rte_atomic64_read(&ms->alloc.nb_call) -
222		rte_atomic64_read(&ms->alloc.nb_fail);
223	alloc_sz = rte_atomic64_read(&ms->alloc.sz) / nb_alloc;
224	nb_alloc -= rte_atomic64_read(&ms->free.nb_call) -
225		rte_atomic64_read(&ms->free.nb_fail);
226	alloc_sz *= nb_alloc;
227	mut = (alloc_sz == 0) ? 1 :
228		(long double)ms->nb_alloc_obj * mtnk_prm.obj_size / alloc_sz;
229	muc = (alloc_sz == 0) ? 1 :
230		(long double)(ms->nb_alloc_obj + mtnk_prm.max_free) *
231		mtnk_prm.obj_size / alloc_sz;
232
233	fprintf(f, "%s(%p)={\n", __func__, ms);
234	fprintf(f, "\talloc={\n");
235	fprintf(f, "\t\tnb_call=%" PRIu64 ",\n",
236		rte_atomic64_read(&ms->alloc.nb_call));
237	fprintf(f, "\t\tnb_fail=%" PRIu64 ",\n",
238		rte_atomic64_read(&ms->alloc.nb_fail));
239	fprintf(f, "\t\tsz=%" PRIu64 ",\n",
240		rte_atomic64_read(&ms->alloc.sz));
241	fprintf(f, "\t},\n");
242	fprintf(f, "\tfree={\n");
243	fprintf(f, "\t\tnb_call=%" PRIu64 ",\n",
244		rte_atomic64_read(&ms->free.nb_call));
245	fprintf(f, "\t\tnb_fail=%" PRIu64 ",\n",
246		rte_atomic64_read(&ms->free.nb_fail));
247	fprintf(f, "\t},\n");
248	fprintf(f, "\tnb_alloc_obj=%" PRIu64 ",\n", ms->nb_alloc_obj);
249	fprintf(f, "\tnb_alloc_chunk=%" PRIu64 ",\n", nb_alloc);
250	fprintf(f, "\talloc_sz=%" PRIu64 ",\n", alloc_sz);
251	fprintf(f, "\tmem_util(total)=%.2Lf %%,\n", mut * 100);
252	fprintf(f, "\tmem_util(cached)=%.2Lf %%,\n", muc * 100);
253	fprintf(f, "};\n");
254
255}
256
257static void
258memtank_stat_reset(struct memtank_stat *ms)
259{
260	static const struct memtank_stat init_stat = {
261		.alloc.min_cycle = UINT64_MAX,
262		.free.min_cycle = UINT64_MAX,
263		.grow.min_cycle = UINT64_MAX,
264		.shrink.min_cycle = UINT64_MAX,
265	};
266
267	*ms = init_stat;
268}
269
270static void
271memtank_stat_aggr(struct memtank_stat *as, const struct memtank_stat *ms)
272{
273	if (ms->alloc.nb_call != 0) {
274		as->alloc.nb_call += ms->alloc.nb_call;
275		as->alloc.nb_req += ms->alloc.nb_req;
276		as->alloc.nb_alloc += ms->alloc.nb_alloc;
277		as->alloc.nb_cycle += ms->alloc.nb_cycle;
278		as->alloc.max_cycle = RTE_MAX(as->alloc.max_cycle,
279					ms->alloc.max_cycle);
280		as->alloc.min_cycle = RTE_MIN(as->alloc.min_cycle,
281					ms->alloc.min_cycle);
282	}
283	if (ms->free.nb_call != 0) {
284		as->free.nb_call += ms->free.nb_call;
285		as->free.nb_free += ms->free.nb_free;
286		as->free.nb_cycle += ms->free.nb_cycle;
287		as->free.max_cycle = RTE_MAX(as->free.max_cycle,
288					ms->free.max_cycle);
289		as->free.min_cycle = RTE_MIN(as->free.min_cycle,
290					ms->free.min_cycle);
291	}
292	if (ms->grow.nb_call != 0) {
293		as->grow.nb_call += ms->grow.nb_call;
294		as->grow.nb_chunk += ms->grow.nb_chunk;
295		as->grow.nb_cycle += ms->grow.nb_cycle;
296		as->grow.max_cycle = RTE_MAX(as->grow.max_cycle,
297					ms->grow.max_cycle);
298		as->grow.min_cycle = RTE_MIN(as->grow.min_cycle,
299					ms->grow.min_cycle);
300	}
301	if (ms->shrink.nb_call != 0) {
302		as->shrink.nb_call += ms->shrink.nb_call;
303		as->shrink.nb_chunk += ms->shrink.nb_chunk;
304		as->shrink.nb_cycle += ms->shrink.nb_cycle;
305		as->shrink.max_cycle = RTE_MAX(as->shrink.max_cycle,
306					ms->shrink.max_cycle);
307		as->shrink.min_cycle = RTE_MIN(as->shrink.min_cycle,
308					ms->shrink.min_cycle);
309	}
310}
311
312static void
313memtank_stat_dump(FILE *f, uint32_t lc, const struct memtank_stat *ms)
314{
315	uint64_t t;
316	long double st;
317
318	 st = (long double)rte_get_timer_hz() / US_PER_S;
319
320	if (lc == UINT32_MAX)
321		fprintf(f, "%s(AGGREGATE)={\n", __func__);
322	else
323		fprintf(f, "%s(lc=%u)={\n", __func__, lc);
324
325	fprintf(f, "\tnb_cycle=%" PRIu64 ",\n", ms->nb_cycle);
326	if (ms->alloc.nb_call != 0) {
327		fprintf(f, "\talloc={\n");
328		fprintf(f, "\t\tnb_call=%" PRIu64 ",\n", ms->alloc.nb_call);
329		fprintf(f, "\t\tnb_req=%" PRIu64 ",\n", ms->alloc.nb_req);
330		fprintf(f, "\t\tnb_alloc=%" PRIu64 ",\n", ms->alloc.nb_alloc);
331		fprintf(f, "\t\tnb_cycle=%" PRIu64 ",\n", ms->alloc.nb_cycle);
332
333		t = ms->alloc.nb_req - ms->alloc.nb_alloc;
334		fprintf(f, "\t\tfailed req: %"PRIu64 "(%.2Lf %%)\n",
335			t, (long double)t * 100 /  ms->alloc.nb_req);
336		fprintf(f, "\t\tcycles/alloc: %.2Lf\n",
337			(long double)ms->alloc.nb_cycle / ms->alloc.nb_alloc);
338		fprintf(f, "\t\tobj/call(avg): %.2Lf\n",
339			(long double)ms->alloc.nb_alloc /  ms->alloc.nb_call);
340
341		fprintf(f, "\t\tmax cycles/call=%" PRIu64 "(%.2Lf usec),\n",
342			ms->alloc.max_cycle,
343			(long double)ms->alloc.max_cycle / st);
344		fprintf(f, "\t\tmin cycles/call=%" PRIu64 "(%.2Lf usec),\n",
345			ms->alloc.min_cycle,
346			(long double)ms->alloc.min_cycle / st);
347
348		fprintf(f, "\t},\n");
349	}
350	if (ms->free.nb_call != 0) {
351		fprintf(f, "\tfree={\n");
352		fprintf(f, "\t\tnb_call=%" PRIu64 ",\n", ms->free.nb_call);
353		fprintf(f, "\t\tnb_free=%" PRIu64 ",\n", ms->free.nb_free);
354		fprintf(f, "\t\tnb_cycle=%" PRIu64 ",\n", ms->free.nb_cycle);
355
356		fprintf(f, "\t\tcycles/free: %.2Lf\n",
357			(long double)ms->free.nb_cycle / ms->free.nb_free);
358		fprintf(f, "\t\tobj/call(avg): %.2Lf\n",
359			(long double)ms->free.nb_free /  ms->free.nb_call);
360
361		fprintf(f, "\t\tmax cycles/call=%" PRIu64 "(%.2Lf usec),\n",
362			ms->free.max_cycle,
363			(long double)ms->free.max_cycle / st);
364		fprintf(f, "\t\tmin cycles/call=%" PRIu64 "(%.2Lf usec),\n",
365			ms->free.min_cycle,
366			(long double)ms->free.min_cycle / st);
367
368		fprintf(f, "\t},\n");
369	}
370	if (ms->grow.nb_call != 0) {
371		fprintf(f, "\tgrow={\n");
372		fprintf(f, "\t\tnb_call=%" PRIu64 ",\n", ms->grow.nb_call);
373		fprintf(f, "\t\tnb_chunk=%" PRIu64 ",\n", ms->grow.nb_chunk);
374		fprintf(f, "\t\tnb_cycle=%" PRIu64 ",\n", ms->grow.nb_cycle);
375
376		fprintf(f, "\t\tcycles/chunk: %.2Lf\n",
377			(long double)ms->grow.nb_cycle / ms->grow.nb_chunk);
378		fprintf(f, "\t\tobj/call(avg): %.2Lf\n",
379			(long double)ms->grow.nb_chunk /  ms->grow.nb_call);
380
381		fprintf(f, "\t\tmax cycles/call=%" PRIu64 "(%.2Lf usec),\n",
382			ms->grow.max_cycle,
383			(long double)ms->grow.max_cycle / st);
384		fprintf(f, "\t\tmin cycles/call=%" PRIu64 "(%.2Lf usec),\n",
385			ms->grow.min_cycle,
386			(long double)ms->grow.min_cycle / st);
387
388		fprintf(f, "\t},\n");
389	}
390	if (ms->shrink.nb_call != 0) {
391		fprintf(f, "\tshrink={\n");
392		fprintf(f, "\t\tnb_call=%" PRIu64 ",\n", ms->shrink.nb_call);
393		fprintf(f, "\t\tnb_chunk=%" PRIu64 ",\n", ms->shrink.nb_chunk);
394		fprintf(f, "\t\tnb_cycle=%" PRIu64 ",\n", ms->shrink.nb_cycle);
395
396		fprintf(f, "\t\tcycles/chunk: %.2Lf\n",
397			(long double)ms->shrink.nb_cycle / ms->shrink.nb_chunk);
398		fprintf(f, "\t\tobj/call(avg): %.2Lf\n",
399			(long double)ms->shrink.nb_chunk /  ms->shrink.nb_call);
400
401		fprintf(f, "\t\tmax cycles/call=%" PRIu64 "(%.2Lf usec),\n",
402			ms->shrink.max_cycle,
403			(long double)ms->shrink.max_cycle / st);
404		fprintf(f, "\t\tmin cycles/call=%" PRIu64 "(%.2Lf usec),\n",
405			ms->shrink.min_cycle,
406			(long double)ms->shrink.min_cycle / st);
407
408		fprintf(f, "\t},\n");
409	}
410	fprintf(f, "};\n");
411}
412
413static int32_t
414check_fill_objs(void *obj[], uint32_t sz, uint32_t num,
415	uint8_t check, uint8_t fill)
416{
417	uint32_t i;
418	uint8_t buf[sz];
419
420	static rte_spinlock_t dump_lock;
421
422	memset(buf, check, sz);
423
424	for (i = 0; i != num; i++) {
425		if (memcmp(buf, obj[i], sz) != 0) {
426			rte_spinlock_lock(&dump_lock);
427			printf ("%s(%u, %u, %hu, %hu) failed at %u-th iter, "
428				"offendig object: %p\n",
429				__func__, sz, num, check, fill, i, obj[i]);
430			rte_memdump(stdout, "expected", buf, sz);
431			rte_memdump(stdout, "result", obj[i], sz);
432			rte_spinlock_unlock(&dump_lock);
433			return -EINVAL;
434		}
435		memset(obj[i], fill, sz);
436	}
437	return 0;
438}
439
440static int
441create_worker_ring(struct worker_args *wa, uint32_t lc)
442{
443	int32_t rc;
444	size_t sz;
445	struct rte_ring *ring;
446
447	sz = rte_ring_get_memsize(wa->max_obj);
448	ring = malloc(sz);
449	if (ring == NULL) {
450		printf("%s(%u): alloca(%zu) for FIFO with %u elems failed",
451			__func__, lc, sz, wa->max_obj);
452		return -ENOMEM;
453	}
454	rc = rte_ring_init(ring, "", wa->max_obj,
455		RING_F_SP_ENQ | RING_F_SC_DEQ);
456	if (rc != 0) {
457		printf("%s(%u): rte_ring_init(%p, %u) failed, error: %d(%s)\n",
458			__func__, lc, ring, wa->max_obj,
459			rc, strerror(-rc));
460		free(ring);
461		return rc;
462	}
463
464	wa->rng = ring;
465	return rc;
466}
467
468static int
469test_worker_cleanup(void *arg)
470{
471	void *obj[BULK_NUM];
472	int32_t rc;
473	uint32_t lc, n, num;
474	struct memtank_arg *ma;
475	struct rte_ring *ring;
476
477	ma = arg;
478	ring = ma->worker.rng;
479	lc = rte_lcore_id();
480
481	rc = 0;
482	for (n = rte_ring_count(ring); rc == 0 && n != 0; n -= num) {
483
484		num = rte_rand() % RTE_DIM(obj);
485		num = RTE_MIN(num, n);
486
487		if (num != 0) {
488			/* retrieve objects to free */
489			rte_ring_dequeue_bulk(ring, obj, num, NULL);
490
491			/* check and fill contents of freeing objects */
492			rc = check_fill_objs(obj, ma->worker.obj_size, num,
493				lc, 0);
494			if (rc == 0) {
495				tle_memtank_free(ma->mt, obj, num,
496					ma->worker.free_flags);
497				ma->stats.free.nb_free += num;
498			}
499		}
500	}
501
502	return rc;
503}
504
505static int
506test_memtank_worker(void *arg)
507{
508	int32_t rc;
509	uint32_t ft, lc, n, num;
510	uint64_t cl, tm0, tm1;
511	struct memtank_arg *ma;
512	struct rte_ring *ring;
513	void *obj[BULK_NUM];
514
515	ma = arg;
516	lc = rte_lcore_id();
517
518	/* calculate free threshold */
519	ft = ma->worker.max_obj * global_cfg.wrk_free_thrsh / FREE_THRSH_MAX;
520	ring = ma->worker.rng;
521
522	while (wrk_cmd != WRK_CMD_RUN) {
523		rte_smp_rmb();
524		rte_pause();
525	}
526
527	cl = rte_rdtsc_precise();
528
529	do {
530		num = rte_rand() % RTE_DIM(obj);
531		n = rte_ring_free_count(ring);
532		num = RTE_MIN(num, n);
533
534		/* perform alloc*/
535		if (num != 0) {
536			tm0 = rte_rdtsc_precise();
537			n = tle_memtank_alloc(ma->mt, obj, num,
538				ma->worker.alloc_flags);
539			tm1 = rte_rdtsc_precise();
540
541			/* check and fill contents of allocated objects */
542			rc = check_fill_objs(obj, ma->worker.obj_size, n,
543				0, lc);
544			if (rc != 0)
545				break;
546
547			tm1 = tm1 - tm0;
548
549			/* collect alloc stat */
550			ma->stats.alloc.nb_call++;
551			ma->stats.alloc.nb_req += num;
552			ma->stats.alloc.nb_alloc += n;
553			ma->stats.alloc.nb_cycle += tm1;
554			ma->stats.alloc.max_cycle =
555				RTE_MAX(ma->stats.alloc.max_cycle, tm1);
556			ma->stats.alloc.min_cycle =
557				RTE_MIN(ma->stats.alloc.min_cycle, tm1);
558
559			/* store allocated objects */
560			rte_ring_enqueue_bulk(ring, obj, n, NULL);
561		}
562
563		/* get some objects to free */
564		num = rte_rand() % RTE_DIM(obj);
565		n = rte_ring_count(ring);
566		num = (n >= ft) ? RTE_MIN(num, n) : 0;
567
568		/* perform free*/
569		if (num != 0) {
570
571			/* retrieve objects to free */
572			rte_ring_dequeue_bulk(ring, obj, num, NULL);
573
574			/* check and fill contents of freeing objects */
575			rc = check_fill_objs(obj, ma->worker.obj_size, num,
576				lc, 0);
577			if (rc != 0)
578				break;
579
580			tm0 = rte_rdtsc_precise();
581			tle_memtank_free(ma->mt, obj, num,
582				ma->worker.free_flags);
583			tm1 = rte_rdtsc_precise();
584
585			tm1 = tm1 - tm0;
586
587			/* collect free stat */
588			ma->stats.free.nb_call++;
589			ma->stats.free.nb_free += num;
590			ma->stats.free.nb_cycle += tm1;
591			ma->stats.free.max_cycle =
592				RTE_MAX(ma->stats.free.max_cycle, tm1);
593			ma->stats.free.min_cycle =
594				RTE_MIN(ma->stats.free.min_cycle, tm1);
595		}
596
597		rte_smp_mb();
598	} while (wrk_cmd == WRK_CMD_RUN);
599
600	ma->stats.nb_cycle = rte_rdtsc_precise() - cl;
601
602	return rc;
603}
604
605static int
606test_memtank_master(void *arg)
607{
608	struct memtank_arg *ma;
609	uint64_t cl, tm0, tm1, tm2;
610	uint32_t i, n;
611
612	ma = (struct memtank_arg *)arg;
613
614	for (cl = 0, i = 0; cl < ma->master.run_cycles;
615			cl += tm2 - tm0, i++)  {
616
617		tm0 = rte_rdtsc_precise();
618
619		if (ma->master.flags & MASTER_FLAG_SHRINK) {
620
621			n = tle_memtank_shrink(ma->mt);
622			tm1 = rte_rdtsc_precise();
623			ma->stats.shrink.nb_call++;
624			ma->stats.shrink.nb_chunk += n;
625			tm1 = tm1 - tm0;
626
627			if (n != 0) {
628				ma->stats.shrink.nb_cycle += tm1;
629				ma->stats.shrink.max_cycle =
630					RTE_MAX(ma->stats.shrink.max_cycle,
631						tm1);
632				ma->stats.shrink.min_cycle =
633					RTE_MIN(ma->stats.shrink.min_cycle,
634						tm1);
635			}
636		}
637
638		if (ma->master.flags & MASTER_FLAG_GROW) {
639
640			tm1 = rte_rdtsc_precise();
641			n = tle_memtank_grow(ma->mt);
642			tm2 = rte_rdtsc_precise();
643			ma->stats.grow.nb_call++;
644			ma->stats.grow.nb_chunk += n;
645			tm2 = tm2 - tm1;
646
647			if (n != 0) {
648				ma->stats.grow.nb_cycle += tm2;
649				ma->stats.grow.max_cycle =
650					RTE_MAX(ma->stats.grow.max_cycle,
651						tm2);
652				ma->stats.grow.min_cycle =
653					RTE_MIN(ma->stats.grow.min_cycle,
654						tm2);
655			}
656		}
657
658		wrk_cmd = WRK_CMD_RUN;
659		rte_smp_mb();
660
661		rte_delay_us(ma->master.delay_us);
662		tm2 = rte_rdtsc_precise();
663	}
664
665	ma->stats.nb_cycle = cl;
666
667	rte_smp_mb();
668	wrk_cmd = WRK_CMD_STOP;
669
670	return 0;
671}
672
673static int
674fill_worker_args(struct worker_args *wa, uint32_t alloc_flags,
675	uint32_t free_flags, uint32_t lc)
676{
677	wa->max_obj = global_cfg.wrk_max_obj;
678	wa->obj_size = mtnk_prm.obj_size;
679	wa->alloc_flags = alloc_flags;
680	wa->free_flags = free_flags;
681
682	return create_worker_ring(wa, lc);
683}
684
685static void
686fill_master_args(struct master_args *ma, uint32_t flags)
687{
688	uint64_t tm;
689
690	tm = global_cfg.run_time * rte_get_timer_hz();
691
692	ma->run_cycles = tm;
693	ma->delay_us = US_PER_S / MS_PER_S;
694	ma->flags = flags;
695}
696
697static int
698test_memtank_cleanup(struct tle_memtank *mt, struct memstat *ms,
699	struct memtank_arg arg[], const char *tname)
700{
701	int32_t rc;
702	uint32_t lc;
703
704	printf("%s(%s)\n", __func__, tname);
705
706	RTE_LCORE_FOREACH_SLAVE(lc)
707		rte_eal_remote_launch(test_worker_cleanup, &arg[lc], lc);
708
709	/* launch on master */
710	lc = rte_lcore_id();
711	arg[lc].master.run_cycles = CLEANUP_TIME * rte_get_timer_hz();
712	test_memtank_master(&arg[lc]);
713
714	ms->nb_alloc_obj = 0;
715	RTE_LCORE_FOREACH_SLAVE(lc) {
716		rc |= rte_eal_wait_lcore(lc);
717		ms->nb_alloc_obj += arg[lc].stats.alloc.nb_alloc -
718			arg[lc].stats.free.nb_free;
719	}
720
721	tle_memtank_dump(stdout, mt, TLE_MTANK_DUMP_STAT);
722
723	memstat_dump(stdout, ms);
724	rc = tle_memtank_sanity_check(mt, 0);
725
726	return rc;
727}
728
729/*
730 * alloc/free by workers threads.
731 * grow/shrink by master
732 */
733static int
734test_memtank_mt(const char *tname, uint32_t alloc_flags, uint32_t free_flags)
735{
736	int32_t rc;
737	uint32_t lc;
738	struct tle_memtank *mt;
739	struct tle_memtank_prm prm;
740	struct memstat ms;
741	struct memtank_stat wrk_stats;
742	struct memtank_arg arg[RTE_MAX_LCORE];
743
744	printf("%s(%s) start\n", __func__, tname);
745
746	memset(&prm, 0, sizeof(prm));
747	memset(&ms, 0, sizeof(ms));
748
749	prm = mtnk_prm;
750	prm.alloc = test_alloc1;
751	prm.free = test_free1;
752	prm.udata = &ms;
753
754	mt = tle_memtank_create(&prm);
755	if (mt == NULL) {
756		printf("%s(%s): memtank_create() failed\n", __func__, tname);
757		return -ENOMEM;
758	}
759
760	/* dump initial memory stats */
761	memstat_dump(stdout, &ms);
762
763	rc = 0;
764	memset(arg, 0, sizeof(arg));
765
766	/* prepare args on all slaves */
767	RTE_LCORE_FOREACH_SLAVE(lc) {
768		arg[lc].mt = mt;
769		rc = fill_worker_args(&arg[lc].worker, alloc_flags,
770			free_flags, lc);
771		if (rc != 0)
772			break;
773		memtank_stat_reset(&arg[lc].stats);
774	}
775
776	if (rc != 0) {
777		tle_memtank_destroy(mt);
778		return rc;
779	}
780
781	/* launch on all slaves */
782	RTE_LCORE_FOREACH_SLAVE(lc)
783		rte_eal_remote_launch(test_memtank_worker, &arg[lc], lc);
784
785	/* launch on master */
786	lc = rte_lcore_id();
787	arg[lc].mt = mt;
788	fill_master_args(&arg[lc].master,
789		MASTER_FLAG_GROW | MASTER_FLAG_SHRINK);
790	test_memtank_master(&arg[lc]);
791
792	/* wait for slaves and collect stats. */
793
794	memtank_stat_reset(&wrk_stats);
795
796	rc = 0;
797	RTE_LCORE_FOREACH_SLAVE(lc) {
798		rc |= rte_eal_wait_lcore(lc);
799		memtank_stat_dump(stdout, lc, &arg[lc].stats);
800		memtank_stat_aggr(&wrk_stats, &arg[lc].stats);
801		ms.nb_alloc_obj += arg[lc].stats.alloc.nb_alloc -
802			arg[lc].stats.free.nb_free;
803	}
804
805	memtank_stat_dump(stdout, UINT32_MAX, &wrk_stats);
806
807	lc = rte_lcore_id();
808	memtank_stat_dump(stdout, lc, &arg[lc].stats);
809	tle_memtank_dump(stdout, mt, TLE_MTANK_DUMP_STAT);
810
811	memstat_dump(stdout, &ms);
812	rc |= tle_memtank_sanity_check(mt, 0);
813
814	/* run cleanup on all slave cores */
815	if (rc == 0)
816		rc = test_memtank_cleanup(mt, &ms, arg, tname);
817
818	tle_memtank_destroy(mt);
819	return rc;
820}
821
822/*
823 * alloc/free by workers threads.
824 * grow/shrink by master
825 */
826static int
827test_memtank_mt1(const char *tname)
828{
829	return test_memtank_mt(tname, 0, 0);
830}
831
832/*
833 * alloc/free with grow/shrink by worker threads.
834 * master does nothing
835 */
836static int
837test_memtank_mt2(const char *tname)
838{
839	const uint32_t alloc_flags = TLE_MTANK_ALLOC_CHUNK |
840				TLE_MTANK_ALLOC_GROW;
841	const uint32_t free_flags = TLE_MTANK_FREE_SHRINK;
842
843	return test_memtank_mt(tname, alloc_flags, free_flags);
844}
845
846static int
847parse_uint_val(const char *str, uint32_t *val, uint32_t min, uint32_t max)
848{
849	unsigned long v;
850	char *end;
851
852	errno = 0;
853	v = strtoul(str, &end, 0);
854	if (errno != 0 || end[0] != 0 || v < min || v > max)
855		return -EINVAL;
856
857	val[0] = v;
858	return 0;
859}
860
861static int
862parse_mem_str(const char *str)
863{
864	uint32_t i;
865
866	static const struct {
867		const char *name;
868		int32_t val;
869	} name2val[] = {
870		{
871			.name = "sys",
872			.val = MEM_FUNC_SYS,
873		},
874		{
875			.name = "rte",
876			.val = MEM_FUNC_RTE,
877		},
878	};
879
880	for (i = 0; i != RTE_DIM(name2val); i++) {
881		if (strcmp(str, name2val[i].name) == 0)
882			return name2val[i].val;
883	}
884	return -EINVAL;
885}
886
887static int
888parse_opt(int argc, char * const argv[])
889{
890	int32_t opt, rc;
891	uint32_t v;
892
893	rc = 0;
894	optind = 0;
895	optarg = NULL;
896
897	while ((opt = getopt(argc, argv, "f:m:s:t:w:")) != EOF) {
898		switch (opt) {
899		case 'f':
900			rc = parse_uint_val(optarg, &v, FREE_THRSH_MIN,
901				FREE_THRSH_MAX);
902			if (rc == 0)
903				global_cfg.wrk_free_thrsh = v;
904			break;
905		case 'm':
906			rc = parse_mem_str(optarg);
907			if (rc >= 0)
908				global_cfg.mem_func = rc;
909			break;
910		case 's':
911			rc = parse_uint_val(optarg, &v, OBJ_SZ_MIN,
912				OBJ_SZ_MAX);
913			if (rc == 0)
914				mtnk_prm.obj_size = v;
915			break;
916		case 't':
917			rc = parse_uint_val(optarg, &v, 0, UINT32_MAX);
918			if (rc == 0)
919				global_cfg.run_time = v;
920			break;
921		case 'w':
922			rc = parse_uint_val(optarg, &v, 0, UINT32_MAX);
923			if (rc == 0)
924				global_cfg.wrk_max_obj = v;
925			break;
926		default:
927			rc = -EINVAL;
928		}
929	}
930
931	if (rc < 0)
932		printf("%s: invalid value: \"%s\" for option: \'%c\'\n",
933			__func__, optarg, opt);
934	return  rc;
935}
936
937int
938main(int argc, char * argv[])
939{
940	int32_t rc;
941	uint32_t i, k;
942
943	const struct {
944		const char *name;
945		int (*func)(const char *);
946	} tests[] = {
947		{
948			.name = "MT1-WRK_ALLOC_FREE-MST_GROW_SHRINK",
949			.func = test_memtank_mt1,
950		},
951		{
952			.name = "MT1-WRK_ALLOC+GROW_FREE+SHRINK",
953			.func = test_memtank_mt2,
954		},
955	};
956
957
958	rc = rte_eal_init(argc, argv);
959	if (rc < 0)
960		rte_exit(EXIT_FAILURE,
961			"%s: rte_eal_init failed with error code: %d\n",
962			__func__, rc);
963
964	rc = parse_opt(argc - rc, argv + rc);
965	if (rc < 0)
966		rte_exit(EXIT_FAILURE,
967			"%s: parse_op failed with error code: %d\n",
968			__func__, rc);
969
970	/* update global values based on provided user input */
971	mtnk_prm.max_obj = global_cfg.wrk_max_obj * rte_lcore_count();
972
973	for (i = 0, k = 0; i != RTE_DIM(tests); i++) {
974
975		printf("TEST %s START\n", tests[i].name);
976
977		rc = tests[i].func(tests[i].name);
978		k += (rc == 0);
979
980		if (rc != 0)
981			printf("TEST %s FAILED\n", tests[i].name);
982		else
983			printf("TEST %s OK\n", tests[i].name);
984	}
985
986	printf("Number of tests:\t%u\nSuccess:\t%u\nFailed:\t%u\n",
987		i, k, i - k);
988	return (k != i);
989}
990