1703faabfSKonstantin Ananyev/*
2703faabfSKonstantin Ananyev * Copyright (c) 2019  Intel Corporation.
3703faabfSKonstantin Ananyev * Licensed under the Apache License, Version 2.0 (the "License");
4703faabfSKonstantin Ananyev * you may not use this file except in compliance with the License.
5703faabfSKonstantin Ananyev * You may obtain a copy of the License at:
6703faabfSKonstantin Ananyev *
7703faabfSKonstantin Ananyev *     http://www.apache.org/licenses/LICENSE-2.0
8703faabfSKonstantin Ananyev *
9703faabfSKonstantin Ananyev * Unless required by applicable law or agreed to in writing, software
10703faabfSKonstantin Ananyev * distributed under the License is distributed on an "AS IS" BASIS,
11703faabfSKonstantin Ananyev * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12703faabfSKonstantin Ananyev * See the License for the specific language governing permissions and
13703faabfSKonstantin Ananyev * limitations under the License.
14703faabfSKonstantin Ananyev */
15703faabfSKonstantin Ananyev
16703faabfSKonstantin Ananyev#include "memtank.h"
17703faabfSKonstantin Ananyev#include <rte_errno.h>
18703faabfSKonstantin Ananyev
19703faabfSKonstantin Ananyev#define	ALIGN_MUL_CEIL(v, mul)	\
20703faabfSKonstantin Ananyev	((typeof(v))(((uint64_t)(v) + (mul) - 1) / (mul)))
21703faabfSKonstantin Ananyev
22703faabfSKonstantin Ananyev
23703faabfSKonstantin Ananyevstatic inline size_t
24703faabfSKonstantin Ananyevmemtank_meta_size(uint32_t nb_free)
25703faabfSKonstantin Ananyev{
26703faabfSKonstantin Ananyev	size_t sz;
27703faabfSKonstantin Ananyev	static const struct memtank *mt;
28703faabfSKonstantin Ananyev
29703faabfSKonstantin Ananyev	sz = sizeof(*mt) + nb_free * sizeof(mt->pub.free[0]);
30703faabfSKonstantin Ananyev	sz = RTE_ALIGN_CEIL(sz, alignof(*mt));
31703faabfSKonstantin Ananyev	return sz;
32703faabfSKonstantin Ananyev}
33703faabfSKonstantin Ananyev
34703faabfSKonstantin Ananyevstatic inline size_t
35703faabfSKonstantin Ananyevmemchunk_meta_size(uint32_t nb_obj)
36703faabfSKonstantin Ananyev{
37703faabfSKonstantin Ananyev	size_t sz;
38703faabfSKonstantin Ananyev	static const struct memchunk *ch;
39703faabfSKonstantin Ananyev
40703faabfSKonstantin Ananyev	sz = sizeof(*ch) +  nb_obj * sizeof(ch->free[0]);
41703faabfSKonstantin Ananyev	sz = RTE_ALIGN_CEIL(sz, alignof(*ch));
42703faabfSKonstantin Ananyev	return sz;
43703faabfSKonstantin Ananyev}
44703faabfSKonstantin Ananyev
45703faabfSKonstantin Ananyevstatic inline size_t
46703faabfSKonstantin Ananyevmemobj_size(uint32_t obj_size, uint32_t obj_align)
47703faabfSKonstantin Ananyev{
48703faabfSKonstantin Ananyev	size_t sz;
49703faabfSKonstantin Ananyev	static const struct memobj *obj;
50703faabfSKonstantin Ananyev
51703faabfSKonstantin Ananyev	sz = sizeof(*obj) + obj_size;
52703faabfSKonstantin Ananyev	sz = RTE_ALIGN_CEIL(sz, obj_align);
53703faabfSKonstantin Ananyev	return sz;
54703faabfSKonstantin Ananyev}
55703faabfSKonstantin Ananyev
56703faabfSKonstantin Ananyevstatic inline size_t
57703faabfSKonstantin Ananyevmemchunk_size(uint32_t nb_obj, uint32_t obj_size, uint32_t obj_align)
58703faabfSKonstantin Ananyev{
59703faabfSKonstantin Ananyev	size_t algn, sz;
60703faabfSKonstantin Ananyev	static const struct memchunk *ch;
61703faabfSKonstantin Ananyev
62703faabfSKonstantin Ananyev	algn = RTE_MAX(alignof(*ch), obj_align);
63703faabfSKonstantin Ananyev	sz = memchunk_meta_size(nb_obj);
64703faabfSKonstantin Ananyev	sz += nb_obj * memobj_size(obj_size, obj_align);
65703faabfSKonstantin Ananyev	sz = RTE_ALIGN_CEIL(sz + algn - 1, algn);
66703faabfSKonstantin Ananyev	return sz;
67703faabfSKonstantin Ananyev}
68703faabfSKonstantin Ananyev
69703faabfSKonstantin Ananyevstatic void
70703faabfSKonstantin Ananyevinit_chunk(struct memtank *mt, struct memchunk *ch)
71703faabfSKonstantin Ananyev{
72703faabfSKonstantin Ananyev	uint32_t i, n, sz;
73703faabfSKonstantin Ananyev	uintptr_t p;
74703faabfSKonstantin Ananyev	struct memobj *obj;
75703faabfSKonstantin Ananyev
76703faabfSKonstantin Ananyev	const struct memobj cobj = {
77703faabfSKonstantin Ananyev		.red_zone1 = RED_ZONE_V1,
78703faabfSKonstantin Ananyev		.chunk = ch,
79703faabfSKonstantin Ananyev		.red_zone2 = RED_ZONE_V2,
80703faabfSKonstantin Ananyev	};
81703faabfSKonstantin Ananyev
82703faabfSKonstantin Ananyev	n = mt->prm.nb_obj_chunk;
83703faabfSKonstantin Ananyev	sz = mt->obj_size;
84703faabfSKonstantin Ananyev
85703faabfSKonstantin Ananyev	/* get start of memobj array */
86703faabfSKonstantin Ananyev	p = (uintptr_t)ch + memchunk_meta_size(n);
87703faabfSKonstantin Ananyev	p = RTE_ALIGN_CEIL(p, mt->prm.obj_align);
88703faabfSKonstantin Ananyev
89703faabfSKonstantin Ananyev	for (i = 0; i != n; i++) {
90703faabfSKonstantin Ananyev		obj = obj_pub_full(p, sz);
91703faabfSKonstantin Ananyev		obj[0] = cobj;
92703faabfSKonstantin Ananyev		ch->free[i] = (void *)p;
93703faabfSKonstantin Ananyev		p += sz;
94703faabfSKonstantin Ananyev	}
95703faabfSKonstantin Ananyev
96703faabfSKonstantin Ananyev	ch->nb_total = n;
97703faabfSKonstantin Ananyev	ch->nb_free = n;
98703faabfSKonstantin Ananyev
99703faabfSKonstantin Ananyev	if (mt->prm.init != NULL)
100703faabfSKonstantin Ananyev		mt->prm.init(ch->free, n, mt->prm.udata);
101703faabfSKonstantin Ananyev}
102703faabfSKonstantin Ananyev
103703faabfSKonstantin Ananyevstatic void
104703faabfSKonstantin Ananyevput_chunk(struct memtank *mt, struct memchunk *ch, void * const obj[],
105703faabfSKonstantin Ananyev	uint32_t num)
106703faabfSKonstantin Ananyev{
107703faabfSKonstantin Ananyev	uint32_t k, n;
108703faabfSKonstantin Ananyev	struct mchunk_list *ls;
109703faabfSKonstantin Ananyev
110703faabfSKonstantin Ananyev	/* chunk should be in the *used* list */
111703faabfSKonstantin Ananyev	k = MC_USED;
112703faabfSKonstantin Ananyev	ls = &mt->chl[k];
113703faabfSKonstantin Ananyev	rte_spinlock_lock(&ls->lock);
114703faabfSKonstantin Ananyev
115703faabfSKonstantin Ananyev	n = ch->nb_free;
116703faabfSKonstantin Ananyev	RTE_ASSERT(n + num <= ch->nb_total);
117703faabfSKonstantin Ananyev
118703faabfSKonstantin Ananyev	_copy_objs(ch->free + n, obj, num);
119703faabfSKonstantin Ananyev	ch->nb_free = n + num;
120703faabfSKonstantin Ananyev
121703faabfSKonstantin Ananyev	/* chunk is full now */
122703faabfSKonstantin Ananyev	if (ch->nb_free == ch->nb_total) {
123703faabfSKonstantin Ananyev		TAILQ_REMOVE(&ls->chunk, ch, link);
124703faabfSKonstantin Ananyev		k = MC_FULL;
125703faabfSKonstantin Ananyev	/* chunk is not empty anymore, move it to the head */
126703faabfSKonstantin Ananyev	} else if (n == 0) {
127703faabfSKonstantin Ananyev		TAILQ_REMOVE(&ls->chunk, ch, link);
128703faabfSKonstantin Ananyev		TAILQ_INSERT_HEAD(&ls->chunk, ch, link);
129703faabfSKonstantin Ananyev	}
130703faabfSKonstantin Ananyev
131703faabfSKonstantin Ananyev	rte_spinlock_unlock(&ls->lock);
132703faabfSKonstantin Ananyev
133703faabfSKonstantin Ananyev	/* insert this chunk into the *full* list */
134703faabfSKonstantin Ananyev	if (k == MC_FULL) {
135703faabfSKonstantin Ananyev		ls = &mt->chl[k];
136703faabfSKonstantin Ananyev		rte_spinlock_lock(&ls->lock);
137703faabfSKonstantin Ananyev		TAILQ_INSERT_HEAD(&ls->chunk, ch, link);
138703faabfSKonstantin Ananyev		rte_spinlock_unlock(&ls->lock);
139703faabfSKonstantin Ananyev	}
140703faabfSKonstantin Ananyev}
141703faabfSKonstantin Ananyev
142703faabfSKonstantin Ananyevstatic uint32_t
143703faabfSKonstantin Ananyevshrink_chunk(struct memtank *mt, uint32_t num)
144703faabfSKonstantin Ananyev{
145703faabfSKonstantin Ananyev	uint32_t i, k;
146703faabfSKonstantin Ananyev	struct mchunk_list *ls;
147703faabfSKonstantin Ananyev	struct memchunk *ch[num];
148703faabfSKonstantin Ananyev
149703faabfSKonstantin Ananyev	ls = &mt->chl[MC_FULL];
150703faabfSKonstantin Ananyev	rte_spinlock_lock(&ls->lock);
151703faabfSKonstantin Ananyev
152703faabfSKonstantin Ananyev	for (k = 0; k != num; k++) {
153703faabfSKonstantin Ananyev		ch[k] = TAILQ_LAST(&ls->chunk, mchunk_head);
154703faabfSKonstantin Ananyev		if (ch[k] == NULL)
155703faabfSKonstantin Ananyev			break;
156703faabfSKonstantin Ananyev		TAILQ_REMOVE(&ls->chunk, ch[k], link);
157703faabfSKonstantin Ananyev	}
158703faabfSKonstantin Ananyev
159703faabfSKonstantin Ananyev	rte_spinlock_unlock(&ls->lock);
160703faabfSKonstantin Ananyev
161703faabfSKonstantin Ananyev	rte_atomic32_sub(&mt->nb_chunks, k);
162703faabfSKonstantin Ananyev
163703faabfSKonstantin Ananyev	for (i = 0; i != k; i++)
164703faabfSKonstantin Ananyev		mt->prm.free(ch[i]->raw, mt->prm.udata);
165703faabfSKonstantin Ananyev
166703faabfSKonstantin Ananyev	return k;
167703faabfSKonstantin Ananyev}
168703faabfSKonstantin Ananyev
169703faabfSKonstantin Ananyevstatic struct memchunk *
170703faabfSKonstantin Ananyevalloc_chunk(struct memtank *mt)
171703faabfSKonstantin Ananyev{
172703faabfSKonstantin Ananyev	void *p;
173703faabfSKonstantin Ananyev	struct memchunk *ch;
174703faabfSKonstantin Ananyev
175703faabfSKonstantin Ananyev	p = mt->prm.alloc(mt->chunk_size, mt->prm.udata);
176703faabfSKonstantin Ananyev	if (p == NULL)
177703faabfSKonstantin Ananyev		return NULL;
178703faabfSKonstantin Ananyev	ch = RTE_PTR_ALIGN_CEIL(p, alignof(*ch));
179703faabfSKonstantin Ananyev	ch->raw = p;
180703faabfSKonstantin Ananyev	return ch;
181703faabfSKonstantin Ananyev}
182703faabfSKonstantin Ananyev
183703faabfSKonstantin Ananyev/* Determine by how many chunks we can actually grow */
184703faabfSKonstantin Ananyevstatic inline uint32_t
185703faabfSKonstantin Ananyevgrow_num(struct memtank *mt, uint32_t num)
186703faabfSKonstantin Ananyev{
187703faabfSKonstantin Ananyev	uint32_t k, n, max;
188703faabfSKonstantin Ananyev
189703faabfSKonstantin Ananyev	max = mt->max_chunk;
190703faabfSKonstantin Ananyev	n = rte_atomic32_add_return(&mt->nb_chunks, num);
191703faabfSKonstantin Ananyev
192703faabfSKonstantin Ananyev	if (n <= max)
193703faabfSKonstantin Ananyev		return num;
194703faabfSKonstantin Ananyev
195703faabfSKonstantin Ananyev	k = n - max;
196703faabfSKonstantin Ananyev	return (k >= num) ? 0 : num - k;
197703faabfSKonstantin Ananyev}
198703faabfSKonstantin Ananyev
199703faabfSKonstantin Ananyevstatic uint32_t
200703faabfSKonstantin Ananyevgrow_chunk(struct memtank *mt, uint32_t num)
201703faabfSKonstantin Ananyev{
202703faabfSKonstantin Ananyev	uint32_t i, k, n;
203703faabfSKonstantin Ananyev	struct mchunk_list *fls;
204703faabfSKonstantin Ananyev	struct mchunk_head ls;
205703faabfSKonstantin Ananyev	struct memchunk *ch[num];
206703faabfSKonstantin Ananyev
207703faabfSKonstantin Ananyev	/* check can we grow further */
208703faabfSKonstantin Ananyev	k = grow_num(mt, num);
209703faabfSKonstantin Ananyev
210703faabfSKonstantin Ananyev	for (n = 0; n != k; n++) {
211703faabfSKonstantin Ananyev		ch[n] = alloc_chunk(mt);
212703faabfSKonstantin Ananyev		if (ch[n] == NULL)
213703faabfSKonstantin Ananyev			break;
214703faabfSKonstantin Ananyev	}
215703faabfSKonstantin Ananyev
216703faabfSKonstantin Ananyev	TAILQ_INIT(&ls);
217703faabfSKonstantin Ananyev
218703faabfSKonstantin Ananyev	for (i = 0; i != n; i++) {
219703faabfSKonstantin Ananyev		init_chunk(mt, ch[i]);
220703faabfSKonstantin Ananyev		TAILQ_INSERT_HEAD(&ls, ch[i], link);
221703faabfSKonstantin Ananyev	}
222703faabfSKonstantin Ananyev
223703faabfSKonstantin Ananyev	if (n != 0) {
224703faabfSKonstantin Ananyev		fls = &mt->chl[MC_FULL];
225703faabfSKonstantin Ananyev		rte_spinlock_lock(&fls->lock);
226703faabfSKonstantin Ananyev		TAILQ_CONCAT(&fls->chunk, &ls, link);
227703faabfSKonstantin Ananyev		rte_spinlock_unlock(&fls->lock);