rte_mempool.h revision 9ca4a157
1/*-
2 *   BSD LICENSE
3 *
4 *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
5 *   Copyright(c) 2016 6WIND S.A.
6 *   All rights reserved.
7 *
8 *   Redistribution and use in source and binary forms, with or without
9 *   modification, are permitted provided that the following conditions
10 *   are met:
11 *
12 *     * Redistributions of source code must retain the above copyright
13 *       notice, this list of conditions and the following disclaimer.
14 *     * Redistributions in binary form must reproduce the above copyright
15 *       notice, this list of conditions and the following disclaimer in
16 *       the documentation and/or other materials provided with the
17 *       distribution.
18 *     * Neither the name of Intel Corporation nor the names of its
19 *       contributors may be used to endorse or promote products derived
20 *       from this software without specific prior written permission.
21 *
22 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 */
34
35#ifndef _RTE_MEMPOOL_H_
36#define _RTE_MEMPOOL_H_
37
38/**
39 * @file
40 * RTE Mempool.
41 *
42 * A memory pool is an allocator of fixed-size object. It is
43 * identified by its name, and uses a ring to store free objects. It
44 * provides some other optional services, like a per-core object
45 * cache, and an alignment helper to ensure that objects are padded
46 * to spread them equally on all RAM channels, ranks, and so on.
47 *
48 * Objects owned by a mempool should never be added in another
49 * mempool. When an object is freed using rte_mempool_put() or
50 * equivalent, the object data is not modified; the user can save some
51 * meta-data in the object data and retrieve them when allocating a
52 * new object.
53 *
54 * Note: the mempool implementation is not preemptable. A lcore must
55 * not be interrupted by another task that uses the same mempool
56 * (because it uses a ring which is not preemptable). Also, mempool
57 * functions must not be used outside the DPDK environment: for
58 * example, in linuxapp environment, a thread that is not created by
59 * the EAL must not use mempools. This is due to the per-lcore cache
60 * that won't work as rte_lcore_id() will not return a correct value.
61 */
62
63#include <stdio.h>
64#include <stdlib.h>
65#include <stdint.h>
66#include <errno.h>
67#include <inttypes.h>
68#include <sys/queue.h>
69
70#include <rte_spinlock.h>
71#include <rte_log.h>
72#include <rte_debug.h>
73#include <rte_lcore.h>
74#include <rte_memory.h>
75#include <rte_branch_prediction.h>
76#include <rte_ring.h>
77#include <rte_memcpy.h>
78#include <rte_common.h>
79
80#ifdef __cplusplus
81extern "C" {
82#endif
83
84#define RTE_MEMPOOL_HEADER_COOKIE1  0xbadbadbadadd2e55ULL /**< Header cookie. */
85#define RTE_MEMPOOL_HEADER_COOKIE2  0xf2eef2eedadd2e55ULL /**< Header cookie. */
86#define RTE_MEMPOOL_TRAILER_COOKIE  0xadd2e55badbadbadULL /**< Trailer cookie.*/
87
88#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
89/**
90 * A structure that stores the mempool statistics (per-lcore).
91 */
92struct rte_mempool_debug_stats {
93	uint64_t put_bulk;         /**< Number of puts. */
94	uint64_t put_objs;         /**< Number of objects successfully put. */
95	uint64_t get_success_bulk; /**< Successful allocation number. */
96	uint64_t get_success_objs; /**< Objects successfully allocated. */
97	uint64_t get_fail_bulk;    /**< Failed allocation number. */
98	uint64_t get_fail_objs;    /**< Objects that failed to be allocated. */
99} __rte_cache_aligned;
100#endif
101
102/**
103 * A structure that stores a per-core object cache.
104 */
105struct rte_mempool_cache {
106	uint32_t size;	      /**< Size of the cache */
107	uint32_t flushthresh; /**< Threshold before we flush excess elements */
108	uint32_t len;	      /**< Current cache count */
109	/*
110	 * Cache is allocated to this size to allow it to overflow in certain
111	 * cases to avoid needless emptying of cache.
112	 */
113	void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 3]; /**< Cache objects */
114} __rte_cache_aligned;
115
116/**
117 * A structure that stores the size of mempool elements.
118 */
119struct rte_mempool_objsz {
120	uint32_t elt_size;     /**< Size of an element. */
121	uint32_t header_size;  /**< Size of header (before elt). */
122	uint32_t trailer_size; /**< Size of trailer (after elt). */
123	uint32_t total_size;
124	/**< Total size of an object (header + elt + trailer). */
125};
126
127/**< Maximum length of a memory pool's name. */
128#define RTE_MEMPOOL_NAMESIZE (RTE_RING_NAMESIZE - \
129			      sizeof(RTE_MEMPOOL_MZ_PREFIX) + 1)
130#define RTE_MEMPOOL_MZ_PREFIX "MP_"
131
132/* "MP_<name>" */
133#define	RTE_MEMPOOL_MZ_FORMAT	RTE_MEMPOOL_MZ_PREFIX "%s"
134
135#define	MEMPOOL_PG_SHIFT_MAX	(sizeof(uintptr_t) * CHAR_BIT - 1)
136
137/** Mempool over one chunk of physically continuous memory */
138#define	MEMPOOL_PG_NUM_DEFAULT	1
139
140#ifndef RTE_MEMPOOL_ALIGN
141#define RTE_MEMPOOL_ALIGN	RTE_CACHE_LINE_SIZE
142#endif
143
144#define RTE_MEMPOOL_ALIGN_MASK	(RTE_MEMPOOL_ALIGN - 1)
145
146/**
147 * Mempool object header structure
148 *
149 * Each object stored in mempools are prefixed by this header structure,
150 * it allows to retrieve the mempool pointer from the object and to
151 * iterate on all objects attached to a mempool. When debug is enabled,
152 * a cookie is also added in this structure preventing corruptions and
153 * double-frees.
154 */
155struct rte_mempool_objhdr {
156	STAILQ_ENTRY(rte_mempool_objhdr) next; /**< Next in list. */
157	struct rte_mempool *mp;          /**< The mempool owning the object. */
158	phys_addr_t physaddr;            /**< Physical address of the object. */
159#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
160	uint64_t cookie;                 /**< Debug cookie. */
161#endif
162};
163
164/**
165 * A list of object headers type
166 */
167STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr);
168
169#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
170
171/**
172 * Mempool object trailer structure
173 *
174 * In debug mode, each object stored in mempools are suffixed by this
175 * trailer structure containing a cookie preventing memory corruptions.
176 */
177struct rte_mempool_objtlr {
178	uint64_t cookie;                 /**< Debug cookie. */
179};
180
181#endif
182
183/**
184 * A list of memory where objects are stored
185 */
186STAILQ_HEAD(rte_mempool_memhdr_list, rte_mempool_memhdr);
187
188/**
189 * Callback used to free a memory chunk
190 */
191typedef void (rte_mempool_memchunk_free_cb_t)(struct rte_mempool_memhdr *memhdr,
192	void *opaque);
193
194/**
195 * Mempool objects memory header structure
196 *
197 * The memory chunks where objects are stored. Each chunk is virtually
198 * and physically contiguous.
199 */
200struct rte_mempool_memhdr {
201	STAILQ_ENTRY(rte_mempool_memhdr) next; /**< Next in list. */
202	struct rte_mempool *mp;  /**< The mempool owning the chunk */
203	void *addr;              /**< Virtual address of the chunk */
204	phys_addr_t phys_addr;   /**< Physical address of the chunk */
205	size_t len;              /**< length of the chunk */
206	rte_mempool_memchunk_free_cb_t *free_cb; /**< Free callback */
207	void *opaque;            /**< Argument passed to the free callback */
208};
209
210/**
211 * The RTE mempool structure.
212 */
213struct rte_mempool {
214	/*
215	 * Note: this field kept the RTE_MEMZONE_NAMESIZE size due to ABI
216	 * compatibility requirements, it could be changed to
217	 * RTE_MEMPOOL_NAMESIZE next time the ABI changes
218	 */
219	char name[RTE_MEMZONE_NAMESIZE]; /**< Name of mempool. */
220	RTE_STD_C11
221	union {
222		void *pool_data;         /**< Ring or pool to store objects. */
223		uint64_t pool_id;        /**< External mempool identifier. */
224	};
225	void *pool_config;               /**< optional args for ops alloc. */
226	const struct rte_memzone *mz;    /**< Memzone where pool is alloc'd. */
227	int flags;                       /**< Flags of the mempool. */
228	int socket_id;                   /**< Socket id passed at create. */
229	uint32_t size;                   /**< Max size of the mempool. */
230	uint32_t cache_size;
231	/**< Size of per-lcore default local cache. */
232
233	uint32_t elt_size;               /**< Size of an element. */
234	uint32_t header_size;            /**< Size of header (before elt). */
235	uint32_t trailer_size;           /**< Size of trailer (after elt). */
236
237	unsigned private_data_size;      /**< Size of private data. */
238	/**
239	 * Index into rte_mempool_ops_table array of mempool ops
240	 * structs, which contain callback function pointers.
241	 * We're using an index here rather than pointers to the callbacks
242	 * to facilitate any secondary processes that may want to use
243	 * this mempool.
244	 */
245	int32_t ops_index;
246
247	struct rte_mempool_cache *local_cache; /**< Per-lcore local cache */
248
249	uint32_t populated_size;         /**< Number of populated objects. */
250	struct rte_mempool_objhdr_list elt_list; /**< List of objects in pool */
251	uint32_t nb_mem_chunks;          /**< Number of memory chunks */
252	struct rte_mempool_memhdr_list mem_list; /**< List of memory chunks */
253
254#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
255	/** Per-lcore statistics. */
256	struct rte_mempool_debug_stats stats[RTE_MAX_LCORE];
257#endif
258}  __rte_cache_aligned;
259
260#define MEMPOOL_F_NO_SPREAD      0x0001 /**< Do not spread among memory channels. */
261#define MEMPOOL_F_NO_CACHE_ALIGN 0x0002 /**< Do not align objs on cache lines.*/
262#define MEMPOOL_F_SP_PUT         0x0004 /**< Default put is "single-producer".*/
263#define MEMPOOL_F_SC_GET         0x0008 /**< Default get is "single-consumer".*/
264#define MEMPOOL_F_POOL_CREATED   0x0010 /**< Internal: pool is created. */
265#define MEMPOOL_F_NO_PHYS_CONTIG 0x0020 /**< Don't need physically contiguous objs. */
266
267/**
268 * @internal When debug is enabled, store some statistics.
269 *
270 * @param mp
271 *   Pointer to the memory pool.
272 * @param name
273 *   Name of the statistics field to increment in the memory pool.
274 * @param n
275 *   Number to add to the object-oriented statistics.
276 */
277#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
278#define __MEMPOOL_STAT_ADD(mp, name, n) do {                    \
279		unsigned __lcore_id = rte_lcore_id();           \
280		if (__lcore_id < RTE_MAX_LCORE) {               \
281			mp->stats[__lcore_id].name##_objs += n;	\
282			mp->stats[__lcore_id].name##_bulk += 1;	\
283		}                                               \
284	} while(0)
285#else
286#define __MEMPOOL_STAT_ADD(mp, name, n) do {} while(0)
287#endif
288
289/**
290 * Calculate the size of the mempool header.
291 *
292 * @param mp
293 *   Pointer to the memory pool.
294 * @param cs
295 *   Size of the per-lcore cache.
296 */
297#define MEMPOOL_HEADER_SIZE(mp, cs) \
298	(sizeof(*(mp)) + (((cs) == 0) ? 0 : \
299	(sizeof(struct rte_mempool_cache) * RTE_MAX_LCORE)))
300
301/* return the header of a mempool object (internal) */
302static inline struct rte_mempool_objhdr *__mempool_get_header(void *obj)
303{
304	return (struct rte_mempool_objhdr *)RTE_PTR_SUB(obj,
305		sizeof(struct rte_mempool_objhdr));
306}
307
308/**
309 * Return a pointer to the mempool owning this object.
310 *
311 * @param obj
312 *   An object that is owned by a pool. If this is not the case,
313 *   the behavior is undefined.
314 * @return
315 *   A pointer to the mempool structure.
316 */
317static inline struct rte_mempool *rte_mempool_from_obj(void *obj)
318{
319	struct rte_mempool_objhdr *hdr = __mempool_get_header(obj);
320	return hdr->mp;
321}
322
323/* return the trailer of a mempool object (internal) */
324static inline struct rte_mempool_objtlr *__mempool_get_trailer(void *obj)
325{
326	struct rte_mempool *mp = rte_mempool_from_obj(obj);
327	return (struct rte_mempool_objtlr *)RTE_PTR_ADD(obj, mp->elt_size);
328}
329
330/**
331 * @internal Check and update cookies or panic.
332 *
333 * @param mp
334 *   Pointer to the memory pool.
335 * @param obj_table_const
336 *   Pointer to a table of void * pointers (objects).
337 * @param n
338 *   Index of object in object table.
339 * @param free
340 *   - 0: object is supposed to be allocated, mark it as free
341 *   - 1: object is supposed to be free, mark it as allocated
342 *   - 2: just check that cookie is valid (free or allocated)
343 */
344void rte_mempool_check_cookies(const struct rte_mempool *mp,
345	void * const *obj_table_const, unsigned n, int free);
346
347#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
348#define __mempool_check_cookies(mp, obj_table_const, n, free) \
349	rte_mempool_check_cookies(mp, obj_table_const, n, free)
350#else
351#define __mempool_check_cookies(mp, obj_table_const, n, free) do {} while(0)
352#endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
353
354#define RTE_MEMPOOL_OPS_NAMESIZE 32 /**< Max length of ops struct name. */
355
356/**
357 * Prototype for implementation specific data provisioning function.
358 *
359 * The function should provide the implementation specific memory for
360 * use by the other mempool ops functions in a given mempool ops struct.
361 * E.g. the default ops provides an instance of the rte_ring for this purpose.
362 * it will most likely point to a different type of data structure, and
363 * will be transparent to the application programmer.
364 * This function should set mp->pool_data.
365 */
366typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp);
367
368/**
369 * Free the opaque private data pointed to by mp->pool_data pointer.
370 */
371typedef void (*rte_mempool_free_t)(struct rte_mempool *mp);
372
373/**
374 * Enqueue an object into the external pool.
375 */
376typedef int (*rte_mempool_enqueue_t)(struct rte_mempool *mp,
377		void * const *obj_table, unsigned int n);
378
379/**
380 * Dequeue an object from the external pool.
381 */
382typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp,
383		void **obj_table, unsigned int n);
384
385/**
386 * Return the number of available objects in the external pool.
387 */
388typedef unsigned (*rte_mempool_get_count)(const struct rte_mempool *mp);
389
390/** Structure defining mempool operations structure */
391struct rte_mempool_ops {
392	char name[RTE_MEMPOOL_OPS_NAMESIZE]; /**< Name of mempool ops struct. */
393	rte_mempool_alloc_t alloc;       /**< Allocate private data. */
394	rte_mempool_free_t free;         /**< Free the external pool. */
395	rte_mempool_enqueue_t enqueue;   /**< Enqueue an object. */
396	rte_mempool_dequeue_t dequeue;   /**< Dequeue an object. */
397	rte_mempool_get_count get_count; /**< Get qty of available objs. */
398} __rte_cache_aligned;
399
400#define RTE_MEMPOOL_MAX_OPS_IDX 16  /**< Max registered ops structs */
401
402/**
403 * Structure storing the table of registered ops structs, each of which contain
404 * the function pointers for the mempool ops functions.
405 * Each process has its own storage for this ops struct array so that
406 * the mempools can be shared across primary and secondary processes.
407 * The indices used to access the array are valid across processes, whereas
408 * any function pointers stored directly in the mempool struct would not be.
409 * This results in us simply having "ops_index" in the mempool struct.
410 */
411struct rte_mempool_ops_table {
412	rte_spinlock_t sl;     /**< Spinlock for add/delete. */
413	uint32_t num_ops;      /**< Number of used ops structs in the table. */
414	/**
415	 * Storage for all possible ops structs.
416	 */
417	struct rte_mempool_ops ops[RTE_MEMPOOL_MAX_OPS_IDX];
418} __rte_cache_aligned;
419
420/** Array of registered ops structs. */
421extern struct rte_mempool_ops_table rte_mempool_ops_table;
422
423/**
424 * @internal Get the mempool ops struct from its index.
425 *
426 * @param ops_index
427 *   The index of the ops struct in the ops struct table. It must be a valid
428 *   index: (0 <= idx < num_ops).
429 * @return
430 *   The pointer to the ops struct in the table.
431 */
432static inline struct rte_mempool_ops *
433rte_mempool_get_ops(int ops_index)
434{
435	RTE_VERIFY((ops_index >= 0) && (ops_index < RTE_MEMPOOL_MAX_OPS_IDX));
436
437	return &rte_mempool_ops_table.ops[ops_index];
438}
439
440/**
441 * @internal Wrapper for mempool_ops alloc callback.
442 *
443 * @param mp
444 *   Pointer to the memory pool.
445 * @return
446 *   - 0: Success; successfully allocated mempool pool_data.
447 *   - <0: Error; code of alloc function.
448 */
449int
450rte_mempool_ops_alloc(struct rte_mempool *mp);
451
452/**
453 * @internal Wrapper for mempool_ops dequeue callback.
454 *
455 * @param mp
456 *   Pointer to the memory pool.
457 * @param obj_table
458 *   Pointer to a table of void * pointers (objects).
459 * @param n
460 *   Number of objects to get.
461 * @return
462 *   - 0: Success; got n objects.
463 *   - <0: Error; code of dequeue function.
464 */
465static inline int
466rte_mempool_ops_dequeue_bulk(struct rte_mempool *mp,
467		void **obj_table, unsigned n)
468{
469	struct rte_mempool_ops *ops;
470
471	ops = rte_mempool_get_ops(mp->ops_index);
472	return ops->dequeue(mp, obj_table, n);
473}
474
475/**
476 * @internal wrapper for mempool_ops enqueue callback.
477 *
478 * @param mp
479 *   Pointer to the memory pool.
480 * @param obj_table
481 *   Pointer to a table of void * pointers (objects).
482 * @param n
483 *   Number of objects to put.
484 * @return
485 *   - 0: Success; n objects supplied.
486 *   - <0: Error; code of enqueue function.
487 */
488static inline int
489rte_mempool_ops_enqueue_bulk(struct rte_mempool *mp, void * const *obj_table,
490		unsigned n)
491{
492	struct rte_mempool_ops *ops;
493
494	ops = rte_mempool_get_ops(mp->ops_index);
495	return ops->enqueue(mp, obj_table, n);
496}
497
498/**
499 * @internal wrapper for mempool_ops get_count callback.
500 *
501 * @param mp
502 *   Pointer to the memory pool.
503 * @return
504 *   The number of available objects in the external pool.
505 */
506unsigned
507rte_mempool_ops_get_count(const struct rte_mempool *mp);
508
509/**
510 * @internal wrapper for mempool_ops free callback.
511 *
512 * @param mp
513 *   Pointer to the memory pool.
514 */
515void
516rte_mempool_ops_free(struct rte_mempool *mp);
517
518/**
519 * Set the ops of a mempool.
520 *
521 * This can only be done on a mempool that is not populated, i.e. just after
522 * a call to rte_mempool_create_empty().
523 *
524 * @param mp
525 *   Pointer to the memory pool.
526 * @param name
527 *   Name of the ops structure to use for this mempool.
528 * @param pool_config
529 *   Opaque data that can be passed by the application to the ops functions.
530 * @return
531 *   - 0: Success; the mempool is now using the requested ops functions.
532 *   - -EINVAL - Invalid ops struct name provided.
533 *   - -EEXIST - mempool already has an ops struct assigned.
534 */
535int
536rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name,
537		void *pool_config);
538
539/**
540 * Register mempool operations.
541 *
542 * @param ops
543 *   Pointer to an ops structure to register.
544 * @return
545 *   - >=0: Success; return the index of the ops struct in the table.
546 *   - -EINVAL - some missing callbacks while registering ops struct.
547 *   - -ENOSPC - the maximum number of ops structs has been reached.
548 */
549int rte_mempool_register_ops(const struct rte_mempool_ops *ops);
550
551/**
552 * Macro to statically register the ops of a mempool handler.
553 * Note that the rte_mempool_register_ops fails silently here when
554 * more than RTE_MEMPOOL_MAX_OPS_IDX is registered.
555 */
556#define MEMPOOL_REGISTER_OPS(ops)					\
557	void mp_hdlr_init_##ops(void);					\
558	void __attribute__((constructor, used)) mp_hdlr_init_##ops(void)\
559	{								\
560		rte_mempool_register_ops(&ops);			\
561	}
562
563/**
564 * An object callback function for mempool.
565 *
566 * Used by rte_mempool_create() and rte_mempool_obj_iter().
567 */
568typedef void (rte_mempool_obj_cb_t)(struct rte_mempool *mp,
569		void *opaque, void *obj, unsigned obj_idx);
570typedef rte_mempool_obj_cb_t rte_mempool_obj_ctor_t; /* compat */
571
572/**
573 * A memory callback function for mempool.
574 *
575 * Used by rte_mempool_mem_iter().
576 */
577typedef void (rte_mempool_mem_cb_t)(struct rte_mempool *mp,
578		void *opaque, struct rte_mempool_memhdr *memhdr,
579		unsigned mem_idx);
580
581/**
582 * A mempool constructor callback function.
583 *
584 * Arguments are the mempool and the opaque pointer given by the user in
585 * rte_mempool_create().
586 */
587typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *);
588
589/**
590 * Create a new mempool named *name* in memory.
591 *
592 * This function uses ``rte_memzone_reserve()`` to allocate memory. The
593 * pool contains n elements of elt_size. Its size is set to n.
594 *
595 * @param name
596 *   The name of the mempool.
597 * @param n
598 *   The number of elements in the mempool. The optimum size (in terms of
599 *   memory usage) for a mempool is when n is a power of two minus one:
600 *   n = (2^q - 1).
601 * @param elt_size
602 *   The size of each element.
603 * @param cache_size
604 *   If cache_size is non-zero, the rte_mempool library will try to
605 *   limit the accesses to the common lockless pool, by maintaining a
606 *   per-lcore object cache. This argument must be lower or equal to
607 *   CONFIG_RTE_MEMPOOL_CACHE_MAX_SIZE and n / 1.5. It is advised to choose
608 *   cache_size to have "n modulo cache_size == 0": if this is
609 *   not the case, some elements will always stay in the pool and will
610 *   never be used. The access to the per-lcore table is of course
611 *   faster than the multi-producer/consumer pool. The cache can be
612 *   disabled if the cache_size argument is set to 0; it can be useful to
613 *   avoid losing objects in cache.
614 * @param private_data_size
615 *   The size of the private data appended after the mempool
616 *   structure. This is useful for storing some private data after the
617 *   mempool structure, as is done for rte_mbuf_pool for example.
618 * @param mp_init
619 *   A function pointer that is called for initialization of the pool,
620 *   before object initialization. The user can initialize the private
621 *   data in this function if needed. This parameter can be NULL if
622 *   not needed.
623 * @param mp_init_arg
624 *   An opaque pointer to data that can be used in the mempool
625 *   constructor function.
626 * @param obj_init
627 *   A function pointer that is called for each object at
628 *   initialization of the pool. The user can set some meta data in
629 *   objects if needed. This parameter can be NULL if not needed.
630 *   The obj_init() function takes the mempool pointer, the init_arg,
631 *   the object pointer and the object number as parameters.
632 * @param obj_init_arg
633 *   An opaque pointer to data that can be used as an argument for
634 *   each call to the object constructor function.
635 * @param socket_id
636 *   The *socket_id* argument is the socket identifier in the case of
637 *   NUMA. The value can be *SOCKET_ID_ANY* if there is no NUMA
638 *   constraint for the reserved zone.
639 * @param flags
640 *   The *flags* arguments is an OR of following flags:
641 *   - MEMPOOL_F_NO_SPREAD: By default, objects addresses are spread
642 *     between channels in RAM: the pool allocator will add padding
643 *     between objects depending on the hardware configuration. See
644 *     Memory alignment constraints for details. If this flag is set,
645 *     the allocator will just align them to a cache line.
646 *   - MEMPOOL_F_NO_CACHE_ALIGN: By default, the returned objects are
647 *     cache-aligned. This flag removes this constraint, and no
648 *     padding will be present between objects. This flag implies
649 *     MEMPOOL_F_NO_SPREAD.
650 *   - MEMPOOL_F_SP_PUT: If this flag is set, the default behavior
651 *     when using rte_mempool_put() or rte_mempool_put_bulk() is
652 *     "single-producer". Otherwise, it is "multi-producers".
653 *   - MEMPOOL_F_SC_GET: If this flag is set, the default behavior
654 *     when using rte_mempool_get() or rte_mempool_get_bulk() is
655 *     "single-consumer". Otherwise, it is "multi-consumers".
656 *   - MEMPOOL_F_NO_PHYS_CONTIG: If set, allocated objects won't
657 *     necessarilly be contiguous in physical memory.
658 * @return
659 *   The pointer to the new allocated mempool, on success. NULL on error
660 *   with rte_errno set appropriately. Possible rte_errno values include:
661 *    - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure
662 *    - E_RTE_SECONDARY - function was called from a secondary process instance
663 *    - EINVAL - cache size provided is too large
664 *    - ENOSPC - the maximum number of memzones has already been allocated
665 *    - EEXIST - a memzone with the same name already exists
666 *    - ENOMEM - no appropriate memory area found in which to create memzone
667 */
668struct rte_mempool *
669rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
670		   unsigned cache_size, unsigned private_data_size,
671		   rte_mempool_ctor_t *mp_init, void *mp_init_arg,
672		   rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
673		   int socket_id, unsigned flags);
674
675/**
676 * Create a new mempool named *name* in memory.
677 *
678 * The pool contains n elements of elt_size. Its size is set to n.
679 * This function uses ``memzone_reserve()`` to allocate the mempool header
680 * (and the objects if vaddr is NULL).
681 * Depending on the input parameters, mempool elements can be either allocated
682 * together with the mempool header, or an externally provided memory buffer
683 * could be used to store mempool objects. In later case, that external
684 * memory buffer can consist of set of disjoint physical pages.
685 *
686 * @param name
687 *   The name of the mempool.
688 * @param n
689 *   The number of elements in the mempool. The optimum size (in terms of
690 *   memory usage) for a mempool is when n is a power of two minus one:
691 *   n = (2^q - 1).
692 * @param elt_size
693 *   The size of each element.
694 * @param cache_size
695 *   Size of the cache. See rte_mempool_create() for details.
696 * @param private_data_size
697 *   The size of the private data appended after the mempool
698 *   structure. This is useful for storing some private data after the
699 *   mempool structure, as is done for rte_mbuf_pool for example.
700 * @param mp_init
701 *   A function pointer that is called for initialization of the pool,
702 *   before object initialization. The user can initialize the private
703 *   data in this function if needed. This parameter can be NULL if
704 *   not needed.
705 * @param mp_init_arg
706 *   An opaque pointer to data that can be used in the mempool
707 *   constructor function.
708 * @param obj_init
709 *   A function called for each object at initialization of the pool.
710 *   See rte_mempool_create() for details.
711 * @param obj_init_arg
712 *   An opaque pointer passed to the object constructor function.
713 * @param socket_id
714 *   The *socket_id* argument is the socket identifier in the case of
715 *   NUMA. The value can be *SOCKET_ID_ANY* if there is no NUMA
716 *   constraint for the reserved zone.
717 * @param flags
718 *   Flags controlling the behavior of the mempool. See
719 *   rte_mempool_create() for details.
720 * @param vaddr
721 *   Virtual address of the externally allocated memory buffer.
722 *   Will be used to store mempool objects.
723 * @param paddr
724 *   Array of physical addresses of the pages that comprises given memory
725 *   buffer.
726 * @param pg_num
727 *   Number of elements in the paddr array.
728 * @param pg_shift
729 *   LOG2 of the physical pages size.
730 * @return
731 *   The pointer to the new allocated mempool, on success. NULL on error
732 *   with rte_errno set appropriately. See rte_mempool_create() for details.
733 */
734struct rte_mempool *
735rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size,
736		unsigned cache_size, unsigned private_data_size,
737		rte_mempool_ctor_t *mp_init, void *mp_init_arg,
738		rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
739		int socket_id, unsigned flags, void *vaddr,
740		const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift);
741
742/**
743 * Create an empty mempool
744 *
745 * The mempool is allocated and initialized, but it is not populated: no
746 * memory is allocated for the mempool elements. The user has to call
747 * rte_mempool_populate_*() to add memory chunks to the pool. Once
748 * populated, the user may also want to initialize each object with
749 * rte_mempool_obj_iter().
750 *
751 * @param name
752 *   The name of the mempool.
753 * @param n
754 *   The maximum number of elements that can be added in the mempool.
755 *   The optimum size (in terms of memory usage) for a mempool is when n
756 *   is a power of two minus one: n = (2^q - 1).
757 * @param elt_size
758 *   The size of each element.
759 * @param cache_size
760 *   Size of the cache. See rte_mempool_create() for details.
761 * @param private_data_size
762 *   The size of the private data appended after the mempool
763 *   structure. This is useful for storing some private data after the
764 *   mempool structure, as is done for rte_mbuf_pool for example.
765 * @param socket_id
766 *   The *socket_id* argument is the socket identifier in the case of
767 *   NUMA. The value can be *SOCKET_ID_ANY* if there is no NUMA
768 *   constraint for the reserved zone.
769 * @param flags
770 *   Flags controlling the behavior of the mempool. See
771 *   rte_mempool_create() for details.
772 * @return
773 *   The pointer to the new allocated mempool, on success. NULL on error
774 *   with rte_errno set appropriately. See rte_mempool_create() for details.
775 */
776struct rte_mempool *
777rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size,
778	unsigned cache_size, unsigned private_data_size,
779	int socket_id, unsigned flags);
780/**
781 * Free a mempool
782 *
783 * Unlink the mempool from global list, free the memory chunks, and all
784 * memory referenced by the mempool. The objects must not be used by
785 * other cores as they will be freed.
786 *
787 * @param mp
788 *   A pointer to the mempool structure.
789 */
790void
791rte_mempool_free(struct rte_mempool *mp);
792
793/**
794 * Add physically contiguous memory for objects in the pool at init
795 *
796 * Add a virtually and physically contiguous memory chunk in the pool
797 * where objects can be instanciated.
798 *
799 * If the given physical address is unknown (paddr = RTE_BAD_PHYS_ADDR),
800 * the chunk doesn't need to be physically contiguous (only virtually),
801 * and allocated objects may span two pages.
802 *
803 * @param mp
804 *   A pointer to the mempool structure.
805 * @param vaddr
806 *   The virtual address of memory that should be used to store objects.
807 * @param paddr
808 *   The physical address
809 * @param len
810 *   The length of memory in bytes.
811 * @param free_cb
812 *   The callback used to free this chunk when destroying the mempool.
813 * @param opaque
814 *   An opaque argument passed to free_cb.
815 * @return
816 *   The number of objects added on success.
817 *   On error, the chunk is not added in the memory list of the
818 *   mempool and a negative errno is returned.
819 */
820int rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr,
821	phys_addr_t paddr, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
822	void *opaque);
823
824/**
825 * Add physical memory for objects in the pool at init
826 *
827 * Add a virtually contiguous memory chunk in the pool where objects can
828 * be instanciated. The physical addresses corresponding to the virtual
829 * area are described in paddr[], pg_num, pg_shift.
830 *
831 * @param mp
832 *   A pointer to the mempool structure.
833 * @param vaddr
834 *   The virtual address of memory that should be used to store objects.
835 * @param paddr
836 *   An array of physical addresses of each page composing the virtual
837 *   area.
838 * @param pg_num
839 *   Number of elements in the paddr array.
840 * @param pg_shift
841 *   LOG2 of the physical pages size.
842 * @param free_cb
843 *   The callback used to free this chunk when destroying the mempool.
844 * @param opaque
845 *   An opaque argument passed to free_cb.
846 * @return
847 *   The number of objects added on success.
848 *   On error, the chunks are not added in the memory list of the
849 *   mempool and a negative errno is returned.
850 */
851int rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr,
852	const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift,
853	rte_mempool_memchunk_free_cb_t *free_cb, void *opaque);
854
855/**
856 * Add virtually contiguous memory for objects in the pool at init
857 *
858 * Add a virtually contiguous memory chunk in the pool where objects can
859 * be instanciated.
860 *
861 * @param mp
862 *   A pointer to the mempool structure.
863 * @param addr
864 *   The virtual address of memory that should be used to store objects.
865 *   Must be page-aligned.
866 * @param len
867 *   The length of memory in bytes. Must be page-aligned.
868 * @param pg_sz
869 *   The size of memory pages in this virtual area.
870 * @param free_cb
871 *   The callback used to free this chunk when destroying the mempool.
872 * @param opaque
873 *   An opaque argument passed to free_cb.
874 * @return
875 *   The number of objects added on success.
876 *   On error, the chunk is not added in the memory list of the
877 *   mempool and a negative errno is returned.
878 */
879int
880rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
881	size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb,
882	void *opaque);
883
884/**
885 * Add memory for objects in the pool at init
886 *
887 * This is the default function used by rte_mempool_create() to populate
888 * the mempool. It adds memory allocated using rte_memzone_reserve().
889 *
890 * @param mp
891 *   A pointer to the mempool structure.
892 * @return
893 *   The number of objects added on success.
894 *   On error, the chunk is not added in the memory list of the
895 *   mempool and a negative errno is returned.
896 */
897int rte_mempool_populate_default(struct rte_mempool *mp);
898
899/**
900 * Add memory from anonymous mapping for objects in the pool at init
901 *
902 * This function mmap an anonymous memory zone that is locked in
903 * memory to store the objects of the mempool.
904 *
905 * @param mp
906 *   A pointer to the mempool structure.
907 * @return
908 *   The number of objects added on success.
909 *   On error, the chunk is not added in the memory list of the
910 *   mempool and a negative errno is returned.
911 */
912int rte_mempool_populate_anon(struct rte_mempool *mp);
913
914/**
915 * Call a function for each mempool element
916 *
917 * Iterate across all objects attached to a rte_mempool and call the
918 * callback function on it.
919 *
920 * @param mp
921 *   A pointer to an initialized mempool.
922 * @param obj_cb
923 *   A function pointer that is called for each object.
924 * @param obj_cb_arg
925 *   An opaque pointer passed to the callback function.
926 * @return
927 *   Number of objects iterated.
928 */
929uint32_t rte_mempool_obj_iter(struct rte_mempool *mp,
930	rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg);
931
932/**
933 * Call a function for each mempool memory chunk
934 *
935 * Iterate across all memory chunks attached to a rte_mempool and call
936 * the callback function on it.
937 *
938 * @param mp
939 *   A pointer to an initialized mempool.
940 * @param mem_cb
941 *   A function pointer that is called for each memory chunk.
942 * @param mem_cb_arg
943 *   An opaque pointer passed to the callback function.
944 * @return
945 *   Number of memory chunks iterated.
946 */
947uint32_t rte_mempool_mem_iter(struct rte_mempool *mp,
948	rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg);
949
950/**
951 * Dump the status of the mempool to a file.
952 *
953 * @param f
954 *   A pointer to a file for output
955 * @param mp
956 *   A pointer to the mempool structure.
957 */
958void rte_mempool_dump(FILE *f, struct rte_mempool *mp);
959
960/**
961 * Create a user-owned mempool cache.
962 *
963 * This can be used by non-EAL threads to enable caching when they
964 * interact with a mempool.
965 *
966 * @param size
967 *   The size of the mempool cache. See rte_mempool_create()'s cache_size
968 *   parameter description for more information. The same limits and
969 *   considerations apply here too.
970 * @param socket_id
971 *   The socket identifier in the case of NUMA. The value can be
972 *   SOCKET_ID_ANY if there is no NUMA constraint for the reserved zone.
973 */
974struct rte_mempool_cache *
975rte_mempool_cache_create(uint32_t size, int socket_id);
976
977/**
978 * Free a user-owned mempool cache.
979 *
980 * @param cache
981 *   A pointer to the mempool cache.
982 */
983void
984rte_mempool_cache_free(struct rte_mempool_cache *cache);
985
986/**
987 * Flush a user-owned mempool cache to the specified mempool.
988 *
989 * @param cache
990 *   A pointer to the mempool cache.
991 * @param mp
992 *   A pointer to the mempool.
993 */
994static inline void __attribute__((always_inline))
995rte_mempool_cache_flush(struct rte_mempool_cache *cache,
996			struct rte_mempool *mp)
997{
998	rte_mempool_ops_enqueue_bulk(mp, cache->objs, cache->len);
999	cache->len = 0;
1000}
1001
1002/**
1003 * Get a pointer to the per-lcore default mempool cache.
1004 *
1005 * @param mp
1006 *   A pointer to the mempool structure.
1007 * @param lcore_id
1008 *   The logical core id.
1009 * @return
1010 *   A pointer to the mempool cache or NULL if disabled or non-EAL thread.
1011 */
1012static inline struct rte_mempool_cache *__attribute__((always_inline))
1013rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
1014{
1015	if (mp->cache_size == 0)
1016		return NULL;
1017
1018	if (lcore_id >= RTE_MAX_LCORE)
1019		return NULL;
1020
1021	return &mp->local_cache[lcore_id];
1022}
1023
1024/**
1025 * @internal Put several objects back in the mempool; used internally.
1026 * @param mp
1027 *   A pointer to the mempool structure.
1028 * @param obj_table
1029 *   A pointer to a table of void * pointers (objects).
1030 * @param n
1031 *   The number of objects to store back in the mempool, must be strictly
1032 *   positive.
1033 * @param cache
1034 *   A pointer to a mempool cache structure. May be NULL if not needed.
1035 * @param flags
1036 *   The flags used for the mempool creation.
1037 *   Single-producer (MEMPOOL_F_SP_PUT flag) or multi-producers.
1038 */
1039static inline void __attribute__((always_inline))
1040__mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1041		      unsigned n, struct rte_mempool_cache *cache)
1042{
1043	void **cache_objs;
1044
1045	/* increment stat now, adding in mempool always success */
1046	__MEMPOOL_STAT_ADD(mp, put, n);
1047
1048	/* No cache provided or if put would overflow mem allocated for cache */
1049	if (unlikely(cache == NULL || n > RTE_MEMPOOL_CACHE_MAX_SIZE))
1050		goto ring_enqueue;
1051
1052	cache_objs = &cache->objs[cache->len];
1053
1054	/*
1055	 * The cache follows the following algorithm
1056	 *   1. Add the objects to the cache
1057	 *   2. Anything greater than the cache min value (if it crosses the
1058	 *   cache flush threshold) is flushed to the ring.
1059	 */
1060
1061	/* Add elements back into the cache */
1062	rte_memcpy(&cache_objs[0], obj_table, sizeof(void *) * n);
1063
1064	cache->len += n;
1065
1066	if (cache->len >= cache->flushthresh) {
1067		rte_mempool_ops_enqueue_bulk(mp, &cache->objs[cache->size],
1068				cache->len - cache->size);
1069		cache->len = cache->size;
1070	}
1071
1072	return;
1073
1074ring_enqueue:
1075
1076	/* push remaining objects in ring */
1077#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
1078	if (rte_mempool_ops_enqueue_bulk(mp, obj_table, n) < 0)
1079		rte_panic("cannot put objects in mempool\n");
1080#else
1081	rte_mempool_ops_enqueue_bulk(mp, obj_table, n);
1082#endif
1083}
1084
1085
1086/**
1087 * Put several objects back in the mempool.
1088 *
1089 * @param mp
1090 *   A pointer to the mempool structure.
1091 * @param obj_table
1092 *   A pointer to a table of void * pointers (objects).
1093 * @param n
1094 *   The number of objects to add in the mempool from the obj_table.
1095 * @param cache
1096 *   A pointer to a mempool cache structure. May be NULL if not needed.
1097 * @param flags
1098 *   The flags used for the mempool creation.
1099 *   Single-producer (MEMPOOL_F_SP_PUT flag) or multi-producers.
1100 */
1101static inline void __attribute__((always_inline))
1102rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1103			unsigned n, struct rte_mempool_cache *cache,
1104			__rte_unused int flags)
1105{
1106	__mempool_check_cookies(mp, obj_table, n, 0);
1107	__mempool_generic_put(mp, obj_table, n, cache);
1108}
1109
1110/**
1111 * @deprecated
1112 * Put several objects back in the mempool (multi-producers safe).
1113 *
1114 * @param mp
1115 *   A pointer to the mempool structure.
1116 * @param obj_table
1117 *   A pointer to a table of void * pointers (objects).
1118 * @param n
1119 *   The number of objects to add in the mempool from the obj_table.
1120 */
1121__rte_deprecated
1122static inline void __attribute__((always_inline))
1123rte_mempool_mp_put_bulk(struct rte_mempool *mp, void * const *obj_table,
1124			unsigned n)
1125{
1126	struct rte_mempool_cache *cache;
1127	cache = rte_mempool_default_cache(mp, rte_lcore_id());
1128	rte_mempool_generic_put(mp, obj_table, n, cache, 0);
1129}
1130
1131/**
1132 * @deprecated
1133 * Put several objects back in the mempool (NOT multi-producers safe).
1134 *
1135 * @param mp
1136 *   A pointer to the mempool structure.
1137 * @param obj_table
1138 *   A pointer to a table of void * pointers (objects).
1139 * @param n
1140 *   The number of objects to add in the mempool from obj_table.
1141 */
1142__rte_deprecated
1143static inline void __attribute__((always_inline))
1144rte_mempool_sp_put_bulk(struct rte_mempool *mp, void * const *obj_table,
1145			unsigned n)
1146{
1147	rte_mempool_generic_put(mp, obj_table, n, NULL, MEMPOOL_F_SP_PUT);
1148}
1149
1150/**
1151 * Put several objects back in the mempool.
1152 *
1153 * This function calls the multi-producer or the single-producer
1154 * version depending on the default behavior that was specified at
1155 * mempool creation time (see flags).
1156 *
1157 * @param mp
1158 *   A pointer to the mempool structure.
1159 * @param obj_table
1160 *   A pointer to a table of void * pointers (objects).
1161 * @param n
1162 *   The number of objects to add in the mempool from obj_table.
1163 */
1164static inline void __attribute__((always_inline))
1165rte_mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,
1166		     unsigned n)
1167{
1168	struct rte_mempool_cache *cache;
1169	cache = rte_mempool_default_cache(mp, rte_lcore_id());
1170	rte_mempool_generic_put(mp, obj_table, n, cache, mp->flags);
1171}
1172
1173/**
1174 * @deprecated
1175 * Put one object in the mempool (multi-producers safe).
1176 *
1177 * @param mp
1178 *   A pointer to the mempool structure.
1179 * @param obj
1180 *   A pointer to the object to be added.
1181 */
1182__rte_deprecated
1183static inline void __attribute__((always_inline))
1184rte_mempool_mp_put(struct rte_mempool *mp, void *obj)
1185{
1186	struct rte_mempool_cache *cache;
1187	cache = rte_mempool_default_cache(mp, rte_lcore_id());
1188	rte_mempool_generic_put(mp, &obj, 1, cache, 0);
1189}
1190
1191/**
1192 * @deprecated
1193 * Put one object back in the mempool (NOT multi-producers safe).
1194 *
1195 * @param mp
1196 *   A pointer to the mempool structure.
1197 * @param obj
1198 *   A pointer to the object to be added.
1199 */
1200__rte_deprecated
1201static inline void __attribute__((always_inline))
1202rte_mempool_sp_put(struct rte_mempool *mp, void *obj)
1203{
1204	rte_mempool_generic_put(mp, &obj, 1, NULL, MEMPOOL_F_SP_PUT);
1205}
1206
1207/**
1208 * Put one object back in the mempool.
1209 *
1210 * This function calls the multi-producer or the single-producer
1211 * version depending on the default behavior that was specified at
1212 * mempool creation time (see flags).
1213 *
1214 * @param mp
1215 *   A pointer to the mempool structure.
1216 * @param obj
1217 *   A pointer to the object to be added.
1218 */
1219static inline void __attribute__((always_inline))
1220rte_mempool_put(struct rte_mempool *mp, void *obj)
1221{
1222	rte_mempool_put_bulk(mp, &obj, 1);
1223}
1224
1225/**
1226 * @internal Get several objects from the mempool; used internally.
1227 * @param mp
1228 *   A pointer to the mempool structure.
1229 * @param obj_table
1230 *   A pointer to a table of void * pointers (objects).
1231 * @param n
1232 *   The number of objects to get, must be strictly positive.
1233 * @param cache
1234 *   A pointer to a mempool cache structure. May be NULL if not needed.
1235 * @param flags
1236 *   The flags used for the mempool creation.
1237 *   Single-consumer (MEMPOOL_F_SC_GET flag) or multi-consumers.
1238 * @return
1239 *   - >=0: Success; number of objects supplied.
1240 *   - <0: Error; code of ring dequeue function.
1241 */
1242static inline int __attribute__((always_inline))
1243__mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1244		      unsigned n, struct rte_mempool_cache *cache)
1245{
1246	int ret;
1247	uint32_t index, len;
1248	void **cache_objs;
1249
1250	/* No cache provided or cannot be satisfied from cache */
1251	if (unlikely(cache == NULL || n >= cache->size))
1252		goto ring_dequeue;
1253
1254	cache_objs = cache->objs;
1255
1256	/* Can this be satisfied from the cache? */
1257	if (cache->len < n) {
1258		/* No. Backfill the cache first, and then fill from it */
1259		uint32_t req = n + (cache->size - cache->len);
1260
1261		/* How many do we require i.e. number to fill the cache + the request */
1262		ret = rte_mempool_ops_dequeue_bulk(mp,
1263			&cache->objs[cache->len], req);
1264		if (unlikely(ret < 0)) {
1265			/*
1266			 * In the offchance that we are buffer constrained,
1267			 * where we are not able to allocate cache + n, go to
1268			 * the ring directly. If that fails, we are truly out of
1269			 * buffers.
1270			 */
1271			goto ring_dequeue;
1272		}
1273
1274		cache->len += req;
1275	}
1276
1277	/* Now fill in the response ... */
1278	for (index = 0, len = cache->len - 1; index < n; ++index, len--, obj_table++)
1279		*obj_table = cache_objs[len];
1280
1281	cache->len -= n;
1282
1283	__MEMPOOL_STAT_ADD(mp, get_success, n);
1284
1285	return 0;
1286
1287ring_dequeue:
1288
1289	/* get remaining objects from ring */
1290	ret = rte_mempool_ops_dequeue_bulk(mp, obj_table, n);
1291
1292	if (ret < 0)
1293		__MEMPOOL_STAT_ADD(mp, get_fail, n);
1294	else
1295		__MEMPOOL_STAT_ADD(mp, get_success, n);
1296
1297	return ret;
1298}
1299
1300/**
1301 * Get several objects from the mempool.
1302 *
1303 * If cache is enabled, objects will be retrieved first from cache,
1304 * subsequently from the common pool. Note that it can return -ENOENT when
1305 * the local cache and common pool are empty, even if cache from other
1306 * lcores are full.
1307 *
1308 * @param mp
1309 *   A pointer to the mempool structure.
1310 * @param obj_table
1311 *   A pointer to a table of void * pointers (objects) that will be filled.
1312 * @param n
1313 *   The number of objects to get from mempool to obj_table.
1314 * @param cache
1315 *   A pointer to a mempool cache structure. May be NULL if not needed.
1316 * @param flags
1317 *   The flags used for the mempool creation.
1318 *   Single-consumer (MEMPOOL_F_SC_GET flag) or multi-consumers.
1319 * @return
1320 *   - 0: Success; objects taken.
1321 *   - -ENOENT: Not enough entries in the mempool; no object is retrieved.
1322 */
1323static inline int __attribute__((always_inline))
1324rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table, unsigned n,
1325			struct rte_mempool_cache *cache, __rte_unused int flags)
1326{
1327	int ret;
1328	ret = __mempool_generic_get(mp, obj_table, n, cache);
1329	if (ret == 0)
1330		__mempool_check_cookies(mp, obj_table, n, 1);
1331	return ret;
1332}
1333
1334/**
1335 * @deprecated
1336 * Get several objects from the mempool (multi-consumers safe).
1337 *
1338 * If cache is enabled, objects will be retrieved first from cache,
1339 * subsequently from the common pool. Note that it can return -ENOENT when
1340 * the local cache and common pool are empty, even if cache from other
1341 * lcores are full.
1342 *
1343 * @param mp
1344 *   A pointer to the mempool structure.
1345 * @param obj_table
1346 *   A pointer to a table of void * pointers (objects) that will be filled.
1347 * @param n
1348 *   The number of objects to get from mempool to obj_table.
1349 * @return
1350 *   - 0: Success; objects taken.
1351 *   - -ENOENT: Not enough entries in the mempool; no object is retrieved.
1352 */
1353__rte_deprecated
1354static inline int __attribute__((always_inline))
1355rte_mempool_mc_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n)
1356{
1357	struct rte_mempool_cache *cache;
1358	cache = rte_mempool_default_cache(mp, rte_lcore_id());
1359	return rte_mempool_generic_get(mp, obj_table, n, cache, 0);
1360}
1361
1362/**
1363 * @deprecated
1364 * Get several objects from the mempool (NOT multi-consumers safe).
1365 *
1366 * If cache is enabled, objects will be retrieved first from cache,
1367 * subsequently from the common pool. Note that it can return -ENOENT when
1368 * the local cache and common pool are empty, even if cache from other
1369 * lcores are full.
1370 *
1371 * @param mp
1372 *   A pointer to the mempool structure.
1373 * @param obj_table
1374 *   A pointer to a table of void * pointers (objects) that will be filled.
1375 * @param n
1376 *   The number of objects to get from the mempool to obj_table.
1377 * @return
1378 *   - 0: Success; objects taken.
1379 *   - -ENOENT: Not enough entries in the mempool; no object is
1380 *     retrieved.
1381 */
1382__rte_deprecated
1383static inline int __attribute__((always_inline))
1384rte_mempool_sc_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n)
1385{
1386	return rte_mempool_generic_get(mp, obj_table, n, NULL,
1387				       MEMPOOL_F_SC_GET);
1388}
1389
1390/**
1391 * Get several objects from the mempool.
1392 *
1393 * This function calls the multi-consumers or the single-consumer
1394 * version, depending on the default behaviour that was specified at
1395 * mempool creation time (see flags).
1396 *
1397 * If cache is enabled, objects will be retrieved first from cache,
1398 * subsequently from the common pool. Note that it can return -ENOENT when
1399 * the local cache and common pool are empty, even if cache from other
1400 * lcores are full.
1401 *
1402 * @param mp
1403 *   A pointer to the mempool structure.
1404 * @param obj_table
1405 *   A pointer to a table of void * pointers (objects) that will be filled.
1406 * @param n
1407 *   The number of objects to get from the mempool to obj_table.
1408 * @return
1409 *   - 0: Success; objects taken
1410 *   - -ENOENT: Not enough entries in the mempool; no object is retrieved.
1411 */
1412static inline int __attribute__((always_inline))
1413rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n)
1414{
1415	struct rte_mempool_cache *cache;
1416	cache = rte_mempool_default_cache(mp, rte_lcore_id());
1417	return rte_mempool_generic_get(mp, obj_table, n, cache, mp->flags);
1418}
1419
1420/**
1421 * @deprecated
1422 * Get one object from the mempool (multi-consumers safe).
1423 *
1424 * If cache is enabled, objects will be retrieved first from cache,
1425 * subsequently from the common pool. Note that it can return -ENOENT when
1426 * the local cache and common pool are empty, even if cache from other
1427 * lcores are full.
1428 *
1429 * @param mp
1430 *   A pointer to the mempool structure.
1431 * @param obj_p
1432 *   A pointer to a void * pointer (object) that will be filled.
1433 * @return
1434 *   - 0: Success; objects taken.
1435 *   - -ENOENT: Not enough entries in the mempool; no object is retrieved.
1436 */
1437__rte_deprecated
1438static inline int __attribute__((always_inline))
1439rte_mempool_mc_get(struct rte_mempool *mp, void **obj_p)
1440{
1441	struct rte_mempool_cache *cache;
1442	cache = rte_mempool_default_cache(mp, rte_lcore_id());
1443	return rte_mempool_generic_get(mp, obj_p, 1, cache, 0);
1444}
1445
1446/**
1447 * @deprecated
1448 * Get one object from the mempool (NOT multi-consumers safe).
1449 *
1450 * If cache is enabled, objects will be retrieved first from cache,
1451 * subsequently from the common pool. Note that it can return -ENOENT when
1452 * the local cache and common pool are empty, even if cache from other
1453 * lcores are full.
1454 *
1455 * @param mp
1456 *   A pointer to the mempool structure.
1457 * @param obj_p
1458 *   A pointer to a void * pointer (object) that will be filled.
1459 * @return
1460 *   - 0: Success; objects taken.
1461 *   - -ENOENT: Not enough entries in the mempool; no object is retrieved.
1462 */
1463__rte_deprecated
1464static inline int __attribute__((always_inline))
1465rte_mempool_sc_get(struct rte_mempool *mp, void **obj_p)
1466{
1467	return rte_mempool_generic_get(mp, obj_p, 1, NULL, MEMPOOL_F_SC_GET);
1468}
1469
1470/**
1471 * Get one object from the mempool.
1472 *
1473 * This function calls the multi-consumers or the single-consumer
1474 * version, depending on the default behavior that was specified at
1475 * mempool creation (see flags).
1476 *
1477 * If cache is enabled, objects will be retrieved first from cache,
1478 * subsequently from the common pool. Note that it can return -ENOENT when
1479 * the local cache and common pool are empty, even if cache from other
1480 * lcores are full.
1481 *
1482 * @param mp
1483 *   A pointer to the mempool structure.
1484 * @param obj_p
1485 *   A pointer to a void * pointer (object) that will be filled.
1486 * @return
1487 *   - 0: Success; objects taken.
1488 *   - -ENOENT: Not enough entries in the mempool; no object is retrieved.
1489 */
1490static inline int __attribute__((always_inline))
1491rte_mempool_get(struct rte_mempool *mp, void **obj_p)
1492{
1493	return rte_mempool_get_bulk(mp, obj_p, 1);
1494}
1495
1496/**
1497 * Return the number of entries in the mempool.
1498 *
1499 * When cache is enabled, this function has to browse the length of
1500 * all lcores, so it should not be used in a data path, but only for
1501 * debug purposes. User-owned mempool caches are not accounted for.
1502 *
1503 * @param mp
1504 *   A pointer to the mempool structure.
1505 * @return
1506 *   The number of entries in the mempool.
1507 */
1508unsigned int rte_mempool_avail_count(const struct rte_mempool *mp);
1509
1510/**
1511 * @deprecated
1512 * Return the number of entries in the mempool.
1513 *
1514 * When cache is enabled, this function has to browse the length of
1515 * all lcores, so it should not be used in a data path, but only for
1516 * debug purposes.
1517 *
1518 * @param mp
1519 *   A pointer to the mempool structure.
1520 * @return
1521 *   The number of entries in the mempool.
1522 */
1523__rte_deprecated
1524unsigned rte_mempool_count(const struct rte_mempool *mp);
1525
1526/**
1527 * Return the number of elements which have been allocated from the mempool
1528 *
1529 * When cache is enabled, this function has to browse the length of
1530 * all lcores, so it should not be used in a data path, but only for
1531 * debug purposes.
1532 *
1533 * @param mp
1534 *   A pointer to the mempool structure.
1535 * @return
1536 *   The number of free entries in the mempool.
1537 */
1538unsigned int
1539rte_mempool_in_use_count(const struct rte_mempool *mp);
1540
1541/**
1542 * @deprecated
1543 * Return the number of free entries in the mempool ring.
1544 * i.e. how many entries can be freed back to the mempool.
1545 *
1546 * NOTE: This corresponds to the number of elements *allocated* from the
1547 * memory pool, not the number of elements in the pool itself. To count
1548 * the number elements currently available in the pool, use "rte_mempool_count"
1549 *
1550 * When cache is enabled, this function has to browse the length of
1551 * all lcores, so it should not be used in a data path, but only for
1552 * debug purposes. User-owned mempool caches are not accounted for.
1553 *
1554 * @param mp
1555 *   A pointer to the mempool structure.
1556 * @return
1557 *   The number of free entries in the mempool.
1558 */
1559__rte_deprecated
1560static inline unsigned
1561rte_mempool_free_count(const struct rte_mempool *mp)
1562{
1563	return rte_mempool_in_use_count(mp);
1564}
1565
1566/**
1567 * Test if the mempool is full.
1568 *
1569 * When cache is enabled, this function has to browse the length of all
1570 * lcores, so it should not be used in a data path, but only for debug
1571 * purposes. User-owned mempool caches are not accounted for.
1572 *
1573 * @param mp
1574 *   A pointer to the mempool structure.
1575 * @return
1576 *   - 1: The mempool is full.
1577 *   - 0: The mempool is not full.
1578 */
1579static inline int
1580rte_mempool_full(const struct rte_mempool *mp)
1581{
1582	return !!(rte_mempool_avail_count(mp) == mp->size);
1583}
1584
1585/**
1586 * Test if the mempool is empty.
1587 *
1588 * When cache is enabled, this function has to browse the length of all
1589 * lcores, so it should not be used in a data path, but only for debug
1590 * purposes. User-owned mempool caches are not accounted for.
1591 *
1592 * @param mp
1593 *   A pointer to the mempool structure.
1594 * @return
1595 *   - 1: The mempool is empty.
1596 *   - 0: The mempool is not empty.
1597 */
1598static inline int
1599rte_mempool_empty(const struct rte_mempool *mp)
1600{
1601	return !!(rte_mempool_avail_count(mp) == 0);
1602}
1603
1604/**
1605 * Return the physical address of elt, which is an element of the pool mp.
1606 *
1607 * @param mp
1608 *   A pointer to the mempool structure.
1609 * @param elt
1610 *   A pointer (virtual address) to the element of the pool.
1611 * @return
1612 *   The physical address of the elt element.
1613 *   If the mempool was created with MEMPOOL_F_NO_PHYS_CONTIG, the
1614 *   returned value is RTE_BAD_PHYS_ADDR.
1615 */
1616static inline phys_addr_t
1617rte_mempool_virt2phy(__rte_unused const struct rte_mempool *mp, const void *elt)
1618{
1619	const struct rte_mempool_objhdr *hdr;
1620	hdr = (const struct rte_mempool_objhdr *)RTE_PTR_SUB(elt,
1621		sizeof(*hdr));
1622	return hdr->physaddr;
1623}
1624
1625/**
1626 * Check the consistency of mempool objects.
1627 *
1628 * Verify the coherency of fields in the mempool structure. Also check
1629 * that the cookies of mempool objects (even the ones that are not
1630 * present in pool) have a correct value. If not, a panic will occur.
1631 *
1632 * @param mp
1633 *   A pointer to the mempool structure.
1634 */
1635void rte_mempool_audit(struct rte_mempool *mp);
1636
1637/**
1638 * Return a pointer to the private data in an mempool structure.
1639 *
1640 * @param mp
1641 *   A pointer to the mempool structure.
1642 * @return
1643 *   A pointer to the private data.
1644 */
1645static inline void *rte_mempool_get_priv(struct rte_mempool *mp)
1646{
1647	return (char *)mp +
1648		MEMPOOL_HEADER_SIZE(mp, mp->cache_size);
1649}
1650
1651/**
1652 * Dump the status of all mempools on the console
1653 *
1654 * @param f
1655 *   A pointer to a file for output
1656 */
1657void rte_mempool_list_dump(FILE *f);
1658
1659/**
1660 * Search a mempool from its name
1661 *
1662 * @param name
1663 *   The name of the mempool.
1664 * @return
1665 *   The pointer to the mempool matching the name, or NULL if not found.
1666 *   NULL on error
1667 *   with rte_errno set appropriately. Possible rte_errno values include:
1668 *    - ENOENT - required entry not available to return.
1669 *
1670 */
1671struct rte_mempool *rte_mempool_lookup(const char *name);
1672
1673/**
1674 * Get the header, trailer and total size of a mempool element.
1675 *
1676 * Given a desired size of the mempool element and mempool flags,
1677 * calculates header, trailer, body and total sizes of the mempool object.
1678 *
1679 * @param elt_size
1680 *   The size of each element, without header and trailer.
1681 * @param flags
1682 *   The flags used for the mempool creation.
1683 *   Consult rte_mempool_create() for more information about possible values.
1684 *   The size of each element.
1685 * @param sz
1686 *   The calculated detailed size the mempool object. May be NULL.
1687 * @return
1688 *   Total size of the mempool object.
1689 */
1690uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
1691	struct rte_mempool_objsz *sz);
1692
1693/**
1694 * Get the size of memory required to store mempool elements.
1695 *
1696 * Calculate the maximum amount of memory required to store given number
1697 * of objects. Assume that the memory buffer will be aligned at page
1698 * boundary.
1699 *
1700 * Note that if object size is bigger then page size, then it assumes
1701 * that pages are grouped in subsets of physically continuous pages big
1702 * enough to store at least one object.
1703 *
1704 * @param elt_num
1705 *   Number of elements.
1706 * @param total_elt_sz
1707 *   The size of each element, including header and trailer, as returned
1708 *   by rte_mempool_calc_obj_size().
1709 * @param pg_shift
1710 *   LOG2 of the physical pages size. If set to 0, ignore page boundaries.
1711 * @return
1712 *   Required memory size aligned at page boundary.
1713 */
1714size_t rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz,
1715	uint32_t pg_shift);
1716
1717/**
1718 * Get the size of memory required to store mempool elements.
1719 *
1720 * Calculate how much memory would be actually required with the given
1721 * memory footprint to store required number of objects.
1722 *
1723 * @param vaddr
1724 *   Virtual address of the externally allocated memory buffer.
1725 *   Will be used to store mempool objects.
1726 * @param elt_num
1727 *   Number of elements.
1728 * @param total_elt_sz
1729 *   The size of each element, including header and trailer, as returned
1730 *   by rte_mempool_calc_obj_size().
1731 * @param paddr
1732 *   Array of physical addresses of the pages that comprises given memory
1733 *   buffer.
1734 * @param pg_num
1735 *   Number of elements in the paddr array.
1736 * @param pg_shift
1737 *   LOG2 of the physical pages size.
1738 * @return
1739 *   On success, the number of bytes needed to store given number of
1740 *   objects, aligned to the given page size. If the provided memory
1741 *   buffer is too small, return a negative value whose absolute value
1742 *   is the actual number of elements that can be stored in that buffer.
1743 */
1744ssize_t rte_mempool_xmem_usage(void *vaddr, uint32_t elt_num,
1745	size_t total_elt_sz, const phys_addr_t paddr[], uint32_t pg_num,
1746	uint32_t pg_shift);
1747
1748/**
1749 * Walk list of all memory pools
1750 *
1751 * @param func
1752 *   Iterator function
1753 * @param arg
1754 *   Argument passed to iterator
1755 */
1756void rte_mempool_walk(void (*func)(struct rte_mempool *, void *arg),
1757		      void *arg);
1758
1759#ifdef __cplusplus
1760}
1761#endif
1762
1763#endif /* _RTE_MEMPOOL_H_ */
1764