main.c revision 47d9763a
1
2/*-
3 *   BSD LICENSE
4 *
5 *   Copyright(c) 2015 Intel Corporation. All rights reserved.
6 *   All rights reserved.
7 *
8 *   Redistribution and use in source and binary forms, with or without
9 *   modification, are permitted provided that the following conditions
10 *   are met:
11 *
12 *     * Redistributions of source code must retain the above copyright
13 *       notice, this list of conditions and the following disclaimer.
14 *     * Redistributions in binary form must reproduce the above copyright
15 *       notice, this list of conditions and the following disclaimer in
16 *       the documentation and/or other materials provided with the
17 *       distribution.
18 *     * Neither the name of Intel Corporation nor the names of its
19 *       contributors may be used to endorse or promote products derived
20 *       from this software without specific prior written permission.
21 *
22 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 */
34
35#define _GNU_SOURCE
36#include <stdio.h>
37#include <stdlib.h>
38#include <stdint.h>
39#include <inttypes.h>
40#include <sys/types.h>
41#include <string.h>
42#include <sys/queue.h>
43#include <stdarg.h>
44#include <errno.h>
45#include <getopt.h>
46#include <unistd.h>
47#include <sched.h>
48#include <pthread.h>
49
50#include <rte_common.h>
51#include <rte_lcore.h>
52#include <rte_per_lcore.h>
53#include <rte_timer.h>
54
55#include "lthread_api.h"
56#include "lthread_diag_api.h"
57#include "pthread_shim.h"
58
59#define DEBUG_APP 0
60#define HELLOW_WORLD_MAX_LTHREADS 10
61
62#ifndef __GLIBC__ /* sched_getcpu() is glibc-specific */
63#define sched_getcpu() rte_lcore_id()
64#endif
65
66__thread int print_count;
67__thread pthread_mutex_t print_lock;
68
69__thread pthread_mutex_t exit_lock;
70__thread pthread_cond_t exit_cond;
71
72/*
73 * A simple thread that demonstrates use of a mutex, a condition
74 * variable, thread local storage, explicit yield, and thread exit.
75 *
76 * The thread uses a mutex to protect a shared counter which is incremented
77 * and then it waits on condition variable before exiting.
78 *
79 * The thread argument is stored in and retrieved from TLS, using
80 * the pthread key create, get and set specific APIs.
81 *
82 * The thread yields while holding the mutex, to provide opportunity
83 * for other threads to contend.
84 *
85 * All of the pthread API functions used by this thread are actually
86 * resolved to corresponding lthread functions by the pthread shim
87 * implemented in pthread_shim.c
88 */
89void *helloworld_pthread(void *arg);
90void *helloworld_pthread(void *arg)
91{
92	pthread_key_t key;
93
94	/* create a key for TLS */
95	pthread_key_create(&key, NULL);
96
97	/* store the arg in TLS */
98	pthread_setspecific(key, arg);
99
100	/* grab lock and increment shared counter */
101	pthread_mutex_lock(&print_lock);
102	print_count++;
103
104	/* yield thread to give opportunity for lock contention */
105	pthread_yield();
106
107	/* retrieve arg from TLS */
108	uint64_t thread_no = (uint64_t) pthread_getspecific(key);
109
110	printf("Hello - lcore = %d count = %d thread_no = %d thread_id = %p\n",
111			sched_getcpu(),
112			print_count,
113			(int) thread_no,
114			(void *)pthread_self());
115
116	/* release the lock */
117	pthread_mutex_unlock(&print_lock);
118
119	/*
120	 * wait on condition variable
121	 * before exiting
122	 */
123	pthread_mutex_lock(&exit_lock);
124	pthread_cond_wait(&exit_cond, &exit_lock);
125	pthread_mutex_unlock(&exit_lock);
126
127	/* exit */
128	pthread_exit((void *) thread_no);
129}
130
131
132/*
133 * This is the initial thread
134 *
135 * It demonstrates pthread, mutex and condition variable creation,
136 * broadcast and pthread join APIs.
137 *
138 * This initial thread must always start life as an lthread.
139 *
140 * This thread creates many more threads then waits a short time
141 * before signalling them to exit using a broadcast.
142 *
143 * All of the pthread API functions used by this thread are actually
144 * resolved to corresponding lthread functions by the pthread shim
145 * implemented in pthread_shim.c
146 *
147 * After all threads have finished the lthread scheduler is shutdown
148 * and normal pthread operation is restored
149 */
150__thread pthread_t tid[HELLOW_WORLD_MAX_LTHREADS];
151
152static void initial_lthread(void *args);
153static void initial_lthread(void *args __attribute__((unused)))
154{
155	int lcore = (int) rte_lcore_id();
156	/*
157	 *
158	 * We can now enable pthread API override
159	 * and start to use the pthread APIs
160	 */
161	pthread_override_set(1);
162
163	uint64_t i;
164	int ret;
165
166	/* initialize mutex for shared counter */
167	print_count = 0;
168	pthread_mutex_init(&print_lock, NULL);
169
170	/* initialize mutex and condition variable controlling thread exit */
171	pthread_mutex_init(&exit_lock, NULL);
172	pthread_cond_init(&exit_cond, NULL);
173
174	/* spawn a number of threads */
175	for (i = 0; i < HELLOW_WORLD_MAX_LTHREADS; i++) {
176
177		/*
178		 * Not strictly necessary but
179		 * for the sake of this example
180		 * use an attribute to pass the desired lcore
181		 */
182		pthread_attr_t attr;
183		rte_cpuset_t cpuset;
184
185		CPU_ZERO(&cpuset);
186		CPU_SET(lcore, &cpuset);
187		pthread_attr_init(&attr);
188		pthread_attr_setaffinity_np(&attr, sizeof(rte_cpuset_t), &cpuset);
189
190		/* create the thread */
191		ret = pthread_create(&tid[i], &attr,
192				helloworld_pthread, (void *) i);
193		if (ret != 0)
194			rte_exit(EXIT_FAILURE, "Cannot create helloworld thread\n");
195	}
196
197	/* wait for 1s to allow threads
198	 * to block on the condition variable
199	 * N.B. nanosleep() is resolved to lthread_sleep()
200	 * by the shim.
201	 */
202	struct timespec time;
203
204	time.tv_sec = 1;
205	time.tv_nsec = 0;
206	nanosleep(&time, NULL);
207
208	/* wake up all the threads */
209	pthread_cond_broadcast(&exit_cond);
210
211	/* wait for them to finish */
212	for (i = 0; i < HELLOW_WORLD_MAX_LTHREADS; i++) {
213
214		uint64_t thread_no;
215
216		pthread_join(tid[i], (void *) &thread_no);
217		if (thread_no != i)
218			printf("error on thread exit\n");
219	}
220
221	pthread_cond_destroy(&exit_cond);
222	pthread_mutex_destroy(&print_lock);
223	pthread_mutex_destroy(&exit_lock);
224
225	/* shutdown the lthread scheduler */
226	lthread_scheduler_shutdown(rte_lcore_id());
227	lthread_detach();
228}
229
230
231
232/* This thread creates a single initial lthread
233 * and then runs the scheduler
234 * An instance of this thread is created on each thread
235 * in the core mask
236 */
237static int
238lthread_scheduler(void *args);
239static int
240lthread_scheduler(void *args __attribute__((unused)))
241{
242	/* create initial thread  */
243	struct lthread *lt;
244
245	lthread_create(&lt, -1, initial_lthread, (void *) NULL);
246
247	/* run the lthread scheduler */
248	lthread_run();
249
250	/* restore genuine pthread operation */
251	pthread_override_set(0);
252	return 0;
253}
254
255int main(int argc, char **argv)
256{
257	int num_sched = 0;
258
259	/* basic DPDK initialization is all that is necessary to run lthreads*/
260	int ret = rte_eal_init(argc, argv);
261
262	if (ret < 0)
263		rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n");
264
265	/* enable timer subsystem */
266	rte_timer_subsystem_init();
267
268#if DEBUG_APP
269	lthread_diagnostic_set_mask(LT_DIAG_ALL);
270#endif
271
272	/* create a scheduler on every core in the core mask
273	 * and launch an initial lthread that will spawn many more.
274	 */
275	unsigned lcore_id;
276
277	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
278		if (rte_lcore_is_enabled(lcore_id))
279			num_sched++;
280	}
281
282	/* set the number of schedulers, this forces all schedulers synchronize
283	 * before entering their main loop
284	 */
285	lthread_num_schedulers_set(num_sched);
286
287	/* launch all threads */
288	rte_eal_mp_remote_launch(lthread_scheduler, (void *)NULL, CALL_MASTER);
289
290	/* wait for threads to stop */
291	RTE_LCORE_FOREACH_SLAVE(lcore_id) {
292		rte_eal_wait_lcore(lcore_id);
293	}
294	return 0;
295}
296