main.c revision 9365d6cf
1
2/*-
3 *   BSD LICENSE
4 *
5 *   Copyright(c) 2015 Intel Corporation. All rights reserved.
6 *   All rights reserved.
7 *
8 *   Redistribution and use in source and binary forms, with or without
9 *   modification, are permitted provided that the following conditions
10 *   are met:
11 *
12 *     * Redistributions of source code must retain the above copyright
13 *       notice, this list of conditions and the following disclaimer.
14 *     * Redistributions in binary form must reproduce the above copyright
15 *       notice, this list of conditions and the following disclaimer in
16 *       the documentation and/or other materials provided with the
17 *       distribution.
18 *     * Neither the name of Intel Corporation nor the names of its
19 *       contributors may be used to endorse or promote products derived
20 *       from this software without specific prior written permission.
21 *
22 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 */
34
35#define _GNU_SOURCE
36#include <stdio.h>
37#include <stdlib.h>
38#include <stdint.h>
39#include <inttypes.h>
40#include <sys/types.h>
41#include <string.h>
42#include <sys/queue.h>
43#include <stdarg.h>
44#include <errno.h>
45#include <getopt.h>
46#include <unistd.h>
47#include <sched.h>
48#include <pthread.h>
49
50#include <rte_common.h>
51#include <rte_lcore.h>
52#include <rte_per_lcore.h>
53#include <rte_timer.h>
54
55#include "lthread_api.h"
56#include "lthread_diag_api.h"
57#include "pthread_shim.h"
58
59#define DEBUG_APP 0
60#define HELLOW_WORLD_MAX_LTHREADS 10
61
62#ifndef __GLIBC__ /* sched_getcpu() is glibc-specific */
63#define sched_getcpu() rte_lcore_id()
64#endif
65
66__thread int print_count;
67__thread pthread_mutex_t print_lock;
68
69__thread pthread_mutex_t exit_lock;
70__thread pthread_cond_t exit_cond;
71
72/*
73 * A simple thread that demonstrates use of a mutex, a condition
74 * variable, thread local storage, explicit yield, and thread exit.
75 *
76 * The thread uses a mutex to protect a shared counter which is incremented
77 * and then it waits on condition variable before exiting.
78 *
79 * The thread argument is stored in and retrieved from TLS, using
80 * the pthread key create, get and set specific APIs.
81 *
82 * The thread yields while holding the mutex, to provide opportunity
83 * for other threads to contend.
84 *
85 * All of the pthread API functions used by this thread are actually
86 * resolved to corresponding lthread functions by the pthread shim
87 * implemented in pthread_shim.c
88 */
89void *helloworld_pthread(void *arg);
90void *helloworld_pthread(void *arg)
91{
92	pthread_key_t key;
93
94	/* create a key for TLS */
95	pthread_key_create(&key, NULL);
96
97	/* store the arg in TLS */
98	pthread_setspecific(key, arg);
99
100	/* grab lock and increment shared counter */
101	pthread_mutex_lock(&print_lock);
102	print_count++;
103
104	/* yield thread to give opportunity for lock contention */
105	pthread_yield();
106
107	/* retrieve arg from TLS */
108	uint64_t thread_no = (uint64_t) pthread_getspecific(key);
109
110	printf("Hello - lcore = %d count = %d thread_no = %d thread_id = %p\n",
111			sched_getcpu(),
112			print_count,
113			(int) thread_no,
114			(void *)pthread_self());
115
116	/* release the lock */
117	pthread_mutex_unlock(&print_lock);
118
119	/*
120	 * wait on condition variable
121	 * before exiting
122	 */
123	pthread_mutex_lock(&exit_lock);
124	pthread_cond_wait(&exit_cond, &exit_lock);
125	pthread_mutex_unlock(&exit_lock);
126
127	/* exit */
128	pthread_exit((void *) thread_no);
129}
130
131
132/*
133 * This is the initial thread
134 *
135 * It demonstrates pthread, mutex and condition variable creation,
136 * broadcast and pthread join APIs.
137 *
138 * This initial thread must always start life as an lthread.
139 *
140 * This thread creates many more threads then waits a short time
141 * before signalling them to exit using a broadcast.
142 *
143 * All of the pthread API functions used by this thread are actually
144 * resolved to corresponding lthread functions by the pthread shim
145 * implemented in pthread_shim.c
146 *
147 * After all threads have finished the lthread scheduler is shutdown
148 * and normal pthread operation is restored
149 */
150__thread pthread_t tid[HELLOW_WORLD_MAX_LTHREADS];
151
152static void initial_lthread(void *args);
153static void initial_lthread(void *args __attribute__((unused)))
154{
155	int lcore = (int) rte_lcore_id();
156	/*
157	 *
158	 * We can now enable pthread API override
159	 * and start to use the pthread APIs
160	 */
161	pthread_override_set(1);
162
163	uint64_t i;
164
165	/* initialize mutex for shared counter */
166	print_count = 0;
167	pthread_mutex_init(&print_lock, NULL);
168
169	/* initialize mutex and condition variable controlling thread exit */
170	pthread_mutex_init(&exit_lock, NULL);
171	pthread_cond_init(&exit_cond, NULL);
172
173	/* spawn a number of threads */
174	for (i = 0; i < HELLOW_WORLD_MAX_LTHREADS; i++) {
175
176		/*
177		 * Not strictly necessary but
178		 * for the sake of this example
179		 * use an attribute to pass the desired lcore
180		 */
181		pthread_attr_t attr;
182		rte_cpuset_t cpuset;
183
184		CPU_ZERO(&cpuset);
185		CPU_SET(lcore, &cpuset);
186		pthread_attr_init(&attr);
187		pthread_attr_setaffinity_np(&attr, sizeof(rte_cpuset_t), &cpuset);
188
189		/* create the thread */
190		pthread_create(&tid[i], &attr, helloworld_pthread, (void *) i);
191	}
192
193	/* wait for 1s to allow threads
194	 * to block on the condition variable
195	 * N.B. nanosleep() is resolved to lthread_sleep()
196	 * by the shim.
197	 */
198	struct timespec time;
199
200	time.tv_sec = 1;
201	time.tv_nsec = 0;
202	nanosleep(&time, NULL);
203
204	/* wake up all the threads */
205	pthread_cond_broadcast(&exit_cond);
206
207	/* wait for them to finish */
208	for (i = 0; i < HELLOW_WORLD_MAX_LTHREADS; i++) {
209
210		uint64_t thread_no;
211
212		pthread_join(tid[i], (void *) &thread_no);
213		if (thread_no != i)
214			printf("error on thread exit\n");
215	}
216
217	pthread_cond_destroy(&exit_cond);
218	pthread_mutex_destroy(&print_lock);
219	pthread_mutex_destroy(&exit_lock);
220
221	/* shutdown the lthread scheduler */
222	lthread_scheduler_shutdown(rte_lcore_id());
223	lthread_detach();
224}
225
226
227
228/* This thread creates a single initial lthread
229 * and then runs the scheduler
230 * An instance of this thread is created on each thread
231 * in the core mask
232 */
233static int
234lthread_scheduler(void *args);
235static int
236lthread_scheduler(void *args __attribute__((unused)))
237{
238	/* create initial thread  */
239	struct lthread *lt;
240
241	lthread_create(&lt, -1, initial_lthread, (void *) NULL);
242
243	/* run the lthread scheduler */
244	lthread_run();
245
246	/* restore genuine pthread operation */
247	pthread_override_set(0);
248	return 0;
249}
250
251int main(int argc, char **argv)
252{
253	int num_sched = 0;
254
255	/* basic DPDK initialization is all that is necessary to run lthreads*/
256	int ret = rte_eal_init(argc, argv);
257
258	if (ret < 0)
259		rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n");
260
261	/* enable timer subsystem */
262	rte_timer_subsystem_init();
263
264#if DEBUG_APP
265	lthread_diagnostic_set_mask(LT_DIAG_ALL);
266#endif
267
268	/* create a scheduler on every core in the core mask
269	 * and launch an initial lthread that will spawn many more.
270	 */
271	unsigned lcore_id;
272
273	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
274		if (rte_lcore_is_enabled(lcore_id))
275			num_sched++;
276	}
277
278	/* set the number of schedulers, this forces all schedulers synchronize
279	 * before entering their main loop
280	 */
281	lthread_num_schedulers_set(num_sched);
282
283	/* launch all threads */
284	rte_eal_mp_remote_launch(lthread_scheduler, (void *)NULL, CALL_MASTER);
285
286	/* wait for threads to stop */
287	RTE_LCORE_FOREACH_SLAVE(lcore_id) {
288		rte_eal_wait_lcore(lcore_id);
289	}
290	return 0;
291}
292