dpdk-setup.sh revision 5b1ff351
1#! /bin/bash
2
3#   BSD LICENSE
4#
5#   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
6#   All rights reserved.
7#
8#   Redistribution and use in source and binary forms, with or without
9#   modification, are permitted provided that the following conditions
10#   are met:
11#
12#     * Redistributions of source code must retain the above copyright
13#       notice, this list of conditions and the following disclaimer.
14#     * Redistributions in binary form must reproduce the above copyright
15#       notice, this list of conditions and the following disclaimer in
16#       the documentation and/or other materials provided with the
17#       distribution.
18#     * Neither the name of Intel Corporation nor the names of its
19#       contributors may be used to endorse or promote products derived
20#       from this software without specific prior written permission.
21#
22#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33
34#
35# Run with "source /path/to/dpdk-setup.sh"
36#
37
38#
39# Change to DPDK directory ( <this-script's-dir>/.. ), and export it as RTE_SDK
40#
41cd $(dirname ${BASH_SOURCE[0]})/..
42export RTE_SDK=$PWD
43echo "------------------------------------------------------------------------------"
44echo " RTE_SDK exported as $RTE_SDK"
45echo "------------------------------------------------------------------------------"
46
47HUGEPGSZ=`cat /proc/meminfo  | grep Hugepagesize | cut -d : -f 2 | tr -d ' '`
48
49#
50# Application EAL parameters for setting memory options (amount/channels/ranks).
51#
52EAL_PARAMS='-n 4'
53
54#
55# Sets QUIT variable so script will finish.
56#
57quit()
58{
59	QUIT=$1
60}
61
62# Shortcut for quit.
63q()
64{
65	quit
66}
67
68#
69# Sets up environmental variables for ICC.
70#
71setup_icc()
72{
73	DEFAULT_PATH=/opt/intel/bin/iccvars.sh
74	param=$1
75	shpath=`which iccvars.sh 2> /dev/null`
76	if [ $? -eq 0 ] ; then
77		echo "Loading iccvars.sh from $shpath for $param"
78		source $shpath $param
79	elif [ -f $DEFAULT_PATH ] ; then
80		echo "Loading iccvars.sh from $DEFAULT_PATH for $param"
81		source $DEFAULT_PATH $param
82	else
83		echo "## ERROR: cannot find 'iccvars.sh' script to set up ICC."
84		echo "##     To fix, please add the directory that contains"
85		echo "##     iccvars.sh  to your 'PATH' environment variable."
86		quit
87	fi
88}
89
90#
91# Sets RTE_TARGET and does a "make install".
92#
93setup_target()
94{
95	option=$1
96	export RTE_TARGET=${TARGETS[option]}
97
98	compiler=${RTE_TARGET##*-}
99	if [ "$compiler" == "icc" ] ; then
100		platform=${RTE_TARGET%%-*}
101		if [ "$platform" == "x86_64" ] ; then
102			setup_icc intel64
103		else
104			setup_icc ia32
105		fi
106	fi
107	if [ "$QUIT" == "0" ] ; then
108		make install T=${RTE_TARGET}
109	fi
110	echo "------------------------------------------------------------------------------"
111	echo " RTE_TARGET exported as $RTE_TARGET"
112	echo "------------------------------------------------------------------------------"
113}
114
115#
116# Creates hugepage filesystem.
117#
118create_mnt_huge()
119{
120	echo "Creating /mnt/huge and mounting as hugetlbfs"
121	sudo mkdir -p /mnt/huge
122
123	grep -s '/mnt/huge' /proc/mounts > /dev/null
124	if [ $? -ne 0 ] ; then
125		sudo mount -t hugetlbfs nodev /mnt/huge
126	fi
127}
128
129#
130# Removes hugepage filesystem.
131#
132remove_mnt_huge()
133{
134	echo "Unmounting /mnt/huge and removing directory"
135	grep -s '/mnt/huge' /proc/mounts > /dev/null
136	if [ $? -eq 0 ] ; then
137		sudo umount /mnt/huge
138	fi
139
140	if [ -d /mnt/huge ] ; then
141		sudo rm -R /mnt/huge
142	fi
143}
144
145#
146# Unloads igb_uio.ko.
147#
148remove_igb_uio_module()
149{
150	echo "Unloading any existing DPDK UIO module"
151	/sbin/lsmod | grep -s igb_uio > /dev/null
152	if [ $? -eq 0 ] ; then
153		sudo /sbin/rmmod igb_uio
154	fi
155}
156
157#
158# Loads new igb_uio.ko (and uio module if needed).
159#
160load_igb_uio_module()
161{
162	if [ ! -f $RTE_SDK/$RTE_TARGET/kmod/igb_uio.ko ];then
163		echo "## ERROR: Target does not have the DPDK UIO Kernel Module."
164		echo "       To fix, please try to rebuild target."
165		return
166	fi
167
168	remove_igb_uio_module
169
170	/sbin/lsmod | grep -s uio > /dev/null
171	if [ $? -ne 0 ] ; then
172		modinfo uio > /dev/null
173		if [ $? -eq 0 ]; then
174			echo "Loading uio module"
175			sudo /sbin/modprobe uio
176		fi
177	fi
178
179	# UIO may be compiled into kernel, so it may not be an error if it can't
180	# be loaded.
181
182	echo "Loading DPDK UIO module"
183	sudo /sbin/insmod $RTE_SDK/$RTE_TARGET/kmod/igb_uio.ko
184	if [ $? -ne 0 ] ; then
185		echo "## ERROR: Could not load kmod/igb_uio.ko."
186		quit
187	fi
188}
189
190#
191# Unloads VFIO modules.
192#
193remove_vfio_module()
194{
195	echo "Unloading any existing VFIO module"
196	/sbin/lsmod | grep -s vfio > /dev/null
197	if [ $? -eq 0 ] ; then
198		sudo /sbin/rmmod vfio-pci
199		sudo /sbin/rmmod vfio_iommu_type1
200		sudo /sbin/rmmod vfio
201	fi
202}
203
204#
205# Loads new vfio-pci (and vfio module if needed).
206#
207load_vfio_module()
208{
209	remove_vfio_module
210
211	VFIO_PATH="kernel/drivers/vfio/pci/vfio-pci.ko"
212
213	echo "Loading VFIO module"
214	/sbin/lsmod | grep -s vfio_pci > /dev/null
215	if [ $? -ne 0 ] ; then
216		if [ -f /lib/modules/$(uname -r)/$VFIO_PATH ] ; then
217			sudo /sbin/modprobe vfio-pci
218		fi
219	fi
220
221	# make sure regular users can read /dev/vfio
222	echo "chmod /dev/vfio"
223	sudo chmod a+x /dev/vfio
224	if [ $? -ne 0 ] ; then
225		echo "FAIL"
226		quit
227	fi
228	echo "OK"
229
230	# check if /dev/vfio/vfio exists - that way we
231	# know we either loaded the module, or it was
232	# compiled into the kernel
233	if [ ! -e /dev/vfio/vfio ] ; then
234		echo "## ERROR: VFIO not found!"
235	fi
236}
237
238#
239# Unloads the rte_kni.ko module.
240#
241remove_kni_module()
242{
243	echo "Unloading any existing DPDK KNI module"
244	/sbin/lsmod | grep -s rte_kni > /dev/null
245	if [ $? -eq 0 ] ; then
246		sudo /sbin/rmmod rte_kni
247	fi
248}
249
250#
251# Loads the rte_kni.ko module.
252#
253load_kni_module()
254{
255    # Check that the KNI module is already built.
256	if [ ! -f $RTE_SDK/$RTE_TARGET/kmod/rte_kni.ko ];then
257		echo "## ERROR: Target does not have the DPDK KNI Module."
258		echo "       To fix, please try to rebuild target."
259		return
260	fi
261
262    # Unload existing version if present.
263	remove_kni_module
264
265    # Now try load the KNI module.
266	echo "Loading DPDK KNI module"
267	sudo /sbin/insmod $RTE_SDK/$RTE_TARGET/kmod/rte_kni.ko
268	if [ $? -ne 0 ] ; then
269		echo "## ERROR: Could not load kmod/rte_kni.ko."
270		quit
271	fi
272}
273
274#
275# Sets appropriate permissions on /dev/vfio/* files
276#
277set_vfio_permissions()
278{
279	# make sure regular users can read /dev/vfio
280	echo "chmod /dev/vfio"
281	sudo chmod a+x /dev/vfio
282	if [ $? -ne 0 ] ; then
283		echo "FAIL"
284		quit
285	fi
286	echo "OK"
287
288	# make sure regular user can access everything inside /dev/vfio
289	echo "chmod /dev/vfio/*"
290	sudo chmod 0666 /dev/vfio/*
291	if [ $? -ne 0 ] ; then
292		echo "FAIL"
293		quit
294	fi
295	echo "OK"
296
297	# since permissions are only to be set when running as
298	# regular user, we only check ulimit here
299	#
300	# warn if regular user is only allowed
301	# to memlock <64M of memory
302	MEMLOCK_AMNT=`ulimit -l`
303
304	if [ "$MEMLOCK_AMNT" != "unlimited" ] ; then
305		MEMLOCK_MB=`expr $MEMLOCK_AMNT / 1024`
306		echo ""
307		echo "Current user memlock limit: ${MEMLOCK_MB} MB"
308		echo ""
309		echo "This is the maximum amount of memory you will be"
310		echo "able to use with DPDK and VFIO if run as current user."
311		echo -n "To change this, please adjust limits.conf memlock "
312		echo "limit for current user."
313
314		if [ $MEMLOCK_AMNT -lt 65536 ] ; then
315			echo ""
316			echo "## WARNING: memlock limit is less than 64MB"
317			echo -n "## DPDK with VFIO may not be able to initialize "
318			echo "if run as current user."
319		fi
320	fi
321}
322
323#
324# Removes all reserved hugepages.
325#
326clear_huge_pages()
327{
328	echo > .echo_tmp
329	for d in /sys/devices/system/node/node? ; do
330		echo "echo 0 > $d/hugepages/hugepages-${HUGEPGSZ}/nr_hugepages" >> .echo_tmp
331	done
332	echo "Removing currently reserved hugepages"
333	sudo sh .echo_tmp
334	rm -f .echo_tmp
335
336	remove_mnt_huge
337}
338
339#
340# Creates hugepages.
341#
342set_non_numa_pages()
343{
344	clear_huge_pages
345
346	echo ""
347	echo "  Input the number of ${HUGEPGSZ} hugepages"
348	echo "  Example: to have 128MB of hugepages available in a 2MB huge page system,"
349	echo "  enter '64' to reserve 64 * 2MB pages"
350	echo -n "Number of pages: "
351	read Pages
352
353	echo "echo $Pages > /sys/kernel/mm/hugepages/hugepages-${HUGEPGSZ}/nr_hugepages" > .echo_tmp
354
355	echo "Reserving hugepages"
356	sudo sh .echo_tmp
357	rm -f .echo_tmp
358
359	create_mnt_huge
360}
361
362#
363# Creates hugepages on specific NUMA nodes.
364#
365set_numa_pages()
366{
367	clear_huge_pages
368
369	echo ""
370	echo "  Input the number of ${HUGEPGSZ} hugepages for each node"
371	echo "  Example: to have 128MB of hugepages available per node in a 2MB huge page system,"
372	echo "  enter '64' to reserve 64 * 2MB pages on each node"
373
374	echo > .echo_tmp
375	for d in /sys/devices/system/node/node? ; do
376		node=$(basename $d)
377		echo -n "Number of pages for $node: "
378		read Pages
379		echo "echo $Pages > $d/hugepages/hugepages-${HUGEPGSZ}/nr_hugepages" >> .echo_tmp
380	done
381	echo "Reserving hugepages"
382	sudo sh .echo_tmp
383	rm -f .echo_tmp
384
385	create_mnt_huge
386}
387
388#
389# Run unit test application.
390#
391run_test_app()
392{
393	echo ""
394	echo "  Enter hex bitmask of cores to execute test app on"
395	echo "  Example: to execute app on cores 0 to 7, enter 0xff"
396	echo -n "bitmask: "
397	read Bitmask
398	echo "Launching app"
399	sudo ${RTE_TARGET}/app/test -c $Bitmask $EAL_PARAMS
400}
401
402#
403# Run unit testpmd application.
404#
405run_testpmd_app()
406{
407	echo ""
408	echo "  Enter hex bitmask of cores to execute testpmd app on"
409	echo "  Example: to execute app on cores 0 to 7, enter 0xff"
410	echo -n "bitmask: "
411	read Bitmask
412	echo "Launching app"
413	sudo ${RTE_TARGET}/app/testpmd -c $Bitmask $EAL_PARAMS -- -i
414}
415
416#
417# Print hugepage information.
418#
419grep_meminfo()
420{
421	grep -i huge /proc/meminfo
422}
423
424#
425# Calls dpdk-devbind.py --status to show the NIC and what they
426# are all bound to, in terms of drivers.
427#
428show_nics()
429{
430	if [ -d /sys/module/vfio_pci -o -d /sys/module/igb_uio ]; then
431		${RTE_SDK}/tools/dpdk-devbind.py --status
432	else
433		echo "# Please load the 'igb_uio' or 'vfio-pci' kernel module before "
434		echo "# querying or adjusting NIC device bindings"
435	fi
436}
437
438#
439# Uses dpdk-devbind.py to move devices to work with vfio-pci
440#
441bind_nics_to_vfio()
442{
443	if [ -d /sys/module/vfio_pci ]; then
444		${RTE_SDK}/tools/dpdk-devbind.py --status
445		echo ""
446		echo -n "Enter PCI address of device to bind to VFIO driver: "
447		read PCI_PATH
448		sudo ${RTE_SDK}/tools/dpdk-devbind.py -b vfio-pci $PCI_PATH &&
449			echo "OK"
450	else
451		echo "# Please load the 'vfio-pci' kernel module before querying or "
452		echo "# adjusting NIC device bindings"
453	fi
454}
455
456#
457# Uses dpdk-devbind.py to move devices to work with igb_uio
458#
459bind_nics_to_igb_uio()
460{
461	if [ -d /sys/module/igb_uio ]; then
462		${RTE_SDK}/tools/dpdk-devbind.py --status
463		echo ""
464		echo -n "Enter PCI address of device to bind to IGB UIO driver: "
465		read PCI_PATH
466		sudo ${RTE_SDK}/tools/dpdk-devbind.py -b igb_uio $PCI_PATH && echo "OK"
467	else
468		echo "# Please load the 'igb_uio' kernel module before querying or "
469		echo "# adjusting NIC device bindings"
470	fi
471}
472
473#
474# Uses dpdk-devbind.py to move devices to work with kernel drivers again
475#
476unbind_nics()
477{
478	${RTE_SDK}/tools/dpdk-devbind.py --status
479	echo ""
480	echo -n "Enter PCI address of device to unbind: "
481	read PCI_PATH
482	echo ""
483	echo -n "Enter name of kernel driver to bind the device to: "
484	read DRV
485	sudo ${RTE_SDK}/tools/dpdk-devbind.py -b $DRV $PCI_PATH && echo "OK"
486}
487
488#
489# Options for building a target. Note that this step MUST be first as it sets
490# up TARGETS[] starting from 1, and this is accessed in setup_target using the
491# user entered option.
492#
493step1_func()
494{
495	TITLE="Select the DPDK environment to build"
496	CONFIG_NUM=1
497	for cfg in config/defconfig_* ; do
498		cfg=${cfg/config\/defconfig_/}
499		TEXT[$CONFIG_NUM]="$cfg"
500		TARGETS[$CONFIG_NUM]=$cfg
501		FUNC[$CONFIG_NUM]="setup_target"
502		let "CONFIG_NUM+=1"
503	done
504}
505
506#
507# Options for setting up environment.
508#
509step2_func()
510{
511	TITLE="Setup linuxapp environment"
512
513	TEXT[1]="Insert IGB UIO module"
514	FUNC[1]="load_igb_uio_module"
515
516	TEXT[2]="Insert VFIO module"
517	FUNC[2]="load_vfio_module"
518
519	TEXT[3]="Insert KNI module"
520	FUNC[3]="load_kni_module"
521
522	TEXT[4]="Setup hugepage mappings for non-NUMA systems"
523	FUNC[4]="set_non_numa_pages"
524
525	TEXT[5]="Setup hugepage mappings for NUMA systems"
526	FUNC[5]="set_numa_pages"
527
528	TEXT[6]="Display current Ethernet device settings"
529	FUNC[6]="show_nics"
530
531	TEXT[7]="Bind Ethernet device to IGB UIO module"
532	FUNC[7]="bind_nics_to_igb_uio"
533
534	TEXT[8]="Bind Ethernet device to VFIO module"
535	FUNC[8]="bind_nics_to_vfio"
536
537	TEXT[9]="Setup VFIO permissions"
538	FUNC[9]="set_vfio_permissions"
539}
540
541#
542# Options for running applications.
543#
544step3_func()
545{
546	TITLE="Run test application for linuxapp environment"
547
548	TEXT[1]="Run test application (\$RTE_TARGET/app/test)"
549	FUNC[1]="run_test_app"
550
551	TEXT[2]="Run testpmd application in interactive mode (\$RTE_TARGET/app/testpmd)"
552	FUNC[2]="run_testpmd_app"
553}
554
555#
556# Other options
557#
558step4_func()
559{
560	TITLE="Other tools"
561
562	TEXT[1]="List hugepage info from /proc/meminfo"
563	FUNC[1]="grep_meminfo"
564
565}
566
567#
568# Options for cleaning up the system
569#
570step5_func()
571{
572	TITLE="Uninstall and system cleanup"
573
574	TEXT[1]="Unbind NICs from IGB UIO or VFIO driver"
575	FUNC[1]="unbind_nics"
576
577	TEXT[2]="Remove IGB UIO module"
578	FUNC[2]="remove_igb_uio_module"
579
580	TEXT[3]="Remove VFIO module"
581	FUNC[3]="remove_vfio_module"
582
583	TEXT[4]="Remove KNI module"
584	FUNC[4]="remove_kni_module"
585
586	TEXT[5]="Remove hugepage mappings"
587	FUNC[5]="clear_huge_pages"
588}
589
590STEPS[1]="step1_func"
591STEPS[2]="step2_func"
592STEPS[3]="step3_func"
593STEPS[4]="step4_func"
594STEPS[5]="step5_func"
595
596QUIT=0
597
598while [ "$QUIT" == "0" ]; do
599	OPTION_NUM=1
600
601	for s in $(seq ${#STEPS[@]}) ; do
602		${STEPS[s]}
603
604		echo "----------------------------------------------------------"
605		echo " Step $s: ${TITLE}"
606		echo "----------------------------------------------------------"
607
608		for i in $(seq ${#TEXT[@]}) ; do
609			echo "[$OPTION_NUM] ${TEXT[i]}"
610			OPTIONS[$OPTION_NUM]=${FUNC[i]}
611			let "OPTION_NUM+=1"
612		done
613
614		# Clear TEXT and FUNC arrays before next step
615		unset TEXT
616		unset FUNC
617
618		echo ""
619	done
620
621	echo "[$OPTION_NUM] Exit Script"
622	OPTIONS[$OPTION_NUM]="quit"
623	echo ""
624	echo -n "Option: "
625	read our_entry
626	echo ""
627	${OPTIONS[our_entry]} ${our_entry}
628
629	if [ "$QUIT" == "0" ] ; then
630		echo
631		echo -n "Press enter to continue ..."; read
632	fi
633
634done
635