vhost.sh revision 08e0618f
1#!/bin/bash -e
2
3CD="$( cd "$( dirname $0 )" && pwd )"
4
5CONFIG_FILE=""
6
7TMP_DIR="${CD}/tmp/"
8VMDIR="${CD}/tmp/vmdir/"
9VMMOUNT="${CD}/vmroot/"
10VMWORK="${CD}/tmp/work/"
11
12BRNAME="vhtestbr0"
13VMTAP="vhtesttap0"
14VPPSCREEN="vhtestvpp"
15
16VM_ROOT="/"
17VM_VNCPORT="3555"
18
19# Those variables are found after parsing the conf
20VPP_BUILD="xxx" 
21VPP_INSTALL="xxx"
22VPP="xxx"
23DPDK_BIND="xxx"
24
25CORES_VM_LIST="xxx"
26CORES_VM_N="xxx"
27declare -a CORES_VM_ARRAY
28CORES_VPP_LIST="xxx"
29CORES_VPP_N="xxx"
30declare -a CORES_VPP_ARRAY
31
32function validate_parameter() {
33	for c in $@; do
34		[ "${!c}" = "" ] && echo "Configuration paramater $c is not set in $CONFIG_FILE" && return 1
35	done
36	return 0
37}
38
39function validate_directory() {
40	for c in $@; do
41		[ ! -d "${!c}" ] && echo "$c=${!c} is not a directory" && return 1
42	done
43	return 0
44}
45
46function validate_file() {
47	for c in $@; do
48		[ ! -f "${!c}" ] && echo "$c=${!c} is not a file" && return 1
49	done
50	return 0
51}
52
53function validate_exec() {
54	for c in $@; do
55		[ ! -e "${!c}" -a "$(echo ${!c} | grep '/')" = "" -a "$(which ${!c})" != "" ] && eval $c=$(which ${!c})
56		[ ! -x "${!c}" ] && echo "$c=${!c} is not executable" && return 1
57	done
58	return 0
59}
60
61function validate_cores() {
62	for c in $@; do
63		LIST_NAME="${c}_LIST"
64		N_NAME="${c}_N"
65		ARRAY_NAME="${c}_ARRAY"
66		eval $LIST_NAME=\"$(echo ${!c} | sed 's/,/ /g')\"
67		COUNT=0
68		for cid in ${!LIST_NAME}; do
69			if ! [[ "$cid" =~ ^[0-9]+$ ]]; then
70				echo "'$cid' is not a valid core ID"
71				return 1
72			fi
73			eval $ARRAY_NAME[$COUNT]=$cid
74			COUNT=$(expr $COUNT + 1)
75		done
76		eval $N_NAME=$COUNT
77		#echo $LIST_NAME=${!LIST_NAME}
78		#echo $N_NAME=${!N_NAME}
79	done
80}
81
82function install_directory() {
83	for c in $@; do
84		echo "mkdir ${!c}"
85		[ ! -d "${!c}" ] && mkdir -p ${!c}
86	done
87	for c in $@; do
88		[ ! -d "${!c}" ] && return 1
89	done
90	return 0
91}
92
93function load_config() {
94	if [ "$CONFIG_FILE" != "" ]; then
95		CONFIG_FILE="$CONFIG_FILE"
96	elif [ -f "${CD}/conf.sh" ]; then
97		CONFIG_FILE="${CD}/conf.sh"
98	else
99		CONFIG_FILE="${CD}/conf.sh.default"
100	fi
101	. $CONFIG_FILE
102	
103	#Validate config
104	validate_parameter VPP_DIR VPP_IF0_PCI VPP_IF0_MAC VPP_IF1_PCI VPP_IF1_MAC VPP_IF0_NAME VPP_IF1_NAME \
105						QEMU VM_ROOT VM_INITRD VM_VMLINUZ VM_VNCPORT VM_USERNAME
106	validate_directory VPP_DIR VM_ROOT
107	validate_file VM_INITRD VM_VMLINUZ
108	validate_exec QEMU
109	validate_cores CORES_VM CORES_VPP
110	
111	[ ! -d "$VPP_DIR/vnet" ] && echo "VPP_DIR=$VPP_DIR is not VPP source directory" && exit 5
112	[ "$($QEMU --version | grep QEMU)" = "" ] && echo "$QEMU is probably not a qemu executable" && exit 6
113	
114	if [ "$VPP_BUILD" = "release" ] ; then
115		VPP_INSTALL="$VPP_DIR/build-root/install-vpp-native/"
116		VPP_BUILD="$VPP_DIR/build-root/build-vpp-native/"
117	elif [ "$VPP_BUILD" = "debug" ] ; then
118		VPP_INSTALL="$VPP_DIR/build-root/install-vpp_debug-native/"
119		VPP_BUILD="$VPP_DIR/build-root/build-vpp-native/"
120	else
121		echo "Invalid VPP_BUILD parameter '$VPP_BUILD'" && exit 1
122	fi
123	
124	if [ "$QUEUES" != "1" -a "$QUEUES" != "2" ]; then
125		echo "QUEUES can only be 1 or 2"
126		exit 7
127	fi
128	
129	VPP="$VPP_INSTALL/vpp/bin/vpp"
130	DPDK_BIND="$(ls $VPP_BUILD/dpdk/dpdk-*/tools/dpdk-devbind.py | head -n 1)"
131	
132	validate_exec VPP DPDK_BIND
133
134	return 0
135}
136
137function banner() {
138	echo "-------------------------------------"
139	echo "        VPP vhost test - BETA"
140	echo "-------------------------------------"
141	echo "VPP_DIR   : $VPP_DIR"
142	echo "VPP       : $VPP"
143	echo "DPDK_BIND : $DPDK_BIND"
144	echo "Qemu      : $QEMU"
145	echo "Qemu      : $($QEMU --version)"
146	echo "VM cores  : ${CORES_VM_ARRAY[*]}"
147	echo "VPP cores : ${CORES_VPP_ARRAY[*]}"
148	echo "-------------------------------------"
149}
150
151function vmdir_umount() {
152	sleep 0.5
153	sudo umount $VMMOUNT
154	sleep 0.5
155	rmdir "$VMMOUNT"
156}
157
158function vmdir_mount() {
159	mkdir -p "$VMDIR"
160	mkdir -p "$VMMOUNT"
161	mkdir -p "$VMWORK"
162	sudo mount -t overlayfs -o lowerdir=${VM_ROOT},workdir=${VMWORK},upperdir=${VMDIR} overlayfs ${VMMOUNT}
163}
164
165function clean() {
166	#Cleaning
167	vmdir_umount > /dev/null 2>&1 || echo -n "" #Just make sure it's not running
168	if [ ! -d "$TMP_DIR" ]; then
169		echo "$TMP_DIR"
170		mkdir -p "$TMP_DIR"
171		touch "$TMP_DIR/.vpp-vhost-test-safety"
172	elif [ ! -f "$TMP_DIR/.vpp-vhost-test-safety" ]; then
173		echo "Error: I will not remove tmp directory as there is no safety file: $TMP_DIR/.vpp-vhost-test-safety"
174		echo "Please do 'touch $TMP_DIR/.vpp-vhost-test-safety' if you are sure the content of this directory can be removed"
175		exit 7
176	else
177		sudo rm -rf $TMP_DIR/
178		mkdir -p "$TMP_DIR"
179		touch "$TMP_DIR/.vpp-vhost-test-safety"
180	fi
181}
182
183function prepare_testpmd() {
184	#Set the VM in testpmd mode
185	cat > "$VMDIR/etc/startup.d/testpmd.sh" << EOF
186#!/bin/sh
187sysctl -w vm.nr_hugepages=1024
188mkdir -p /mnt/huge
189mount -t hugetlbfs none /mnt/huge
190modprobe uio
191insmod ${VPP_INSTALL}/dpdk/kmod/igb_uio.ko
192$DPDK_BIND -b igb_uio 00:07.0
193$DPDK_BIND -b igb_uio 00:08.0
194#gdb -ex run --args
195screen -d -m     ${VPP_INSTALL}/dpdk/app/testpmd -l 0,1,2,3,4 --master-lcore 0 --socket-mem 512 --proc-type auto --file-prefix pg -w 0000:00:07.0 -w 0000:00:08.0 -- --disable-hw-vlan --rxq=$QUEUES --txq=$QUEUES --rxd=256 --txd=256 --auto-start --nb-cores=4  --eth-peer=0,aa:aa:aa:aa:bb:b1 --eth-peer=1,aa:aa:aa:aa:bb:b2 --port-topology=chained
196#--log-level 10
197#--rxq=2 --txq=2
198
199for i in \$(ls /proc/irq/ | grep [0-9]); do echo 1 > /proc/irq/\$i/smp_affinity ; done
200echo "0" > /proc/sys/kernel/watchdog_cpumask
201EOF
202sudo chmod u+x "$VMDIR/etc/startup.d/testpmd.sh"
203}
204
205function prepare_vm() {
206	#Generate VM configuration files in $VMDIR
207	mkdir -p "$VMDIR/etc/network"
208	echo "vpp-vhost-test-vm" > "$VMDIR/etc/hostname"
209	
210	cat > "$VMDIR/etc/hosts" << EOF
211127.0.0.1    localhost.localdomain localhost
212127.0.1.1    $vpp-vhost-test-vm
213EOF
214	
215	cat > "$VMDIR/etc/rc.local" << EOF
216#!/bin/sh
217mkdir -p /var/log/startup/
218for exe in \`ls /etc/startup.d\`; do
219  echo -n "Startup script \$exe    "
220  ( (nohup /etc/startup.d/\$exe > /var/log/startup/\$exe 2>&1 &) && echo "[OK]") || echo "[Failed]"
221done
222exit 0
223EOF
224	sudo chmod a+x "$VMDIR/etc/rc.local"
225	
226	mkdir -p $VMDIR/etc/udev/rules.d/
227	cat > "$VMDIR/etc/udev/rules.d/70-persistent-net.rules" << EOF
228SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="00:00:00:10:10:10", ATTR{type}=="1", KERNEL=="eth*", NAME="eth0"
229SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="de:ad:be:ef:01:00", ATTR{type}=="1", KERNEL=="eth*", NAME="vhost0"
230SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="de:ad:be:ef:01:01", ATTR{type}=="1", KERNEL=="eth*", NAME="vhost1"
231EOF
232	cat > "$VMDIR/etc/fstab" << EOF
233/dev/sda1 /               ext4    errors=remount-ro 0       1
234EOF
235
236	cat > "$VMDIR/etc/network/interfaces" << EOF
237auto lo
238iface lo inet loopback
239auto eth0
240iface eth0 inet manual
241iface eth0 inet6 static
242	address fd00::1
243	netmask 64
244EOF
245
246	mkdir -p "$VMDIR/etc/startup.d"
247	cat > "$VMDIR/etc/startup.d/dmesg.sh" << EOF
248#!/bin/sh
249while [ "1" = "1" ]; do
250	dmesg > /var/log/startup/dmesg
251	sleep 10
252	echo "--------------------"
253done
254EOF
255	chmod u+x $VMDIR/etc/startup.d/*.sh
256	mkdir -p "$VMDIR/etc/default"
257	cat > "$VMDIR/etc/default/irqbalance" << EOF
258ENABLED="0"
259ONESHOT="0"
260EOF
261
262	prepare_testpmd
263	
264	MQ=""
265	if [ "$QUEUES" != "1" ]; then
266		MQ=",mq=on"
267	fi
268	LAST_VM_CPU="$(expr $CORES_VM_N - 1)"
269	
270	cat << EOF >  "$TMP_DIR/vm.conf"
271-enable-kvm -machine pc -initrd $VM_INITRD -kernel $VM_VMLINUZ -vnc 127.0.0.1:1 -m 4G
272-append 'root=ro ro rootfstype=9p rootflags=trans=virtio nohz_full=1-$LAST_VM_CPU isolcpus=1-$LAST_VM_CPU rcu_nocbs=1-$LAST_VM_CPU selinux=0 audit=0 net.ifnames=0 biosdevname=0'
273-cpu host -smp $CORES_VM_N
274-device e1000,netdev=network0,mac=00:00:00:10:10:10,bus=pci.0,addr=3.0
275-netdev tap,id=network0,ifname=$VMTAP,script=no,downscript=no
276-fsdev local,security_model=none,id=fsdev_id,path=${VMMOUNT}
277-device virtio-9p-pci,id=dev_fs,fsdev=fsdev_id,mount_tag=ro
278-daemonize -pidfile $TMP_DIR/qemu.pid
279
280-chardev socket,id=chr0,path=$TMP_DIR/sock0,server
281-netdev type=vhost-user,id=thrnet0,chardev=chr0,queues=$QUEUES
282-device virtio-net-pci,netdev=thrnet0,mac=de:ad:be:ef:01:00,bus=pci.0,addr=7.0${MQ}
283-object memory-backend-file,id=mem,size=4096M,mem-path=/mnt/huge,share=on
284-numa node,memdev=mem
285-chardev socket,id=chr1,path=$TMP_DIR/sock1,server
286-netdev type=vhost-user,id=thrnet1,chardev=chr1,queues=$QUEUES
287-device virtio-net-pci,netdev=thrnet1,mac=de:ad:be:ef:01:01,bus=pci.0,addr=8.0${MQ}
288EOF
289}
290
291function prepare_vpp() {
292	VPP_CPU=""
293	VPP_DEV0=""
294	VPP_DEV1=""
295	ENABLE_MULTIQUEUE=""
296	if [ "$CORES_VPP_N" = "0" ]; then
297		VPP_CPU=""
298	elif [ "$CORES_VPP_N" = "1" ]; then
299		VPP_CPU="corelist-workers ${CORES_VPP_ARRAY[0]}"
300	elif [ "$CORES_VPP_N" -lt "4" ]; then
301		VPP_CPU="corelist-workers ${CORES_VPP_ARRAY[0]},${CORES_VPP_ARRAY[1]}"
302		VPP_DEV0="workers 0"
303		VPP_DEV1="workers 1"
304	else
305		VPP_CPU="corelist-workers ${CORES_VPP_ARRAY[0]},${CORES_VPP_ARRAY[1]},${CORES_VPP_ARRAY[2]},${CORES_VPP_ARRAY[3]}"
306		VPP_DEV0="workers 0,1"
307		VPP_DEV1="workers 2,3"
308		ENABLE_MULTIQUEUE="1"
309	fi
310	
311	if [ "$QUEUES" = "2" -a "$ENABLE_MULTIQUEUE" = "1" ]; then
312		VPP_DEV0="$VPP_DEV0 num-rx-queues 2 num-tx-queues 2"
313		VPP_DEV1="$VPP_DEV1 num-rx-queues 2 num-tx-queues 2"
314	fi
315	
316	cat << EOF > "$TMP_DIR/vpp.cmdline"
317cpu { $VPP_CPU }
318unix { startup-config $TMP_DIR/vpp.conf interactive }
319dpdk { dev $VPP_IF0_PCI { $VPP_DEV0 } dev $VPP_IF1_PCI { $VPP_DEV1 } }
320EOF
321
322	cat << EOF >  "$TMP_DIR/vpp.conf"
323create vhost-user socket $TMP_DIR/sock0 hwaddr aa:aa:aa:aa:bb:b1
324create vhost-user socket $TMP_DIR/sock1 hwaddr aa:aa:aa:aa:bb:b2
325set interface l2 xconnect VirtualEthernet0/0/0 $VPP_IF0_NAME
326set interface l2 xconnect VirtualEthernet0/0/1 $VPP_IF1_NAME
327set interface l2 xconnect $VPP_IF0_NAME VirtualEthernet0/0/0
328set interface l2 xconnect $VPP_IF1_NAME VirtualEthernet0/0/1
329set interface state VirtualEthernet0/0/0 up
330set interface state VirtualEthernet0/0/1 up
331set interface state $VPP_IF1_NAME up
332set interface state $VPP_IF0_NAME up
333EOF
334
335	#VHOST queue pining
336	if [ "$QUEUES" = "1" -a "$USE_DEFAULT_VHOST_PLACEMENT" != "1" ]; then
337		if [ "$CORES_VPP_N" = "0" ]; then
338			echo -n ""
339		elif [ "$CORES_VPP_N" = "1" ]; then
340			cat << EOF >> "$TMP_DIR/vpp.conf"
341vhost thread VirtualEthernet0/0/1 1
342vhost thread VirtualEthernet0/0/0 1
343EOF
344		elif [ "$CORES_VPP_N" -lt "4" ]; then
345			cat << EOF >> "$TMP_DIR/vpp.conf"
346vhost thread VirtualEthernet0/0/1 2
347vhost thread VirtualEthernet0/0/0 1
348EOF
349		else
350			cat << EOF >> "$TMP_DIR/vpp.conf"
351vhost thread VirtualEthernet0/0/1 3
352vhost thread VirtualEthernet0/0/0 4
353EOF
354		fi
355	elif [ "$QUEUES" = "2" -a "$USE_DEFAULT_VHOST_PLACEMENT" != "1" ]; then
356		if [ "$CORES_VPP_N" = "0" ]; then
357			echo -n ""
358		elif [ "$CORES_VPP_N" = "1" ]; then
359			cat << EOF >> "$TMP_DIR/vpp.conf"
360vhost thread VirtualEthernet0/0/1 1
361vhost thread VirtualEthernet0/0/1 1
362vhost thread VirtualEthernet0/0/0 1
363vhost thread VirtualEthernet0/0/0 1
364EOF
365		elif [ "$CORES_VPP_N" -lt "4" ]; then
366			cat << EOF >> "$TMP_DIR/vpp.conf"
367vhost thread VirtualEthernet0/0/1 2
368vhost thread VirtualEthernet0/0/1 2
369vhost thread VirtualEthernet0/0/0 1
370vhost thread VirtualEthernet0/0/0 1
371EOF
372		else
373			cat << EOF >> "$TMP_DIR/vpp.conf"
374vhost thread VirtualEthernet0/0/1 3
375vhost thread VirtualEthernet0/0/1 4
376vhost thread VirtualEthernet0/0/0 1
377vhost thread VirtualEthernet0/0/0 2
378EOF
379		fi
380	fi
381
382}
383
384function prepare() {
385	#Generate all configuration and VM files
386	clean
387	prepare_vm
388	prepare_vpp
389}
390
391function start_vpp() {
392	GDB=""
393	if [ "$VPP_GDB" = "1" ]; then
394		[ -e "$TMP_DIR/vpp.sh.gdbinit" ] && sudo rm "$TMP_DIR/vpp.sh.gdbinit"
395		cat << EOF > "$TMP_DIR/vpp.sh.gdbinit"
396handle SIGUSR1 nostop
397handle SIGUSR1 noprint
398set print thread-events off
399run
400EOF
401		GDB="gdb -x $TMP_DIR/vpp.sh.gdbinit --args "
402	fi
403	
404	echo "------- Starting VPP --------"
405	echo "   Screen $VPPSCREEN (sudo screen -r $VPPSCREEN)"
406	echo "   Command-line Conf:"
407	cat $TMP_DIR/vpp.cmdline
408	echo "   CLI Conf:"
409	cat $TMP_DIR/vpp.conf
410	echo "-----------------------------"
411	
412	sudo screen -d -m -S "$VPPSCREEN" $GDB $VPP_DIR/build-root/install-vpp-native/vpp/bin/vpp -c $TMP_DIR/vpp.cmdline
413}
414
415function start_vm() {
416	echo "------- Starting VM --------"
417	echo "   VM conf:"
418	cat $TMP_DIR/vm.conf
419	echo "----------------------------"
420	
421	#Eval is used so that ' characters are not ignored
422	eval sudo chrt -rr 1 taskset -c $CORES_VM $QEMU $(cat $TMP_DIR/vm.conf)
423	
424	echo "Started QEMU with PID $(sudo cat $TMP_DIR/qemu.pid)"
425	
426	sudo brctl addbr $BRNAME
427	sudo ip link set $BRNAME up
428	sudo ip addr add fd00::ffff/64 dev $BRNAME
429	sudo brctl addif $BRNAME $VMTAP
430	sudo ip link set $VMTAP up
431}
432
433function start() {
434	if [ -f "$TMP_DIR/.started" ]; then
435		echo "$TMP_DIR/.started exists"
436		echo "This means the setup is probably running already."
437		echo "Please stop the setup first, or remove this file."
438		exit 2
439	fi
440	
441	banner
442	
443	prepare
444	
445	touch "$TMP_DIR/.started"
446	
447	echo "0" | sudo tee /proc/sys/kernel/watchdog_cpumask
448	
449	start_vpp
450	
451	vmdir_mount
452	
453	start_vm
454}
455
456function pin_vm() {
457	PIDS=$(ps -eLf | grep  qemu-system-x86_64 | awk '$5 > 50 { print $4; }')
458	idx=1
459	for p in $PIDS; do
460		if [ "${CORES_VM_ARRAY[$idx]}" = "" ]; then
461			echo "Too many working threads in VM"
462			return 1
463		fi
464		echo "VM PID $p on core ${CORES_VM_ARRAY[$idx]}"
465		sudo taskset -pc ${CORES_VM_ARRAY[$idx]} $p && sudo chrt -r -p 1 $p
466		idx=$(expr $idx + 1)
467	done
468}
469
470function pin_vpp() {
471	
472	for i in $(ls /proc/irq/ | grep [0-9]); do 
473		echo 1 | sudo tee /proc/irq/$i/smp_affinity > /dev/null || true ; 
474	done
475	
476	
477	PIDS=$(ps -eLf | grep  /bin/vpp | awk '$5 > 50 { print $4; }')
478	idx=0
479	for p in $PIDS; do
480		if [ "${CORES_VPP_ARRAY[$idx]}" = "" ]; then
481			echo "Too many working threads in VPP"
482			return 1
483		fi
484		echo "VPP PID $p on core ${CORES_VPP_ARRAY[$idx]}"
485		sudo taskset -pc ${CORES_VPP_ARRAY[$idx]} $p && sudo chrt -r -p 1 $p
486		idx=$(expr $idx + 1)
487	done
488}
489
490function pin() {
491	pin_vm
492	pin_vpp
493}
494
495function stop() {
496	set +e
497	
498	[ -f "$TMP_DIR/qemu.pid" ] && echo "Stopping VM ($(sudo cat $TMP_DIR/qemu.pid))" && sudo kill "$(sudo cat $TMP_DIR/qemu.pid)" && sudo rm $TMP_DIR/qemu.pid
499	
500	vmdir_umount
501	
502	sudo ip link set $BRNAME down
503	sudo brctl delbr $BRNAME
504	
505	sudo screen -S "$VPPSCREEN" -X quit && echo "Stopping VPP"
506	
507	[ -f "$TMP_DIR/.started" ] && rm "$TMP_DIR/.started"
508}
509
510function cmd_openvnc() {
511	load_config
512    echo Please VNC to 5900 to connect to this VM console
513    socat TCP6-LISTEN:5900,reuseaddr TCP:localhost:5901 &
514}
515
516function cmd_start() {
517	load_config
518	start
519}
520
521function cmd_pin() {
522	load_config
523	pin
524}
525
526function cmd_turbo_disable() {
527	load_config
528	sudo modprobe msr
529	echo "Disabling turboboost on CPUs $CORES_VPP_LIST $CORES_VM_LIST"
530	for cpu in $CORES_VPP_LIST $CORES_VM_LIST ; do
531		sudo wrmsr -p ${cpu} 0x1a0 0x4000850089
532	done
533}
534
535function cmd_turbo_enable() {
536	load_config
537	sudo modprobe msr
538	echo "Enabling turboboost on CPUs $CORES_VPP_LIST $CORES_VM_LIST"
539	for cpu in $CORES_VPP_LIST $CORES_VM_LIST ; do
540		sudo wrmsr -p ${cpu} 0x1a0 0x850089
541	done
542}
543
544function cmd_stop() {
545	load_config
546	stop
547}
548
549function cmd_clean() {
550	load_config
551	clean
552}
553
554function cmd_ssh() {
555	load_config
556	ssh ${VM_USERNAME}@fd00::1
557}
558
559function cmd_config() {
560	load_config
561}
562
563[ "$1" = "" ] && echo "Missing arguments" && usage && exit 1
564CMD="$1"
565shift
566eval "cmd_$CMD" "$@"
567
568