vhost.sh revision ba114e80
1#!/bin/bash -e
2
3CD="$( cd "$( dirname $0 )" && pwd )"
4
5CONFIG_FILE=""
6
7TMP_DIR="${CD}/tmp/"
8VMDIR="${CD}/tmp/vmdir/"
9VMMOUNT="${CD}/vmroot/"
10VMWORK="${CD}/tmp/work/"
11
12BRNAME="vhtestbr0"
13VMTAP="vhtesttap0"
14VPPSCREEN="vhtestvpp"
15
16VM_ROOT="/"
17VM_VNCPORT="3555"
18
19# Those variables are found after parsing the conf
20VPP_BUILD="xxx" 
21VPP_INSTALL="xxx"
22VPP="xxx"
23DPDK_BIND="xxx"
24VPPCTL="xxx"
25
26CORES_VM_LIST="xxx"
27CORES_VM_N="xxx"
28declare -a CORES_VM_ARRAY
29CORES_VPP_LIST="xxx"
30CORES_VPP_N="xxx"
31declare -a CORES_VPP_ARRAY
32
33function validate_parameter() {
34	for c in $@; do
35		[ "${!c}" = "" ] && echo "Configuration paramater $c is not set in $CONFIG_FILE" && return 1
36	done
37	return 0
38}
39
40function validate_directory() {
41	for c in $@; do
42		[ ! -d "${!c}" ] && echo "$c=${!c} is not a directory" && return 1
43	done
44	return 0
45}
46
47function validate_file() {
48	for c in $@; do
49		[ ! -f "${!c}" ] && echo "$c=${!c} is not a file" && return 1
50	done
51	return 0
52}
53
54function validate_exec() {
55	for c in $@; do
56		[ ! -e "${!c}" -a "$(echo ${!c} | grep '/')" = "" -a "$(which ${!c})" != "" ] && eval $c=$(which ${!c})
57		[ ! -x "${!c}" ] && echo "$c=${!c} is not executable" && return 1
58	done
59	return 0
60}
61
62function validate_cores() {
63	for c in $@; do
64		LIST_NAME="${c}_LIST"
65		N_NAME="${c}_N"
66		ARRAY_NAME="${c}_ARRAY"
67		eval $LIST_NAME=\"$(echo ${!c} | sed 's/,/ /g')\"
68		COUNT=0
69		for cid in ${!LIST_NAME}; do
70			if ! [[ "$cid" =~ ^[0-9]+$ ]]; then
71				echo "'$cid' is not a valid core ID"
72				return 1
73			fi
74			eval $ARRAY_NAME[$COUNT]=$cid
75			COUNT=$(expr $COUNT + 1)
76		done
77		eval $N_NAME=$COUNT
78		#echo $LIST_NAME=${!LIST_NAME}
79		#echo $N_NAME=${!N_NAME}
80	done
81}
82
83function install_directory() {
84	for c in $@; do
85		echo "mkdir ${!c}"
86		[ ! -d "${!c}" ] && mkdir -p ${!c}
87	done
88	for c in $@; do
89		[ ! -d "${!c}" ] && return 1
90	done
91	return 0
92}
93
94function load_config() {
95	if [ "$CONFIG_FILE" != "" ]; then
96		CONFIG_FILE="$CONFIG_FILE"
97	elif [ -f "${CD}/conf.sh" ]; then
98		CONFIG_FILE="${CD}/conf.sh"
99	else
100		CONFIG_FILE="${CD}/conf.sh.default"
101	fi
102	. $CONFIG_FILE
103	
104	#Validate config
105	validate_parameter VPP_DIR VPP_IF0_PCI VPP_IF0_MAC VPP_IF1_PCI VPP_IF1_MAC VPP_IF0_NAME VPP_IF1_NAME \
106						QEMU VM_ROOT VM_INITRD VM_VMLINUZ VM_VNCPORT VM_USERNAME VPP_VHOST_MODE VIRTIO_QSZ
107	validate_directory VPP_DIR VM_ROOT
108	validate_file VM_INITRD VM_VMLINUZ
109	validate_exec QEMU
110	validate_cores CORES_VM CORES_VPP
111	
112	[ ! -d "$VPP_DIR/vnet" ] && echo "VPP_DIR=$VPP_DIR is not VPP source directory" && exit 5
113	[ "$($QEMU --version | grep QEMU)" = "" ] && echo "$QEMU is probably not a qemu executable" && exit 6
114	
115	if [ "$VPP_BUILD" = "release" ] ; then
116		VPP_INSTALL="$VPP_DIR/build-root/install-vpp-native/"
117		VPP_BUILD="$VPP_DIR/build-root/build-vpp-native/"
118	elif [ "$VPP_BUILD" = "debug" ] ; then
119		VPP_INSTALL="$VPP_DIR/build-root/install-vpp_debug-native/"
120		VPP_BUILD="$VPP_DIR/build-root/build-vpp-native/"
121	else
122		echo "Invalid VPP_BUILD parameter '$VPP_BUILD'" && exit 1
123	fi
124	
125	if [ "$VPP_VHOST_MODE" != "client" -a "$VPP_VHOST_MODE" != "server" ]; then
126		echo "Invalid VPP_VHOST_MODE (must be client or server)" && exit 1
127	fi
128	
129	if [ "$QUEUES" != "1" -a "$QUEUES" != "2" ]; then
130		echo "QUEUES can only be 1 or 2"
131		exit 7
132	fi
133	
134	VPP="$VPP_INSTALL/vpp/bin/vpp"
135	DPDK_BIND="$(ls $VPP_BUILD/dpdk/dpdk-*/tools/dpdk-devbind.py | head -n 1)"
136	VPPCTL="sudo env PATH=$PATH:${VPP_INSTALL}/../install-vpp_debug-native/vpp-api-test/bin/ vppctl"
137	
138	validate_exec VPP DPDK_BIND
139
140	return 0
141}
142
143function banner() {
144	echo "-------------------------------------"
145	echo "        VPP vhost test - BETA"
146	echo "-------------------------------------"
147	echo "VPP_DIR   : $VPP_DIR"
148	echo "VPP       : $VPP"
149	echo "DPDK_BIND : $DPDK_BIND"
150	echo "Qemu      : $QEMU"
151	echo "Qemu      : $($QEMU --version)"
152	echo "VM cores  : ${CORES_VM_ARRAY[*]}"
153	echo "VPP cores : ${CORES_VPP_ARRAY[*]}"
154	echo "-------------------------------------"
155}
156
157function vmdir_umount() {
158	sleep 0.5
159	sudo umount $VMMOUNT
160	sleep 0.5
161	rmdir "$VMMOUNT"
162}
163
164function vmdir_mount() {
165	mkdir -p "$VMDIR"
166	mkdir -p "$VMMOUNT"
167	mkdir -p "$VMWORK"
168	sudo mount -t overlayfs -o lowerdir=${VM_ROOT},workdir=${VMWORK},upperdir=${VMDIR} overlayfs ${VMMOUNT}
169}
170
171function clean() {
172	#Cleaning
173	vmdir_umount > /dev/null 2>&1 || echo -n "" #Just make sure it's not running
174	if [ ! -d "$TMP_DIR" ]; then
175		echo "$TMP_DIR"
176		mkdir -p "$TMP_DIR"
177		touch "$TMP_DIR/.vpp-vhost-test-safety"
178	elif [ ! -f "$TMP_DIR/.vpp-vhost-test-safety" ]; then
179		echo "Error: I will not remove tmp directory as there is no safety file: $TMP_DIR/.vpp-vhost-test-safety"
180		echo "Please do 'touch $TMP_DIR/.vpp-vhost-test-safety' if you are sure the content of this directory can be removed"
181		exit 7
182	else
183		sudo rm -rf $TMP_DIR/
184		mkdir -p "$TMP_DIR"
185		touch "$TMP_DIR/.vpp-vhost-test-safety"
186	fi
187}
188
189function prepare_testpmd() {
190	#Set the VM in testpmd mode
191	cat > "$VMDIR/etc/startup.d/testpmd.sh" << EOF
192#!/bin/sh
193sysctl -w vm.nr_hugepages=1024
194mkdir -p /mnt/huge
195mount -t hugetlbfs none /mnt/huge
196modprobe uio
197insmod ${VPP_INSTALL}/dpdk/kmod/igb_uio.ko
198$DPDK_BIND -b igb_uio 00:07.0
199$DPDK_BIND -b igb_uio 00:08.0
200#gdb -ex run --args
201screen -d -m     ${VPP_INSTALL}/dpdk/app/testpmd -l 0,1,2,3,4 --master-lcore 0 --socket-mem 512 --proc-type auto --file-prefix pg -w 0000:00:07.0 -w 0000:00:08.0 -- --disable-hw-vlan --rxq=$QUEUES --txq=$QUEUES --rxd=256 --txd=256 --auto-start --nb-cores=4  --eth-peer=0,aa:aa:aa:aa:bb:b1 --eth-peer=1,aa:aa:aa:aa:bb:b2 --port-topology=chained
202#--log-level 10
203#--rxq=2 --txq=2
204
205for i in \$(ls /proc/irq/ | grep [0-9]); do echo 1 > /proc/irq/\$i/smp_affinity ; done
206echo "0" > /proc/sys/kernel/watchdog_cpumask
207EOF
208sudo chmod u+x "$VMDIR/etc/startup.d/testpmd.sh"
209}
210
211function prepare_vm() {
212	#Generate VM configuration files in $VMDIR
213	mkdir -p "$VMDIR/etc/network"
214	echo "vpp-vhost-test-vm" > "$VMDIR/etc/hostname"
215	
216	cat > "$VMDIR/etc/hosts" << EOF
217127.0.0.1    localhost.localdomain localhost
218127.0.1.1    $vpp-vhost-test-vm
219EOF
220	
221	cat > "$VMDIR/etc/rc.local" << EOF
222#!/bin/sh
223mkdir -p /var/log/startup/
224for exe in \$(ls /etc/startup.d); do
225  echo -n "Startup script \$exe    "
226  ( (nohup /etc/startup.d/\$exe > /var/log/startup/\$exe 2>&1 &) && echo "[OK]") || echo "[Failed]"
227done
228exit 0
229EOF
230	sudo chmod a+x "$VMDIR/etc/rc.local"
231	
232	mkdir -p $VMDIR/etc/udev/rules.d/
233	cat > "$VMDIR/etc/udev/rules.d/70-persistent-net.rules" << EOF
234SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="00:00:00:10:10:10", ATTR{type}=="1", KERNEL=="eth*", NAME="eth0"
235SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="de:ad:be:ef:01:00", ATTR{type}=="1", KERNEL=="eth*", NAME="vhost0"
236SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="de:ad:be:ef:01:01", ATTR{type}=="1", KERNEL=="eth*", NAME="vhost1"
237EOF
238	cat > "$VMDIR/etc/fstab" << EOF
239/dev/sda1 /               ext4    errors=remount-ro 0       1
240EOF
241
242	cat > "$VMDIR/etc/network/interfaces" << EOF
243auto lo
244iface lo inet loopback
245auto eth0
246iface eth0 inet manual
247iface eth0 inet6 static
248	address fd00::1
249	netmask 64
250EOF
251
252	mkdir -p "$VMDIR/etc/startup.d"
253	cat > "$VMDIR/etc/startup.d/dmesg.sh" << EOF
254#!/bin/sh
255while [ "1" = "1" ]; do
256	dmesg > /var/log/startup/dmesg
257	sleep 10
258	echo "--------------------"
259done
260EOF
261	chmod u+x $VMDIR/etc/startup.d/*.sh
262	mkdir -p "$VMDIR/etc/default"
263	cat > "$VMDIR/etc/default/irqbalance" << EOF
264ENABLED="0"
265ONESHOT="0"
266EOF
267
268	prepare_testpmd
269	
270	MQ=""
271	if [ "$QUEUES" != "1" ]; then
272		MQ=",mq=on"
273	fi
274	LAST_VM_CPU="$(expr $CORES_VM_N - 1)"
275	
276	VM_VH_SERV_PARAM=""
277	if [ "$VPP_VHOST_MODE" = "client" ]; then
278		VM_VH_SERV_PARAM=",server"
279	fi
280	
281	QSZ=""
282	if [ "$VIRTIO_QSZ" != "256" ]; then
283		QSZ=",rx_virtqueue_sz=$VIRTIO_QSZ,tx_virtqueue_sz=$VIRTIO_QSZ"
284	fi
285	
286	cat << EOF >  "$TMP_DIR/vm.conf"
287-enable-kvm -machine pc -initrd $VM_INITRD -kernel $VM_VMLINUZ -vnc 127.0.0.1:1 -m 4G
288-append 'root=ro ro rootfstype=9p rootflags=trans=virtio nohz_full=1-$LAST_VM_CPU isolcpus=1-$LAST_VM_CPU rcu_nocbs=1-$LAST_VM_CPU selinux=0 audit=0 net.ifnames=0 biosdevname=0'
289-cpu host -smp $CORES_VM_N
290-device e1000,netdev=network0,mac=00:00:00:10:10:10,bus=pci.0,addr=3.0
291-netdev tap,id=network0,ifname=$VMTAP,script=no,downscript=no
292-fsdev local,security_model=none,id=fsdev_id,path=${VMMOUNT}
293-device virtio-9p-pci,id=dev_fs,fsdev=fsdev_id,mount_tag=ro
294-daemonize -pidfile $TMP_DIR/qemu.pid
295
296-chardev socket,id=chr0,path=$TMP_DIR/sock0${VM_VH_SERV_PARAM}
297-netdev type=vhost-user,id=thrnet0,chardev=chr0,queues=$QUEUES
298-device virtio-net-pci,netdev=thrnet0,mac=de:ad:be:ef:01:00,bus=pci.0,addr=7.0,mrg_rxbuf=on,indirect_desc=on${MQ}${QSZ}
299-object memory-backend-file,id=mem,size=4096M,mem-path=/mnt/huge,share=on
300-numa node,memdev=mem
301-chardev socket,id=chr1,path=$TMP_DIR/sock1${VM_VH_SERV_PARAM}
302-netdev type=vhost-user,id=thrnet1,chardev=chr1,queues=$QUEUES
303-device virtio-net-pci,netdev=thrnet1,mac=de:ad:be:ef:01:01,bus=pci.0,addr=8.0,mrg_rxbuf=on,indirect_desc=on${MQ}${QSZ}
304EOF
305}
306
307function get_vhost_thread_config() {
308	VH_IF0="$1"
309	VH_IF1="$2"
310	
311	if [ "$VH_IF0" = "" -o "$VH_IF1" = "" ]; then
312		echo "Missing get_vhost_thread_config interface argument"
313		exit 1
314	fi
315	
316	DEL=""
317	if [ "$3" = "del" ]; then
318		DEL="del"
319	elif [ "$3" != "" ]; then
320		echo "Invalid get_vhost_thread_config argument"
321		exit 1
322	fi
323	
324	#VHOST queue pining
325	if [ "$QUEUES" = "1" ]; then
326		if [ "$CORES_VPP_N" = "0" ]; then
327			echo -n ""
328		elif [ "$CORES_VPP_N" = "1" ]; then
329			echo "vhost thread $VH_IF1 1 ${DEL}"
330			echo "vhost thread $VH_IF0 1 ${DEL}"
331		elif [ "$CORES_VPP_N" -lt "4" ]; then
332			echo "vhost thread $VH_IF1 2 ${DEL}"
333			echo "vhost thread $VH_IF0 1 ${DEL}"
334		else
335			echo "vhost thread $VH_IF1 3 ${DEL}"
336			echo "vhost thread $VH_IF0 4 ${DEL}"
337		fi
338	elif [ "$QUEUES" = "2" ]; then
339		if [ "$CORES_VPP_N" = "0" ]; then
340			echo -n ""
341		elif [ "$CORES_VPP_N" = "1" ]; then
342			echo "vhost thread $VH_IF1 1 ${DEL}"
343			echo "vhost thread $VH_IF1 1 ${DEL}"
344			echo "vhost thread $VH_IF0 1 ${DEL}"
345			echo "vhost thread $VH_IF0 1 ${DEL}"
346		elif [ "$CORES_VPP_N" -lt "4" ]; then
347			echo "vhost thread $VH_IF1 2 ${DEL}"
348			echo "vhost thread $VH_IF1 2 ${DEL}"
349			echo "vhost thread $VH_IF0 1 ${DEL}"
350			echo "vhost thread $VH_IF0 1 ${DEL}"
351		else
352			echo "vhost thread $VH_IF1 3 ${DEL}"
353			echo "vhost thread $VH_IF1 4 ${DEL}"
354			echo "vhost thread $VH_IF0 1 ${DEL}"
355			echo "vhost thread $VH_IF0 2 ${DEL}"
356		fi
357	fi
358}
359
360function disconnect_vhost() {
361	VH_INST=$1
362	
363	if [ ! -f "$TMP_DIR/vpp-vhost-disconnect$VH_INST.conf" ]; then
364		echo "No configured vhost devices $VH_INST"
365		exit 1
366	fi
367	
368	echo "------- Disconnect vhost --------"
369	cat "$TMP_DIR/vpp-vhost-disconnect$VH_INST.conf"
370	echo "-------------------------------"
371	
372	$VPPCTL exec "$TMP_DIR/vpp-vhost-disconnect$VH_INST.conf"
373	rm "$TMP_DIR/vpp-vhost-disconnect$VH_INST.conf"
374}
375
376function connect_vhost() {
377	VH_INST=$1
378	
379	if [ -f "$TMP_DIR/vpp-vhost-disconnect$VH_INST.conf" ]; then
380		echo "Vhost devices $VH_INST already configured"
381		exit 1
382	fi
383	
384	VH_SERV_PARAM=""
385	if [ "$VPP_VHOST_MODE" = "server" ]; then
386		VH_SERV_PARAM="server"
387	fi
388    echo "create vhost-user socket $TMP_DIR/sock0$VH_INST $VH_SERV_PARAM hwaddr aa:aa:aa:aa:bb:b1"
389	VH_IF0=$($VPPCTL "create vhost-user socket $TMP_DIR/sock0$VH_INST $VH_SERV_PARAM hwaddr aa:aa:aa:aa:bb:b1")
390	VH_IF1=$($VPPCTL "create vhost-user socket $TMP_DIR/sock1$VH_INST $VH_SERV_PARAM hwaddr aa:aa:aa:aa:bb:b2")
391	
392	if [ "$VH_INST" = "" ]; then
393	cat << EOF >  "$TMP_DIR/vpp-vhost-connect$VH_INST.conf"
394set interface l2 xconnect $VH_IF0 $VPP_IF0_NAME
395set interface l2 xconnect $VH_IF1 $VPP_IF1_NAME
396set interface l2 xconnect $VPP_IF0_NAME $VH_IF0
397set interface l2 xconnect $VPP_IF1_NAME $VH_IF1
398set interface state $VH_IF0 up
399set interface state $VH_IF1 up
400EOF
401	else
402		cat << EOF >  "$TMP_DIR/vpp-vhost-connect$VH_INST.conf"
403set interface state $VH_IF0 up
404set interface state $VH_IF1 up
405EOF
406	fi
407
408	cat << EOF > "$TMP_DIR/vpp-vhost-disconnect$VH_INST.conf"
409delete vhost-user $VH_IF0
410delete vhost-user $VH_IF1
411EOF
412
413	if [ "$USE_DEFAULT_VHOST_PLACEMENT" != "1" ]; then
414		get_vhost_thread_config $VH_IF0 $VH_IF1 >> "$TMP_DIR/vpp-vhost-connect$VH_INST.conf"
415		#get_vhost_thread_config $VH_IF0 $VH_IF1 del >> "$TMP_DIR/vpp-vhost-disconnect$VH_INST.conf"
416	fi
417	
418	echo "------- Connect vhost --------"
419	echo "create vhost-user socket $TMP_DIR/sock0$VH_INST $VH_SERV_PARAM hwaddr aa:aa:aa:aa:bb:b1"
420	echo "create vhost-user socket $TMP_DIR/sock1$VH_INST $VH_SERV_PARAM hwaddr aa:aa:aa:aa:bb:b2"
421	cat "$TMP_DIR/vpp-vhost-connect$VH_INST.conf"
422	echo "-------------------------------"
423	
424	$VPPCTL exec "$TMP_DIR/vpp-vhost-connect$VH_INST.conf"
425}
426
427function prepare_vpp() {
428	VPP_CPU=""
429	VPP_DEV0=""
430	VPP_DEV1=""
431	ENABLE_MULTIQUEUE=""
432	if [ "$CORES_VPP_N" = "0" ]; then
433		VPP_CPU=""
434	elif [ "$CORES_VPP_N" = "1" ]; then
435		VPP_CPU="corelist-workers ${CORES_VPP_ARRAY[0]}"
436	elif [ "$CORES_VPP_N" -lt "4" ]; then
437		VPP_CPU="corelist-workers ${CORES_VPP_ARRAY[0]},${CORES_VPP_ARRAY[1]}"
438		VPP_DEV0="workers 0"
439		VPP_DEV1="workers 1"
440	else
441		VPP_CPU="corelist-workers ${CORES_VPP_ARRAY[0]},${CORES_VPP_ARRAY[1]},${CORES_VPP_ARRAY[2]},${CORES_VPP_ARRAY[3]}"
442		VPP_DEV0="workers 0,1"
443		VPP_DEV1="workers 2,3"
444		ENABLE_MULTIQUEUE="1"
445	fi
446	
447	if [ "$QUEUES" = "2" -a "$ENABLE_MULTIQUEUE" = "1" ]; then
448		VPP_DEV0="$VPP_DEV0 num-rx-queues 2 num-tx-queues 2"
449		VPP_DEV1="$VPP_DEV1 num-rx-queues 2 num-tx-queues 2"
450	fi
451	
452	cat << EOF > "$TMP_DIR/vpp.cmdline"
453cpu { $VPP_CPU }
454unix { startup-config $TMP_DIR/vpp.conf interactive }
455dpdk { dev $VPP_IF0_PCI { $VPP_DEV0 } dev $VPP_IF1_PCI { $VPP_DEV1 } }
456EOF
457
458	cat << EOF >  "$TMP_DIR/vpp.conf"
459set interface state $VPP_IF1_NAME up
460set interface state $VPP_IF0_NAME up
461EOF
462}
463
464function prepare() {
465	#Generate all configuration and VM files
466	clean
467	prepare_vm
468	prepare_vpp
469}
470
471function start_vpp() {
472	if [ -f "$TMP_DIR/vpp-running" ]; then
473		echo "VPP is already running"
474		return 1
475	fi
476	
477	GDB=""
478	if [ "$VPP_GDB" = "1" ]; then
479		[ -e "$TMP_DIR/vpp.sh.gdbinit" ] && sudo rm "$TMP_DIR/vpp.sh.gdbinit"
480		cat << EOF > "$TMP_DIR/vpp.sh.gdbinit"
481handle SIGUSR1 nostop
482handle SIGUSR1 noprint
483set print thread-events off
484run
485EOF
486		GDB="gdb -x $TMP_DIR/vpp.sh.gdbinit --args "
487	fi
488
489	echo "------- Starting VPP --------"
490	echo "   Screen $VPPSCREEN (sudo screen -r $VPPSCREEN)"
491	echo "   Command-line Conf:"
492	cat $TMP_DIR/vpp.cmdline
493	echo "   CLI Conf:"
494	cat $TMP_DIR/vpp.conf
495	echo "-----------------------------"
496
497	sudo screen -d -m -S "$VPPSCREEN" $GDB $VPP_DIR/build-root/install-vpp-native/vpp/bin/vpp -c $TMP_DIR/vpp.cmdline
498	touch "$TMP_DIR/vpp-running"
499}
500
501function stop_vpp() {
502	set +e
503	sudo screen -S "$VPPSCREEN" -X quit && echo "Stopping VPP"
504	[ -f "$TMP_DIR/vpp-running" ] && rm "$TMP_DIR/vpp-running"
505}
506
507function start_vm() {
508	if [ -f "$TMP_DIR/qemu.pid" ]; then
509		echo "VM already running"
510		return 1
511	fi
512	
513	echo "------- Starting VM --------"
514	echo "   VM conf:"
515	cat $TMP_DIR/vm.conf
516	echo "----------------------------"
517	
518	vmdir_mount
519	
520	#Eval is used so that ' characters are not ignored
521	eval sudo chrt -rr 1 taskset -c $CORES_VM $QEMU $(cat $TMP_DIR/vm.conf)
522	
523	echo "Started QEMU with PID $(sudo cat $TMP_DIR/qemu.pid)"
524	
525	sudo brctl addbr $BRNAME
526	sudo ip link set $BRNAME up
527	sudo ip addr add fd00::ffff/64 dev $BRNAME
528	sudo brctl addif $BRNAME $VMTAP
529	sudo ip link set $VMTAP up
530}
531
532function stop_vm() {
533	set +e
534	
535	[ -f "$TMP_DIR/qemu.pid" ] && echo "Stopping VM ($(sudo cat $TMP_DIR/qemu.pid))" && sudo kill "$(sudo cat $TMP_DIR/qemu.pid)" && sudo rm $TMP_DIR/qemu.pid
536	
537	sudo ip link set $BRNAME down
538	sudo brctl delbr $BRNAME
539	
540	vmdir_umount
541}
542
543function start() {
544	if [ -f "$TMP_DIR/.started" ]; then
545		echo "$TMP_DIR/.started exists"
546		echo "This means the setup is probably running already."
547		echo "Please stop the setup first, or remove this file."
548		exit 2
549	fi
550	
551	banner
552	prepare
553	
554	touch "$TMP_DIR/.started"
555	echo "0" | sudo tee /proc/sys/kernel/watchdog_cpumask
556	
557	start_vpp
558	sleep 10
559	connect_vhost
560	start_vm
561}
562
563function pin_vm() {
564	PIDS=$(ps -eLf | grep  qemu-system-x86_64 | awk '$5 > 50 { print $4; }')
565	idx=1
566	for p in $PIDS; do
567		if [ "${CORES_VM_ARRAY[$idx]}" = "" ]; then
568			echo "Too many working threads in VM"
569			return 1
570		fi
571		echo "VM PID $p on core ${CORES_VM_ARRAY[$idx]}"
572		sudo taskset -pc ${CORES_VM_ARRAY[$idx]} $p && sudo chrt -r -p 1 $p
573		idx=$(expr $idx + 1)
574	done
575	
576	#pin non-running processes to other cores
577	if [ "${CORES_VM_ARRAY[$idx]}" != "" ]; then
578		PIDS=$(ps -eLf | grep  qemu-system-x86_64 | awk '$5 < 20 { print $4; }')
579		for p in $PIDS; do
580			echo "VM lazy process $p on core ${CORES_VM_ARRAY[$idx]}"
581			sudo taskset -pc ${CORES_VM_ARRAY[$idx]} $p || echo err
582		done
583	fi
584}
585
586function pin_vpp() {
587
588	for i in $(ls /proc/irq/ | grep [0-9]); do 
589		echo 1 | sudo tee /proc/irq/$i/smp_affinity > /dev/null || true ; 
590	done
591	
592	
593	PIDS=$(ps -eLf | grep  /bin/vpp | awk '$5 > 50 { print $4; }')
594	skip_first=1
595	idx=0
596	for p in $PIDS; do
597		if [ "${CORES_VPP_ARRAY[$idx]}" = "" ]; then
598			echo "Too many working threads in VPP"
599			return 1
600		fi
601		if [ "$skip_first" = "1" ]; then
602			echo "Skipping $p"
603			skip_first=0
604		else
605			echo "VPP PID $p on core ${CORES_VPP_ARRAY[$idx]}"
606			sudo taskset -pc ${CORES_VPP_ARRAY[$idx]} $p && sudo chrt -r -p 1 $p
607			idx=$(expr $idx + 1)
608		fi
609	done
610}
611
612function pin() {
613	pin_vm
614	pin_vpp
615}
616
617function stop() {
618	set +e
619	stop_vm
620	stop_vpp
621	[ -f "$TMP_DIR/.started" ] && rm "$TMP_DIR/.started"
622}
623
624function cmd_stop_vm() {
625	load_config
626	stop_vm
627}
628
629function cmd_start_vm() {
630	load_config
631	start_vm
632}
633
634function cmd_openvnc() {
635	load_config
636    echo Please VNC to 5900 to connect to this VM console
637    socat TCP6-LISTEN:5900,reuseaddr TCP:localhost:5901 &
638}
639
640function cmd_start() {
641	load_config
642	start
643}
644
645function cmd_pin() {
646	load_config
647	pin
648}
649
650function cmd_turbo_disable() {
651	load_config
652	sudo modprobe msr
653	echo "Disabling turboboost on CPUs $CORES_VPP_LIST $CORES_VM_LIST"
654	for cpu in $CORES_VPP_LIST $CORES_VM_LIST ; do
655		sudo wrmsr -p ${cpu} 0x1a0 0x4000850089
656	done
657}
658
659function cmd_turbo_enable() {
660	load_config
661	sudo modprobe msr
662	echo "Enabling turboboost on CPUs $CORES_VPP_LIST $CORES_VM_LIST"
663	for cpu in $CORES_VPP_LIST $CORES_VM_LIST ; do
664		sudo wrmsr -p ${cpu} 0x1a0 0x850089
665	done
666}
667
668function cmd_stop() {
669	load_config
670	stop
671}
672
673function cmd_clean() {
674	load_config
675	clean
676}
677
678function cmd_ssh() {
679	load_config
680	ssh ${VM_USERNAME}@fd00::1
681}
682
683function cmd_config() {
684	load_config
685}
686
687function cmd_disconnect() {
688	load_config
689	disconnect_vhost $@
690}
691
692function cmd_connect() {
693	load_config
694	connect_vhost $@
695}
696
697[ "$1" = "" ] && echo "Missing arguments" && usage && exit 1
698CMD="$1"
699shift
700eval "cmd_$CMD" "$@"
701
702