libvirt_cloud/Makefile

ROLES=base proxy toxcore
SHELLCHECK_OPTS=SC2003,SC2006,SC2010,SC2039,SC2181,SC2046,SC2086,SC2048,SC2162,SC2034,SC2030,SC2166,SC2242,SC2223,SC2319,SC3009,SC3011,SC3030,SC3043,SC3054,SC2009,SC1090,SC2164,SC3044

# FixMe
ANSIBLE_PLUGINS=/usr/local/lib/python3.11/site-packages/ansible/plugins/connection/

# Edit this to be one of pentoo or devuan depending on your host platform
# Find the corresponding host in hosts.yml and edit the settings, then
# change this to be that hostname
LOCALHOST=`cat /etc/hostname`

# set this to the name linux_local_group host in hosts.yml
LOCAL_HOSTS_NAME=pentoo
# set this to the name linux_chroot_group host in hosts.yml
YAML_CHROOT_NAME=linuxGentoo
# set this to the libvirt name of the linux_libvirt_group host in hosts.yml
OVERLAY_HOSTS_NAME=gentoo_overlay-2

BOX_NBD_BASE_QCOW="`/usr/local/bin/ansible_get_inventory.bash BOX_NBD_BASE_QCOW ${OVERLAY_HOSTS_NAME}`"
BOX_NBD_OVERLAY_DIR="`/usr/local/bin/ansible_get_inventory.bash BOX_NBD_OVERLAT_DIR ${OVERLAY_HOSTS_NAME}`"
BOX_NBD_OVERLAY_QCOW="`/usr/local/bin/ansible_get_inventory.bash BOX_NBD_OVERLAT_QCOW ${OVERLAY_HOSTS_NAME}`"
BOX_NBD_OVERLAY_XML=${BOX_NBD_OVERLAY_DIR}/xml/${OVERLAY_HOSTS_NAME}.xml
BOX_NBD_OVERLAY_NAME="`/usr/local/bin/ansible_get_inventory.bash BOX_NBD_OVERLAY_NAME ${OVERLAY_HOSTS_NAME}`"

PWD=/o/var/local/src/play_tox/
NETWORK=Whonix-External
VERBOSE=2

all: install lint build check run test

lint::
	@sudo xmllint -noout roles/ansible-gentoo_install/templates/etc/libvirt/qemu/gentoo.xml
	@yamllint -c .yamllint.yml -f standard *.yml roles/*/*s/*yml 2>&1| \
		grep -v 'truthy\|indentation' | \
		sed -e '/^$$/d' | tee .yamllint.out | \
		grep -B 2 error | tee .yamllint.err || true
	grep Error .yamllint.out || true

# groddy but works for me
install:: lint 
#	( /usr/local/src ; ansible-galaxy collection install \
#	  file:///usr/local/src/community.general )
	[ -e $(ANSIBLE_PLUGINS)/connection/libvirt_qemu.py ] \
		|| ln -s ${PWD}/lib/plugins/libvirt_qemu.py \
		$(ANSIBLE_PLUGINS)/connection/ || true
	@[ -f ${BOX_NBD_BASE_QCOW} ] || { \
          echo ERROR: not created BOX_NBD_DEV="${BOX_NBD_DEV}" - use ; \
          echo qemu-img create -f qcow2 "${BOX_NBD_BASE_QCOW}" 20G ; \
			exit 2 ; }
	@( ps ax | grep -v grep | \
		grep "qemu-nbd.*/dev/nbd.*${BOX_NBD_BASE_QCOW}" ) || { \
	        echo ERROR: not mounted BOX_NBD_DEV="${BOX_NBD_DEV}" - use ; \
	        echo qemu-nbd -n -f qcow2 -c /dev/nbd1 ${BOX_NBD_BASE_QCOW} ;\
		exit 1 ; }

build::
	sudo $(MAKE) -$(MAKEFLAGS) build_base
	sudo $(MAKE) -$(MAKEFLAGS) build_overlay

build_base:: install
	echo INFO: $@ "${BOX_NBD_BASE_QCOW}"
	echo INFO running the toxcore role will build ${BOX_NBD_BASE_QCOW}
	sudo sh ansible_local.bash --diff -i ${PWD}/hosts.yml \
		-l ${LOCALHOST} -c local --verbose ${VERBOSE} \
		$(ROLES) > .$@-${LOCALHOST} 2>&1
	[ -f ${BOX_NBD_BASE_QCOW} ]

build_overlay::
	@virsh list | grep "${OVERLAY_HOSTS_NAME}.*running" && \
		virsh destroy ${OVERLAY_HOSTS_NAME} ; true
#	@virsh list | grep "${OVERLAY_HOSTS_NAME}.*running" && exit 1
	@virsh list --all | grep ${OVERLAY_HOSTS_NAME} && \
		virsh undefine ${OVERLAY_HOSTS_NAME} && \
		rm -f \
		 ${BOX_NBD_OVERLAY_XML} \
		 ${BOX_NBD_OVERLAY_QCOW} ; true
# /a/tmp/GentooImgr/create-vm/xml/gentoo1.xml 
#	! virsh list --all | grep "${OVERLAY_HOSTS_NAME}" && exit 2
	[ ! -f ${BOX_NBD_OVERLAY_QCOW} ] || { \
		echo WARN delete this file to continue; \
		echo rm -f ${BOX_NBD_OVERLAY_QCOW} ; \
		exit 3 ; }
	[ ! -f ${BOX_NBD_OVERLAY_XML} ] || { \
		echo WARN delete this file to continue ; \
		echo rm -f ${BOX_NBD_OVERLAY_XML} ; \
		exit 4 ; }
	PLAY_ANSIBLE_SRC=${PWD} bash bin/toxcore_build_overlay_qcow.bash
	[ -f ${BOX_NBD_OVERLAY_XML} ]
	xmllint -noout ${BOX_NBD_OVERLAY_XML}

check::
	grep -n 'shell: *$$' roles/*/tasks/*.yml && { echo ERROR: "shell: in .yml" ; false ; } || true
	grep -n '^[a-z ]*: {{' roles/*/tasks/*.yml && { echo WARN: "{{ in .yml" ; false ; } || true
	@bash .pylint.sh ; cat .pylint.err ; true
	@shellcheck -s bash -W 0 -x -a -e ${SHELLCHECK_OPTS} bin/*sh || true
	$(MAKE) -$(MAKEFLAGS) check_base
	@[ -d /mnt/gentoo/lost+found ] && \
		sudo $(MAKE) -$(MAKEFLAGS) $@_chroot
	@[ -f ${BOX_NBD_OVERLAY_QCOW} ] && \
		sudo $(MAKE) -$(MAKEFLAGS) $@_overlay

check_localhost::
	sudo sh ansible_local.bash --diff -i hosts.yml -l ${LOCALHOST} \
		--check -c local --verbose ${VERBOSE} \
		$(ROLES) > .$@-$@-${LOCALHOST} 2>&1

check_base::
	ls ${BOX_NBD_BASE_QCOW}
	ls ${BOX_NBD_OVERLAY_QCOW}
	ls ${BOX_NBD_OVERLAY_XML}
	ps axf | grep 'qemu-system-x86_64 -name guest='${OVERLAY_HOSTS_NAME} ; \
		true

check_chroot::
	[ -d /mnt/gentoo/lost+found ] || exit 0
	ansible -c chroot -l ${YAML_CHROOT_NAME} -i hosts.yml \
		-m setup -vvv ${YAML_CHROOT_NAME}
	sudo sh ansible_local.bash --diff -i hosts.yml -l ${YAML_CHROOT_NAME} \
		--check -c chroot --verbose ${VERBOSE} \
		$(ROLES) > .$@-${YAML_CHROOT_NAME}-${LOCALHOST} 2>&1

check_overlay::
	sudo /usr/local/sbin/toxcore_libvirt_test_ga.bash ${OVERLAY_HOSTS_NAME} ls /
	sudo /usr/local/sbin/toxcore_libvirt_test_ga.bash ${OVERLAY_HOSTS_NAME}
# domain-*-gentoo/org.qemu.guest_agent.0 || true
	sudo virsh list | grep -q ${OVERLAY_HOSTS_NAME} || exit 0
	sudo find /var/lib/libvirt/qemu/channel/target/	| \
		grep org.qemu.guest_agent.0
	sudo find /var/lib/libvirt/qemu/channel/target/ -type s | \
		grep ${OVERLAY_HOSTS_NAME}
	ansible -c libvirt_qemu -l ${OVERLAY_HOSTS_NAME} -i hosts.yml \
		-m setup -vvv ${OVERLAY_HOSTS_NAME}
	sudo sh ansible_local.bash --diff -i hosts.yml -l ${OVERLAY_HOSTS_NAME} \
		--check -c libvirt_qemu --verbose ${VERBOSE} \
		$(ROLES) > .$@-${OVERLAY_HOSTS_NAME}-${LOCALHOST} 2>&1

# Edit hosts.yml and customize this target if you are on a Debianish
devuan::
	sudo sh ansible_local.bash --diff -i ${PWD}/hosts.yml \
		-l devuan -c local --verbose ${VERBOSE} $(ROLES) \
		> .$@-${LOCALHOST} 2>&1

# Edit hosts.yml and customize this target if you are on a Gentoo
pentoo::
	sudo sh ansible_local.bash --diff -i ${PWD}/hosts.yml \
		-l pentoo -c local --verbose ${VERBOSE} $(ROLES) \
		> .$@-${LOCALHOST} 2>&1

run::
	@[ ! -f ${BOX_NBD_BASE_QCOW} ] && \
		$(MAKE) -$(MAKEFLAGS) $@_local
	@[ -d /mnt/gentoo/lost+found ] && \
		sudo $(MAKE) -$(MAKEFLAGS) $@_chroot
	@[ ! -f ${BOX_NBD_OVERLAY_QCOW} ] && \
		sudo $(MAKE) -$(MAKEFLAGS) $@_overlay

run_local:: lint
	A=`grep nbd /proc/partitions | wc -l`
	@[ $${A} -eq 0 ] && echo WARN looks like theres no nbd mount && \
		exit 0
	@[ -f ${BOX_NBD_BASE_QCOW} ] && \
		echo WARN  ${BOX_NBD_BASE_QCOW} exists - delete or rename \
		&& exit 0
	BOX_NBD_DEV=$( /usr/local/bin/ansible_get_inventory.bash BOX_NBD_DEV  $BOX)
	A=`grep $BOX_NBD_DEV /proc/partitions | wc -l`
	@[ $${A} -eq 0 ] && echo WARN looks like theres no $BOX_NBD_DEV nbd mount && \
		exit 0
	sudo sh ansible_local.bash --diff -i hosts.yml -l ${LOCALHOST} \
		-c local --verbose ${VERBOSE} $(ROLES) \
		> .$@-$@-${LOCALHOST} 2>&1

run_chroot::
	[ -d /mnt/gentoo/lost+found ] || exit 0
	sudo sh ansible_local.bash --diff -i hosts.yml -l ${YAML_CHROOT_NAME} \
		 -c chroot --verbose ${VERBOSE} $(ROLES) \
		> .$@-${YAML_CHROOT_NAME}-${LOCALHOST} 2>&1

install_libvirt::
	@virsh net-list | grep "${NETWORK}.*active" || \
		sudo virsh net-start "${NETWORK}" || { \
		echo WARN: error virsh net-start "${NETWORK}" ; }
	[ -f ${BOX_NBD_OVERLAY_XML} ]
# xmlstarlet sel -t -v 
	A=$(grep 'source file=' ${BOX_NBD_OVERLAY_XML} | sed -e 's@.*file=.@@' -e "s@'.*@@" )
	[ -n "${A}" ] && [ -f "${A}" ]
	@virsh list --all | grep ${OVERLAY_HOSTS_NAME} || \
	  virsh define ${BOX_NBD_OVERLAY_XML}
	@virsh list | grep "${OVERLAY_HOSTS_NAME}.*running" || \
	  { virsh start ${OVERLAY_HOSTS_NAME} ; sleep 40 ; }

run_overlay:: install_libvirt
	[ -f ${BOX_NBD_OVERLAY_QCOW} ] || { \
		echo WARN  ${BOX_NBD_OVERLAY_QCOW} doesnt exist - make build_overlay ; \
		exit 1 ; }
	sh ansible_local.bash --diff -i hosts.yml -l ${OVERLAY_HOSTS_NAME} \
		-c libvirt_qemu --verbose ${VERBOSE} $(ROLES) \
		> .run-${OVERLAY_HOSTS_NAME}-${LOCALHOST} 2>&1

# hourly is quick tests, weekly is medium tests, monthly is long tests
weekly:: test
test:: 
#	bash .pyanal.sh &
	@[ -d /mnt/gentoo/lost+found ] && \
		sudo $(MAKE) -$(MAKEFLAGS) $@_local
	@[ -f ${BOX_NBD_OVERLAY_QCOW} ] && \
		sudo $(MAKE) -$(MAKEFLAGS) $@_overlay
test_local::
	bash .pyanal.sh &
	sudo sh ansible_local.bash --diff -i ${PWD}/hosts.yml -l ${LOCALHOST} \
		-c local \
		--verbose ${VERBOSE} -t weekly \
		$(ROLES) > .$@-${LOCALHOST}  2>&1

test_overlay:: install_libvirt
	! sudo virsh list | grep -q ${OVERLAY_HOSTS_NAME} && exit 0
	sudo sh ansible_local.bash --diff -i ${PWD}/hosts.yml \
		-l ${OVERLAY_HOSTS_NAME} -c libvirt_qemu \
		--verbose ${VERBOSE} -t weekly \
		$(ROLES) > .$@-${LOCALHOST}  2>&1

# this is a special test target to test a copy of the base qcow2
VM_HOSTS_NAME=gentoo_vm-2
VM_XML=/etc/libvirt/qemu/${VM_HOSTS_NAME}.xml
A="`grep 'source file=.*qcow2' ${VM_XML} | sed -e 's@.*file=.@@' -e "s@'.*@@"`"
install_vm::
	@virsh net-list | grep "${NETWORK}.*active" || \
		sudo virsh net-start "${NETWORK}" || { \
		echo WARN: error virsh net-start "${NETWORK}" ; }
	[ -f ${VM_XML} ]
	@virsh list --all | grep ${VM_HOSTS_NAME} || { \
	  echo ERROR virsh define ${VM_XML} ; exit 8 ; }
# xmlstarlet sel -t -v 
	[ -n "${A}" ] && [ -f "${A}" ]
	@virsh list | grep "${VM_HOSTS_NAME}.*running" || \
	  { virsh start ${VM_HOSTS_NAME} ; sleep 40 ; }

test_vm:: install_vm
	sudo sh ansible_local.bash --diff -i ${PWD}/hosts.yml \
		-l ${VM_HOSTS_NAME} -c libvirt_qemu \
		--check --verbose ${VERBOSE} -t daily \
		$(ROLES) > .$@-${LOCALHOST}  2>&1
	sudo sh ansible_local.bash --diff -i ${PWD}/hosts.yml \
		-l ${VM_HOSTS_NAME} -c libvirt_qemu \
		--verbose ${VERBOSE} -t daily \
		$(ROLES) > .$@-${LOCALHOST}  2>&1
# ${VERBOSE}
rsync::
	bash .rsync.sh

veryclean:: clean
	rm -f .run* .check*

clean::
	find . -name \*~ -delete
	rm roles/*/vars/*.txt
	rm -rf roles/toxcore/overlay/Linux/usr/local/src/_Old

test::
	DEBUG=1 sudo bash -x /usr/local/sbin/toxcore_libvirt_test_ga.bash gentoo_vm-2 ls /