emdee/libvirt_cloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add roles/toxcore/

Browse Source
This commit is contained in:
emdee 2023-12-30 12:52:24 +00:00
parent ac375a87c7
commit 273e6b2282
51 changed files with 9643 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile
View File

@ -209,3 +209,4 @@ veryclean:: clean
clean::
find . -name \*~ -delete
rm roles/*/vars/*.txt

13
README.md
View File

@ -151,9 +151,22 @@ On Ansibles from 2.10 and later, you will need the community plugins installed.
## Advanced Usage
### GentooImgr: Gentoo Image Builder for Cloud and Turnkey ISO installers
There is a modified version of https://github.com/NucleaPeon/gentooimgr/
where we've modified the code a little to do use Python logging. We can
still use it for the build stage, but we think the install stage is better
done using ansible, hence the libvirt_cloud playbook.
The code is in src/ansible_gentooimgr The code is being supported as
an ansible module using library/ansible_gentooimgr.py which is a work
in progress; the idea is to use it for the build and status actions,
but handle the install tasks using ansible.
### ansible_local.bash
We have a script that calls ansible to run our play: ansible_local.yml
but sets some defaults
[ -l limit ]
[ -c connection ]

233
library/ansible_gentooimgr.py Executable file
View File

@ -0,0 +1,233 @@
#!/usr/bin/python3
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import os
import sys
import logging
from argparse import Namespace
import pathlib
import traceback
# in the library
if os.environ.get('PLAY_ANSIBLE_SRC',''):
# running from source
mod_path = os.environ.get('PLAY_ANSIBLE_SRC','')
mod_path = os.path.join(mod_path, 'src', 'ansible_gentooimgr')
assert os.path.isdir(mod_path), f"parent {mod_path}"
assert os.path.isfile(os.path.join(mod_path, '__init__.py')),f"index {mod_path}"
assert os.path.isdir(os.path.join(mod_path, 'gentooimgr')), f"sub {mod_path}"
sys.path.append(mod_path)
else:
# in the library
mod_path = os.path.dirname(os.path.realpath('__file__'))
mod_path = os.path.join(mod_path, 'src', 'ansible_gentooimgr')
assert os.path.isdir(mod_path), f"parent {mod_path}"
assert os.path.isfile(os.path.join(mod_path, '__init__.py')),f"index {mod_path}"
assert os.path.isdir(os.path.join(mod_path, 'gentooimgr')), f"sub {mod_path}"
sys.path.append(mod_path)
try:
import gentooimgr
except Exception as e:
sys.stderr.write(f"{mod_path} {sys.path} {traceback.print_exc()}")
raise
import ansible
DOCUMENTATION = rf'''
---
module: gentooimgr
short_description: Gentoo Image Builder for Cloud and Turnkey ISO installers
version_added: "1.0.0"
description:
* This project enables easy access to building ``systemd`` or ``openrc`` -based images.
* Performs automatic download AND verification of the linux iso, stage3 tarball and portage.
* Caches the iso and stage3 .txt files for at most a day before redownloading and rechecking for new files
* Sane and readable cli commands to build, run and test.
* Step system to enable user to continue off at the same place if a step fails
* No heavy packages like rust included ** TODO
options:
action:
description: The action to be run by the image builder
choices:
- build
- run
- status
- install
- chroot
- unchroot
- command
- shrink
- kernel
required: true
# clean test
config:
default: cloud.json
description: init configuration file or or base.json or cloud.json
required: false
loglevel:
default: {logging.INFO}
description: python logging level <= 50, INFO=20
required: false
threads:
default: 1
description: Number of threads to use
required: false
profile:
default: openrc
description: The init system
choices:
- openrc
- systemd
required: false
kernel_dir:
default: /usr/src/linux
description: Where kernel is specified. By default uses the active linux kernel
required: false
portage:
description: Extract the specified portage tarball onto the filesystem
required: false
stage3:
description: Extract the specified stage3 package onto the filesystema
required: false
action_args:
default: []
description: Arguments for some of the actions - UNUSED!
required: false
temporary_dir:
description: Path to temporary directory for downloading files (20G)
required: false
qcow:
description: Path to file to serve as the base image
required: false
# Specify this value according to your collection
# in format of namespace.collection.doc_fragment_name
# extends_documentation_fragment:
# - my_namespace.my_collection.my_doc_fragment_name
author:
- Your Name (@yourGitHubHandle)
'''
#[-y DAYS]
# [-d DOWNLOAD_DIR]
# [-f]
# [--format FORMAT]
EXAMPLES = r'''
# Pass in a message
- name: Test with a message
my_namespace.my_collection.my_test:
name: hello world
# pass in a message and have changed true
- name: Test with a message and changed output
my_namespace.my_collection.my_test:
name: hello world
new: true
# fail the module
- name: Test failure of the module
my_namespace.my_collection.my_test:
name: fail me
'''
RETURN = r'''
# These are examples of possible return values, and in general should use other names for return values.
message:
description: The output message that the test module generates.
type: str
returned: always
sample: 'goodbye'
'''
from ansible.module_utils.basic import AnsibleModule
def run_module():
# define available arguments/parameters a user can pass to the module
module_args = dict(
action=dict(type='str', required=True),
loglevel=dict(type='int', required=False, default=logging.INFO),
threads=dict(type='int', required=False, default=1),
config=dict(type='str', default='cloud.json', required=False),
profile=dict(type='str', required=False),
kernel_dir=dict(type='path', required=False),
portage=dict(type='path', required=False),
stage3=dict(type='path', required=False),
temporary_dir=dict(type='path', required=False, default=pathlib.Path(os.getcwd())),
download_dir=dict(type='path', required=False, default=pathlib.Path(os.getcwd())),
qcow=dict(type='path', required=False),
)
# seed the result dict in the object
# we primarily care about changed and state
# changed is if this module effectively modified the target
# state will include any data that you want your module to pass back
# for consumption, for example, in a subsequent task
result = dict(
changed=False,
original_message='',
message=''
)
# the AnsibleModule object will be our abstraction working with Ansible
# this includes instantiation, a couple of common attr would be the
# args/params passed to the execution, as well as if the module
# supports check mode
module = AnsibleModule(
argument_spec=module_args,
supports_check_mode=True
)
# if the user is working with this module in only check mode we do not
# want to make any changes to the environment, just return the current
# state with no modifications
if module.check_mode:
module.exit_json(**result)
# manipulate or modify the state as needed (this is going to be the
# part where your module will do what it needs to do)
# if module.params.get('thirsty'):
oargs = Namespace(**module.params)
# during the execution of the module, if there is an exception or a
# conditional state that effectively causes a failure, run
# AnsibleModule.fail_json() to pass in the message and the result
result['original_message'] = ""
try:
from gentooimgr.__main__ import main
retval = main(oargs)
except Exception as e:
result['message'] = str(e)
e = traceback.print_exc()
if e: result['original_message'] += f"{e}"
module.fail_json(msg='Exception', **result)
else:
result['message'] = str(retval)
# use whatever logic you need to determine whether or not this module
# made any modifications to your target
if dArgs['action'] in ['status']:
result['changed'] = False
else:
result['changed'] = True
# in the event of a successful module execution, you will want to
# simple AnsibleModule.exit_json(), passing the key/value results
module.exit_json(**result)
def main():
run_module()
if __name__ == '__main__':
main()

17
roles/ansible-gentoo_install/tasks/disk.yml
View File

@ -18,15 +18,21 @@
- name: create disklabel
command: parted -s {{ AGI_install_disk }} mklabel {{ AGI_install_disklabel }}
register: disklabel_out
# stderr: 'Warning: Error fsyncing/closing /dev/nbd1: Input/output error'
failed_when: false
- name: disklabel_out
debug:
var: disklabel_out
# We need to leave a small gap at the beginning of the disk, or grub won't be
# able to install to the MBR
- name: create boot partition
shell: |
parted -s {{ AGI_install_disk }} mkpart primary ext2 1M 200M
e2label {{ AGI_install_disk }}p1 boot
args:
creates: "{{ AGI_install_disk }}p2"
creates: "{{ AGI_install_disk }}p1"
- name: mark boot partition as active
shell: |
@ -35,14 +41,12 @@
- name: create swap partition
shell: |
parted -s {{ AGI_install_disk }} -- mkpart primary linux-swap 201M 2200M
mkswap -L swap "{{ AGI_install_disk }}p2"
args:
creates: "{{ AGI_install_disk }}p2"
- name: create root partition
shell: |
parted -s {{ AGI_install_disk }} -- mkpart primary ext4 2201M 20070M
e2label {{ AGI_install_disk }}p3 root
args:
creates: "{{ AGI_install_disk }}p3"
@ -61,3 +65,8 @@
check_mode: false
when: not ansible_check_mode
- name: label partitions
shell: |
e2label {{ AGI_install_disk }}p3 root
e2label {{ AGI_install_disk }}p1 boot
mkswap -L swap "{{ AGI_install_disk }}p2"

24
roles/toxcore/Makefile Normal file
View File

@ -0,0 +1,24 @@
# -*-mode: makefile; fill-column: 75; tab-width: 8; coding: utf-8-unix -*-
ROLE=hostvms
USER=`grep PRIV_SKEL_USERS_LIST /usr/local/etc/testforge/testforge.bash|sed -e 's/.*=//' -e 's/"//g'`
VAGRANT_DOCTEST=python3.sh ../../../src/ansible/bin/vagrant_doctest-ssh.py
PYTHON_DOCTEST=python3.sh -m doctest
SSH_DOCTEST=python3.sh ../../../src/ansible/bin/vagrant_doctest-ssh.py -p 2222 --box="vagrant@127.0.0.1"
LOCAL_DOCTEST=/var/local/bin/testforge_run_doctest2.bash
DOCTEST=$(LOCAL_DOCTEST)
default:: local
test:: local # hourly
sudo -u ${USER} /var/local/src/var_local_$(ROLE).bash $@
hourly::
/var/local/bin/$(ROLE)_hourly.bash
local::
$(DOCTEST) overlay/Linux/var/local/share/doc/txt/$(ROLE)3.txt
# $(DOCTEST) overlay/Linux/var/local/share/doc/txt/proxy2.txt

30
roles/toxcore/defaults/main.yml Normal file
View File

@ -0,0 +1,30 @@
# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
---
# be careful - these are running on the guest which may have no relationship with the other roles
# "ansible_virtualization_role": ["host", "guest"]
# "ansible_virtualization_type": ["kvm", "virtualbox", "chroot", ...]
TOXCORE_USR_LOCAL: "{{ USR_LOCAL }}"
TOXCORE_VAR_LOCAL: "{{ USR_LOCAL }}"
TOXCORE_LOG_DIR: "{{ TOXCORE_VAR_LOCAL }}/var/log"
TOXCORE_HOURLY_LOG: "{{TOXCORE_LOG_DIR}}/hourly"
TOXCORE_DAILY_LOG: "{{TOXCORE_LOG_DIR}}/daily"
TOXCORE_WEEKLY_LOG: "{{TOXCORE_LOG_DIR}}/weekly"
TOXCORE_MONTHLY_LOG: "{{TOXCORE_LOG_DIR}}/monthly"
TOXCORE_VMS_DIR: "{{TOXCORE_VAR_LOCAL}}/data/Vms"
TOXCORE_LOCAL_SRC: "{{TOXCORE_VAR_LOCAL}}/src"
TOXCORE_GPG_SERVER: "{{BASE_GPG_SERVER}}"
TOXCORE_GENTOO_FROM_MP: "{{BOX_GENTOO_FROM_MP}}"
# TOXCORE_CTOXCORE_CMAKE:
TOXCORE_FEATURES:
- libvirt
- docker

15
roles/toxcore/meta/main.yml Normal file
View File

@ -0,0 +1,15 @@
# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
---
galaxy_info:
author: Funtoo
company: Remote
description: Toxcore provisioning
license: MIT
min_ansible_version: 2.9
platforms:
- name: Gentoo
categories:
- system
dependencies: []

33
roles/toxcore/tasks/Devuan.yml Normal file
View File

@ -0,0 +1,33 @@
# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
---
- name: "toxcore Devuan.yml"
debug:
verbosity: 1
msg: "toxcore Devuan.yml"
- name: "install toxcore_debs_inst packages"
environment:
- "RUNLEVEL": 1
apt:
force_apt_get: true
name: "{{ item }}"
state: latest
update_cache: no
when:
- item != '' and item != []
- not ansible_check_mode
- "BASE_ARE_CONNECTED|default('') != ''"
with_items:
- "{{ toxcore_debs_inst }}"
- "{{ toxcore_qemu_debs_inst if 'qemu' in TOXCORE_FEATURES or 'libvirt' in TOXCORE_FEATURES else [] }}"
- "{{ proxy_libvirt_debs_inst if 'libvirt' in TOXCORE_FEATURES else [] }}"
- "{{ toxcore_libvirt_debs_inst if 'libvirt' in TOXCORE_FEATURES else [] }}"
- "{{ toxcore_virtualbox_debs_inst if 'virtualbox' in TOXCORE_FEATURES else [] }}"
- "{{ toxcore_vagrant_debs_inst if 'vagrant' in TOXCORE_FEATURES else [] }}"
- "{{ toxcore_packer_debs_inst if 'packer' in TOXCORE_FEATURES else [] }}"
- "{{ toxcore_docker_debs_inst if 'docker' in TOXCORE_FEATURES else [] }}"
ignore_errors: "{{ BASE_PKG_IGNORE_ERRORS }}"

171
roles/toxcore/tasks/Gentoo.yml Normal file
View File

@ -0,0 +1,171 @@
# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
---
- name: "DEBUG: toxcore Gentoo.yml"
debug:
verbosity: 1
msg: "DEBUG: Including toxcore Gentoo.ym"
# - "{{ ansible_distribution }}/{{ BOX_SERVICE_MGR }}"
- name: install toxcore packages
environment: "{{ portage_proxy_env }}"
shell: |
role=toxcore
cd {{ BASE_ROOT_LOG_DIR }} || exit 2
/usr/local/bin/usr_local_base.bash box_gentoo_emerge {{item}} || exit $?
with_items:
- "{{ toxcore_pkgs_inst }}"
- "{{ toxcore_qemu_pkgs_inst if 'qemu' in TOXCORE_FEATURES }}"
- "{{ toxcore_qemu_pkgs_inst if 'libvirt' in TOXCORE_FEATURES }}"
- "{{ toxcore_libvirt_pkgs_inst if 'libvirt' in TOXCORE_FEATURES }}"
- "{{ toxcore_docker_pkgs_inst if 'DOCKER' in TOXCORE_FEATURES }}"
ignore_errors: "{{ BASE_PKG_IGNORE_ERRORS }}"
when:
- item != '' and item != []
- BASE_ARE_CONNECTED|default('') != ''
- "{{ ansible_virtualization_role|replace('NA', 'host') == 'host' }}"
- name: install toxcore packages GUEST
environment: "{{ portage_proxy_env }}"
shell: |
cd {{ BASE_ROOT_LOG_DIR }} || exit 2
/usr/local/bin/usr_local_base.bash box_gentoo_emerge {{item}} || exit $?
with_items:
- "{{ toxcore_pkgs_inst_guest }}"
ignore_errors: "{{ BASE_PKG_IGNORE_ERRORS }}"
when:
- item != '' and item != []
- BASE_ARE_CONNECTED|default('') != ''
- "{{ ansible_virtualization_role|replace('NA', 'host') != 'host' }}"
- name: /etc/conf.d/consolefont
blockinfile:
dest: "/etc/{{ETC_CONF_D}}/consolefont"
marker: "# {mark} ANSIBLE MANAGED BLOCK proxy Gentoo"
mode: 0644
owner: "{{BOX_ROOT_USER}}"
group: "{{BOX_ROOT_GROUP}}"
create: yes
block: |
# for 80x24
consolefont="ter-v24b"
- name: /etc/local.d/rc.local.start later
shell: |
# not systemd
[ -d /etc/local.d ] || exit 0
[ -f "/etc/local.d/rc.local.start" ] && exit 0
echo /etc/rc.local > /etc/local.d/rc.local.start
chmod 755 /etc/local.d/rc.local.start
cat >> /etc/rc.local << EOF
/etc/init.d/consolefont stop; /etc/init.d/consolefont start
stty -F /dev/tty1 cols 80 rows 24
grep vda /proc/partitions && \
e2label /dev/vda3 root && \
e2label /dev/vda1 boot
sed -e 's/^#L/L/' -i /etc/fstab
EOF
bash /etc/rc.local
exit 0
when:
- not ansible_check_mode
- BOX_SERVICE_MGR != 'systemd' # maybe
# safe ones
- block:
- name: "/etc/portage/make.conf base Gentoo PORTAGE_ELOG"
blockinfile:
dest: /etc/portage/make.conf
create: false
marker: "# {mark} ANSIBLE MANAGED BLOCK base Gentoo.yml [PORTAGE_ELOG]"
block: |
PORTAGE_ELOG_CLASSES="warn error"
# NOT syslog
PORTAGE_ELOG_SYSTEM="save"
- name: "/etc/portage/make.conf base Gentoo2.yml CFLAGS"
blockinfile:
dest: /etc/portage/make.conf
create: false
marker: "# {mark} ANSIBLE MANAGED BLOCK base Gentoo.yml [CFLAGS]"
block: |
# -pipe
CFLAGS="-mtune=generic -O2"
CXXFLAGS="-mtune=generic -O2"
- name: "/etc/portage/make.conf base Gentoo.yml PORT_LOGDIR"
blockinfile:
dest: /etc/portage/make.conf
create: false
marker: "# {mark} ANSIBLE MANAGED BLOCK base Gentoo.yml [PORT_LOGDIR]"
block: |
PORT_LOGDIR="/var/log/portage"
PORTAGE_ELOG_CLASSES="log warn error info"
PORTAGE_ELOG_SYSTEM="echo:log,warn save:log,warn,error,info syslog:error"
- name: "/etc/portage/make.conf base Gentoo.yml ACCEPT_KEYWORDS"
blockinfile:
dest: /etc/portage/make.conf
create: false
marker: "# {mark} ANSIBLE MANAGED BLOCK base Gentoo.yml [ACCEPT_KEYWORDS]"
block: |
# In ACCEPT_KEYWORDS, ~amd64 is used for current 64-bit builds;
# There is no tilde for the stable build.
ACCEPT_KEYWORDS="~amd64"
- name: "/etc/portage/make.conf base Gentoo.yml GENTOO_MIRRORS"
blockinfile:
dest: /etc/portage/make.conf
create: false
marker: "# {mark} ANSIBLE MANAGED BLOCK base Gentoo.yml [GENTOO_MIRRORS]"
block: |
GENTOO_MIRRORS="http://distfiles.gentoo.org"
- name: "/etc/portage/make.conf Gentoo2 base"
blockinfile:
dest: /etc/portage/make.conf
create: false
marker: "# {mark} ANSIBLE MANAGED BLOCK base Gentoo2.yml [COLLISION_IGNORE]"
block: |
# 2016-01 added for some py packages
COLLISION_IGNORE="/lib/modules/* *.py[co] __init__.py"
- name: "/etc/portage/make.conf Gentoo2 UNINSTALL_IGNORE"
blockinfile:
dest: /etc/portage/make.conf
create: false
marker: "# {mark} ANSIBLE MANAGED BLOCK base Gentoo2.yml [UNINSTALL_IGNORE]"
block: |
# 2017-01 added for kernel compiling
UNINSTALL_IGNORE="/usr/src /opt /usr/lib32 /usr/lib64/python2.7"
- name: "/etc/portage/make.conf Gentoo2 FEATURES-preserve-libs"
blockinfile:
dest: /etc/portage/make.conf
create: false
marker: "# {mark} ANSIBLE MANAGED BLOCK base Gentoo2.yml [FEATURES-preserve-libs]"
block: |
# https://wiki.gentoo.org/wiki/Project:Toolchain/libcrypt_implementation
FEATURES="${FEATURES} preserve-libs"
- name: "/etc/portage/make.conf Gentoo2 FEATURES"
blockinfile:
dest: /etc/portage/make.conf
create: false
marker: "# {mark} ANSIBLE MANAGED BLOCK base Gentoo2.yml [CFLAGS]"
block: |
#Please adjust your CFLAGS as desired, information can be found here: https://wiki.gentoo.org/wiki/CFLAGS
#Do not modify these FLAGS unless you know what you are doing, always check the defaults first with "portageq envvar CFLAGS"
#This is the default for pentoo at the time of build:
#CFLAGS="-Os -mtune=nocona -pipe -frecord-gcc-switches"
#A safe choice would be to keep whatever Pentoo defaults are, but optimize for your specific machine:
CFLAGS="-mtune=generic -O2 -pipe"
#If you do change your CFLAGS, it is best for all the compile flags to match so uncomment the following three lines:
CXXFLAGS="-mtune=generic -O2 -pipe"
FCFLAGS="${CFLAGS}"
FFLAGS="${CFLAGS}"
when: true

48
roles/toxcore/tasks/Ubuntu.yml Normal file
View File

@ -0,0 +1,48 @@
# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
---
- name: "toxcore Ubuntu.yml"
debug:
verbosity: 1
msg: "toxcore Ubuntu.yml"
- name: uninstall toxcore_lxd_debs_absent packages
environment:
- "RUNLEVEL": 1
apt:
force_apt_get: true
name: "{{item}}"
state: absent
update_cache: no
when:
- item != ''
- not ansible_check_mode
- "'lxd' not in TOXCORE_FEATURES"
with_items: "{{ toxcore_lxd_debs_absent }}"
- name: "install toxcore_debs_inst packages"
environment:
- "RUNLEVEL": 1
shell: |
apt-get install -y \
"{{ toxcore_debs_inst|join('') }}" \
"{{ toxcore_qemu_debs_inst|join('') if 'qemu' in TOXCORE_FEATURES else '' }}" \
"{{ toxcore_libvirt_debs_inst|join('') if 'libvirt' in TOXCORE_FEATURES else '' }}" \
"{{ toxcore_docker_debs_inst|join('') if 'docker' in TOXCORE_FEATURES else '' }}" \
{{ '--print-uris' if BASE_ARE_CONNECTED|default('') == '' else '' }}
when:
- "item != ''"
- not ansible_check_mode
- "BASE_ARE_CONNECTED|default('') != ''"
ignore_errors: "{{ BASE_PKG_IGNORE_ERRORS }}"
- block:
- name: "toxcore Ubuntu18.yml"
debug:
verbosity: 1
msg: "toxcore Ubuntu18.yml"
when: ansible_distribution_major_version == 18

7
roles/toxcore/tasks/Ubuntu18.yml Normal file
View File

@ -0,0 +1,7 @@
# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
---
- debug:
verbosity: 1
msg: "hostvms/Ubuntu18"

83
roles/toxcore/tasks/bootstrap_nbd.yml Normal file
View File

@ -0,0 +1,83 @@
# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
---
- name: "DEBUG: base bootstrap_nbd.yml nbd_out={{nbd_disk}}"
debug:
verbosity: 1
msg: "DEBUG: Including bootstrap_nbd nbd_out={{nbd_disk}}"
- block:
- name: "partition the disk"
shell: |
cat /proc/partitions | grep {{BASE_NBD_DEV}} || exit 1
cat /proc/partitions | grep {{BASE_NBD_DEV}}p1 || exit 0
[ {{BASE_NBD_DEV}}p1 = {{nbd_disk}} ]
parted -s {{nbd_disk}} mklabel msdos
parted -s {{nbd_disk}} mkpart primary 2048s 100%
partprobe
mkfs.ext4 -FF {{nbd_disk}}p1
- name: "mount the disk"
shell: |
df | grep {{nbd_disk}}p1 && exit 0
[ -d {{BASE_NBD_MP}} ] || mkdir {{BASE_NBD_MP}}
mount {{nbd_disk}}p1 {{BASE_NBD_MP}}
[ -d {{BASE_NBD_MP}}/lost+found ]
- name: "check the disk"
shell: |
df | grep {{nbd_disk}}p1 || exit 1
[ -d {{BASE_NBD_MP}}/lost+found ] || exit 2
- name: check the downloads step5
shell: |
stage3_asc=stage3-amd64-openrc-20231217T170203Z.tar.xz.sha256
stage3_xz=stage3-amd64-openrc-20231217T170203Z.tar.xz
sha256sum -c $stage3_asc
tar xJpf $stage3 --xattrs-include='*.*' --numeric-owner -C {{BASE_NBD_MP}
portage_xz=portage-20231221.tar.xz
tar xpJf $portage -C {{BASE_NBD_MP}}/usr
[ -d etc/portage/package.env ] || mkdir -p etc/portage/package.env
cat >> etc/portage/package.env/singlejob.txt << EOF
app-portage/eix singlejob.conf
dev-util/maturin singlejob.conf
dev-util/cmake singlejob.conf
[ -d etc/portage/package.license ] || mkdir -p etc/portage/package.license
EOF
chdir: "{{BASE_NBD_MP}}"
creates: "{{BASE_NBD_MP}}/etc"
- name: chroot into the partition - step4
shell: |
df | grep {{nbd_disk}}p1 || exit 1
[ -d {{BASE_NBD_MP}}/lost+found ] || exit 2
[ -d {{BASE_NBD_MP}}/proc ] && exit 0
/usr/local/sbin/base_chroot.bash {{BASE_NBD_MP}}
register: base_bootstrap_chroot_rc
- name: check the sources
shell: |
true
- name: install the downloads
shell: |
/usr/local/sbin/base_chroot_unbind.bash {{BASE_NBD_MP}}
when: base_bootstrap_chroot_rc.rc == 0
- name: stop here
shell: |
exit 1
delegate_to: localhost
when:
- nbd_disk != ''
#gpg --keyserver hkps://keys.gentoo.org --recv-keys 0xBB572E0E2D182910

37
roles/toxcore/tasks/daily.yml Normal file
View File

@ -0,0 +1,37 @@
# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
---
- name: "toxcore daily.yml"
debug:
msg: "toxcore daily.yml ansible_virtualization_role {{ansible_virtualization_role|replace('NA', 'host')}}"
# these should test on the host
# delegate_to: localhost? - no - per test
- name: "usr_local_toxcore.bash"
environment: "{{ shell_proxy_env }}"
shell: |
umask 0002
bash {{TOXCORE_USR_LOCAL}}/src/usr_local_toxcore.bash \
{{ 'test' if not ansible_check_mode }}
exit 0
args:
chdir: "{{TOXCORE_USR_LOCAL}}/src"
become: yes
become_user: "{{ BOX_USER_NAME }}"
ignore_errors: true
check_mode: false
- name: "run toxcore_daily.bash"
environment: "{{ shell_proxy_env }}"
shell: |
umask 0027
echo "INFO: toxcore_log_daily {{HARDEN_LOG_DIR}}"
cd {{USR_LOCAL}}/bin
toxcore_daily.bash
register: toxcore_log_daily
notify: summary of logs
ignore_errors: true
when:
- not ansible_check_mode
- false

58
roles/toxcore/tasks/hourly.yml Normal file
View File

@ -0,0 +1,58 @@
# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
---
- name: "toxcore hourly.yml"
debug:
msg: "toxcore hourly.yml ansible_virtualization_role {{ansible_virtualization_role|replace('NA', 'host')}}"
# these should test on the host
- block:
- name: check ifconfig
# was toxcore_bin.bash
shell: |
echo WARN: this maybe vagrant only
ifconfig eth0 | grep 'inet ' || \
ifconfig eth0 netmask 255.255.255.0 192.168.122.22
when:
- false # this is vagrant only
- "ansible_virtualization_role|replace('NA', 'host') != 'host'"
- name: check guest-agent
# was toxcore_bin.bash
shell: |
UNIT=qemu-guest-agent
S=/etc/init.d/$UNIT
$S $UNIT status || {
retval=$?
echo WARN: $UNIT not running
exit 0 # $retval
}
exit 0
when: "ansible_virtualization_role|replace('NA', 'host') != 'host'"
- name: run toxcore_check_modules.bash
# was toxcore_bin.bash
shell: |
bash toxcore_check_modules.bash {{TOXCORE_FEATURES|join(' ')}}
args:
chdir: "{{ TOXCORE_VAR_LOCAL }}/bin"
environment: "{{ shell_proxy_env }}"
register: toxcore_log_bin
notify: summary of logs
# FixMe?
ignore_errors: true
- name: "run toxcore_hourly.bash"
environment: "{{ shell_proxy_env }}"
shell: |
umask 0027
echo "INFO: toxcore_log_hourly {{HARDEN_LOG_DIR}}"
cd {{VAR_LOCAL}}/bin
toxcore_hourly.bash
register: toxcore_log_hourly
notify: summary of logs
ignore_errors: true
when:
- not ansible_check_mode
- false

342
roles/toxcore/tasks/main.yml Normal file
View File

@ -0,0 +1,342 @@
# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
---
- name: "toxcore main.yml"
debug:
verbosity: 1
msg: "toxcore main.yml BOX_OS_FAMILY={{BOX_OS_FAMILY}} {{BOX_GENTOO_FROM_MP}} {{ansible_virtualization_role|replace('NA', 'host')}}"
- name: toxcore include_vars
include_vars: "{{item}}.yml"
with_items:
- Linux
- "{{ ansible_distribution }}{{ ansible_distribution_major_version }}"
tags: always
- name: "rsync toxcore root_overlay"
synchronize:
src: "roles/toxcore/overlay/{{item}}/"
dest: /
compress: no
copy_links: yes
archive: no
recursive: yes
links: no
owner: no
perms: no
times: yes
rsync_opts: "{{base_rsync_opts}}"
with_items:
- Linux
- "{{ ansible_distribution }}"
notify:
- chmod /var/local
when:
- not ansible_check_mode
# FixAns: This remote host is being accessed via chroot instead so it cannot work
- ansible_connection|default('') not in PLAY_NORSYNC_CONNECTIONS
tags:
- always
- name: rsync root_overlay - tar before running
unarchive:
dest: /
src: "{{item}}.tar"
keep_newer: true
owner: "{{BOX_ROOT_USER}}"
# extra_opts: "{{ BASE_UNTAR_ARGS }}"
with_items:
- Linux
- "{{ ansible_distribution }}"
notify:
- chmod /usr/local
ignore_errors: true
when:
- not ansible_check_mode
# FixAns: This remote host is being accessed via chroot instead so it cannot work
- ansible_connection|default('') in PLAY_NORSYNC_CONNECTIONS
- name: "make /var/local/var/log/testforge toxcore"
file:
path: "{{ item }}"
state: directory
mode: 0775
with_items:
- "{{ TOXCORE_LOG_DIR }}"
- "{{ TOXCORE_DAILY_LOG }}"
# - "{{ TOXCORE_WEEKLY_LOG }}"
# - "{{ TOXCORE_MONTHLY_LOG }}"
tags:
- hourly
- daily
- weekly
- monthly
- name: "/var/local/bin/toxcore_sign_modules.bash modules needed toxcore_kmods_in_host"
shell: |
KVER=`cat /proc/cmdline |sed -e 's/ .*//' -e 's/BOOT_IMAGE=//' -e 's/kernel-[a-z]*-x86_64-//'`
KDIR=/lib/modules/$KVER
DIR=$KDIR/misc
[ -d $DIR ] || { echo WARN: no directory $DIR ; exit 0 ; }
/var/local/bin/toxcore_sign_modules.bash {{ toxcore_kmods_in_host|flatten|join(' ') }}
when: false
# this changes with kernel 4 -> 5
- name: modprobe VM modules toxcore_kmods_in_host
modprobe:
name: "{{ item }}"
state: "{{ 'absent' if ansible_virtualization_role|replace('NA', 'host') != 'host' else 'present'}}"
when:
- item != ''
with_items: "{{ toxcore_kmods_in_host }}"
ignore_errors: true
- name: modprobe VM modules toxcore_kmods_not_in_host
modprobe:
name: "{{ item }}"
state: absent
with_items: "{{ toxcore_kmods_not_in_host }}"
when: false
# I'm not sure kvm should be missing anymore 5+
ignore_errors: true
- name: include by-platform tasks
include_tasks: "{{ ansible_distribution }}.yml"
- name: grub.cfg from roles/ansible-gentoo_install/tasks/
shell: |
LINE="rd.skipfsck=1 ipv6.disable=1 console=tty1 lang=en keymap=us"
# LINE="$LINE pti=on doscsi iommu=pt amd_iommu=on debugfs=off efi=disable_early_pci_dma extra_latent_entropy init_on_free=1 kvm.nx_huge_pages=force l1tf=full,force mce=0 mds=full,nosmt nosmt=force page_alloc.shuffle=1 pti=on random.trust_cpu=off slab_nomerge slub_debug=FZ spec_store_bypass_disable=on spectre_v2=on tsx_async_abort=full,nosmt vsyscall=none "
LINE="$LINE intel_iommu=on vga=0x315 text"
grep /boot /etc/fstab || exit 1
df | grep /boot || mount /boot || exit 2
[ -d /boot/grub ] || exit 3
[ -f /boot/grub/grub.cfg ] || exit 4
[ -f /boot/grub/grub.cfg ] && cp -p /boot/grub/grub.cfg /boot/grub/grub.cfg.dst
sed -e 's@ ro *$@ '"$LINE"' ro@' -i /boot/grub/grub.cfg
ignore_errors: true
- name: add standard_users to groups
user:
name: "{{ item.0 }}"
append: true
groups: "{{ item.1 }}"
when:
- item != ''
# some groups may not be there
ignore_errors: true
with_nested:
- "{{ base_system_users }}"
- "{{ toxcore_standard_users_groups }}"
- name: "make a directory for /data/Vms"
file:
path: "{{item }}"
state: directory
mode: 0775
with_items:
- "{{TOXCORE_VMS_DIR}}"
- "{{TOXCORE_VMS_DIR}}/Instances"
- "{{TOXCORE_VMS_DIR}}/Qemu"
- "{{TOXCORE_VMS_DIR}}/Boxes"
#? - "{{TOXCORE_VMS_DIR}}/vagrant.d/tmp"
check_mode: false
- name: "toxcore gpg keys system"
# Option --keyserver must be used to
environment: "{{ shell_proxy_env }}"
shell: |
/usr/bin/gpg --list-keys | grep "{{ item.primary }}" || \
/usr/bin/gpg --recv-keys "{{ item.uid }}" # --keyserver "{{ TOXCORE_GPG_SERVER }}"
with_items: "{{ toxcore_gpg_keys_system }}"
when:
- toxcore_gpg_keys_system|length > 0
- BASE_ARE_CONNECTED|default('') != ''
ignore_errors: true
- name: "toxcore gpg keys gentoo"
environment: "{{ shell_proxy_env }}"
shell: |
[ -f /usr/share/openpgp-keys/gentoo-release.asc ] && exit 0
gpg --import /usr/local/share/openpgp-keys/gentoo-release.asc
# this should not run as root
# delegate_to: localhost? - no - per test
- name: "usr_local_toxcore.bash"
environment: "{{ shell_proxy_env }}"
shell: |
umask 0002
bash {{TOXCORE_USR_LOCAL}}/src/usr_local_toxcore.bash \
{{ 'check' if ansible_check_mode }}
exit 0
args:
chdir: "{{TOXCORE_USR_LOCAL}}/src"
become: yes
become_user: "{{ BOX_USER_NAME }}"
ignore_errors: true
check_mode: false
- name: "include_tasks toxcore vms as root"
include_tasks:
file: "{{LOOP_ITEM}}.yml"
apply:
environment: "{{ proxy_env }}"
when:
- LOOP_ITEM != '' and LOOP_ITEM != []
with_items:
- "vms"
loop_control:
loop_var: LOOP_ITEM
- name: "include_tasks toxcore users on the command host"
include_tasks:
file: "{{ LOOP_USER_F[1] }}"
apply:
environment: "{{ proxy_env }}"
become_user: "{{ LOOP_USER_F[0] }}"
when:
- "LOOP_USER_F[1] != ''"
- "ansible_virtualization_role|replace('NA', 'host') == 'host'"
with_nested:
-
- "{{ toxcore_system_users }}"
-
- users.yml
loop_control:
loop_var: LOOP_USER_F
- name: "include_tasks toxcore users as user"
include_tasks:
file: "{{ LOOP_USER_F[1] }}.yml"
apply:
environment: "{{ proxy_env }}"
become_user: "{{ LOOP_USER_F[0] }}"
when:
- "LOOP_USER_F[1] != ''"
- "ansible_virtualization_role|replace('NA', 'host') == 'host'"
- false
with_nested:
- "{{ toxcore_system_users }}"
-
#no - users
- "{{ 'libvirt_users' if 'libvirt' in TOXCORE_FEATURES else '' }}"
# - "{{ 'qemu_users' if 'qemu' in TOXCORE_FEATURES else '' }}"
# - "{{ 'docker_users' if 'docker' in TOXCORE_FEATURES else '' }}"
loop_control:
loop_var: LOOP_USER_F
- name: enable and start toxcore services
service:
name: "{{ item }}"
enabled: yes
state: started
when:
- "item != ''"
- ansible_connection|default('') not in PLAY_CHROOT_CONNECTIONS
- "toxcore_services_enabled|length > 0"
with_items: "{{ toxcore_services_enabled }}"
ignore_errors: true
- name: start toxcore services
service:
name: "{{ item }}"
enabled: no
state: started
when:
- "item != ''"
- ansible_connection|default('') not in PLAY_CHROOT_CONNECTIONS
- false
with_items: "{{ toxcore_services_started }}"
ignore_errors: true
- name: stop toxcore services
service:
name: "{{ item }}"
enabled: no
state: stopped
failed_when: false
when:
- "item != ''"
- ansible_connection|default('') not in PLAY_CHROOT_CONNECTIONS
- false
with_items: "{{ toxcore_services_stopped }}"
- name: run ansible-gentoo_install
include_role:
name: ansible-gentoo_install
when:
- "ansible_virtualization_role|replace('NA', 'host') == 'host'"
# - BOX_OS_FAMILY == 'Gentoo' or BOX_GENTOO_FROM_MP not in ['/', '']
# Set BOX_NBD_OVERLAY_DIR environment variable to override storage location for VMs
# -b Bridge interface to use (defaults to virbr0)
# -m MAC address to use (default is to use a randomly-generated MAC)
# -p ansible users plaintext password
# -k Full path and name of the ansible user's public key file (required)
- name: run toxcore_create-vm.bash
delegate_to: localhost
shell: |
export BOX_NBD_OVERLAY_DIR=${BOX_NBD_OVERLAY_DIR:-"${HOME}/vms/virsh"}
[ '{{BOX_NBD_OVERLAY_NAME}}' != '' ] || exit 1
[ '{{BOX_NBD_OVERLAY_DIR}}' != '' ] || exit 2
[ '{{BOX_NBD_BASE_PUBKEY}}' != '' ] || exit 3
[ '{{BOX_NBD_BASE_QCOW}}' != '' ] || exit 4
[ -f '{{BOX_NBD_BASE_QCOW}}' ] || exit 5
toxcore_create-vm.bash \
-n {{BOX_NBD_OVERLAY_NAME}} \
-k {{BOX_NBD_BASE_PUBKEY}} \
-i {{BOX_NBD_BASE_QCOW}} \
-s {{BOX_NBD_OVERLAY_GB}} \
-c {{BOX_NBD_OVERLAY_CPUS}} \
-r {{BOX_NBD_OVERLAY_RAM}} \
-d {{BOX_NBD_OVERLAY_DIR}} \
-b virbr0 \
-p gentoo \
-o gentoo
ignore_errors: true
when:
- "ansible_virtualization_role|replace('NA', 'host') == 'host'"
# run this last
# works with apply tags hourly
- name: toxcore hourly include_tasks
include_tasks:
file: "hourly.yml"
apply:
tags:
- always
with_items:
- "{{ toxcore_system_users }}"
loop_control:
loop_var: LOOP_USER
tags:
- always
- name: toxcore daily include_tasks
include_tasks:
file: "daily.yml"
apply:
environment: "{{ proxy_env }}"
tags:
- daily
tags:
- daily
- name: toxcore weekly include_tasks
include_tasks:
file: "weekly.yml"
apply:
environment: "{{ proxy_env }}"
tags:
- weekly
tags:
- weekly
when: false
# replace this ^A with a control-A to provoke an error in the error handler of yamlint
# yaml.reader.ReaderError: unacceptable character #x0001: special characters are not allowed

118
roles/toxcore/tasks/users.yml Normal file
View File

@ -0,0 +1,118 @@
# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
---
- name: "toxcore users.yml LOOP_USER_F[0]"
debug:
verbosity: 1
msg: "toxcore users.yml LOOP_USER_F={{LOOP_USER_F[0]}}"
- block:
- name: /etc/sysconfig/docker
blockinfile:
dest: /etc/sysconfig/docker
create: yes
mode: 0644
marker: "# {mark} ANSIBLE MANAGED BLOCK proxy proxy_post.yml"
state: "{{ 'present' if HTTP_PROXYHOST != '' else 'absent' }}"
block: |
HTTP_PROXY={{HTTP_PROXYTYPE}}://{{HTTP_PROXYHOST}}:{{HTTP_PROXYPORT}}
HTTPS_PROXY={{HTTPS_PROXYTYPE}}://{{HTTPS_PROXYHOST}}:{{HTTPS_PROXYPORT}}
when:
- "'docker' in HOSTVMS_FEATURES|default([])"
- name: Run c-toxcore ctest on the tester
delegate_to: localhost
shell: |
[ -d {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build ] || exit 0
cd {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build || exit 1
ctest
exit 0
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_TCP_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_announce_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_bootstrap_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_conference_av_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_conference_double_invite_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_conference_invite_merge_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_conference_peer_nick_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_conference_simple_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_conference_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_conference_two_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_crypto_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_dht_getnodes_api_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_encryptsave_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_file_saving_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_file_transfer_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_forwarding_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_friend_connection_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_friend_request_spam_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_friend_request_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_group_general_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_group_invite_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_group_message_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_group_moderation_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_group_save_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_group_state_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_group_sync_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_group_tcp_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_group_topic_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_invalid_tcp_proxy_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_invalid_udp_proxy_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_lan_discovery_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_lossless_packet_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_lossy_packet_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_network_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_onion_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_overflow_recvq_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_overflow_sendq_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_reconnect_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_save_compatibility_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_save_friend_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_save_load_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_send_message_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_set_name_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_set_status_message_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_tcp_relay_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_tox_dispatch_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_tox_events_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_tox_many_tcp_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_tox_many_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_tox_strncasecmp_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_toxav_basic_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_toxav_many_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_typing_test
# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_version_test
- name: Run c-toxcore dockerfiles on the tester
delegate_to: localhost
shell: |
[ -d {{TOXCORE_LOCAL_SRC}}/dockerfiles ] || exit 0
cd {{TOXCORE_LOCAL_SRC}}/dockerfiles || exit 1
exit 0
# - alpine-s390x
# - alpine-x86_64
# - buildfarm
# - compcert
# - flutter
# - frama-c
# - freebsd
# - ghc-android
# - ghc
# - haskell
# - infer
# - kythe
# - nacl
# - windows-qt5
# - windows
- name: Run toxygen_wrapper on the tester
delegate_to: localhost
shell: |
[ -d {{TOXCORE_LOCAL_SRC}}/toxygen_wrapper ] || exit 0
cd {{TOXCORE_LOCAL_SRC}}/toxygen_wrapper || exit 1
exit 0

144
roles/toxcore/tasks/vms.yml Normal file
View File

@ -0,0 +1,144 @@
# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
---
- name: "toxcore vms.yml"
debug:
verbosity: 1
msg: "toxcore vms.yml TOXCORE_FEATURES={{TOXCORE_FEATURES}}"
- name: "assert /dev/kvm"
shell: |
[ -e /dev/kvm ]
delegate_to: localhost
- name: "make a directory for /etc/pki/qemu"
file:
path: "{{item }}"
state: directory
owner: "{{BOX_ROOT_USER}}"
group: "{{BOX_ROOT_GROUP}}"
mode: 0770
when:
- "item != ''"
with_items:
# see /etc/libvirt/qemu.conf
- "{{ '/etc/pki/qemu' if ( 'qemu' in TOXCORE_FEATURES or 'libvirt' in TOXCORE_FEATURES ) else '' }}"
- block:
- name: increase fs.inotify.max_user_instances (default 128)
blockinfile:
dest: /etc/sysctl.d/70_toxcore_libvirt.conf
marker: "# {mark} ANSIBLE MANAGED BLOCK toxcore lxd"
mode: 0440
group: "{{BOX_ALSO_GROUP}}"
create: yes
block: |
net.ipv4.ip_forward = 1
# NB this is per user
- name: check ulimit
shell: |
[ `ulimit -n` -lt 4913709 ]
register: ulimit_retval
# Kernel ulimit is less than the expected value! This might induce RC test
- name: /etc/security/limits.conf
blockinfile:
dest: /etc/security/limits.conf
create: yes
marker: "# {mark} ANSIBLE MANAGED BLOCK harden vms.yml lxd"
backup: 'yes'
mode: 0644
owner: "{{BOX_ROOT_USER}}"
group: "{{BOX_ROOT_GROUP}}"
block: |
root soft nofile 1048576
root hard nofile 1048576
# libvirt/whonix seems to run out at 1024 user tor in the contrainer
* soft nofile 1048576
* hard nofile 1048576
* soft memlock unlimited
* hard memlock unlimited
# FixMe: either this or @ docker break su with a cryptic
# pam_open_session: Permission denied
# with dmesg entries of
# elogind-daemon[7069]: Failed to open pin file: No such file or
# elogind-daemon[7069]: Failed to allocate manager object: No such file or
# and then later when with no limts.conf but still with a chown error on ~/
# su: Error in services module
# need this in libvirt guest.
when:
# do this anyway for tor and everybody else
- true or ulimit_retval.rc|default(1) == 0
when:
- true or ansible_connection|default('') not in PLAY_CHROOT_CONNECTIONS
# https://github.com/lxc/lxc/issues/3068
# lxc-create -t download -n my-container -- --keyserver hkp://p80.pool.sks-keyservers.net:80
# Hope this helps others if it didn't at least help you
# libvirt lxc docker
- block:
# see also https://stackoverflow.com/questions/26415833/docker-ignores-limits-conf-trying-to-solve-too-many-open-files-error
# FixMe: conflicts with harden - use sysctl.d
# add to your /etc/sysctl.conf : fs.file-max = 64000
- name: Add fs.file-max to /etc/sysctl.conf
lineinfile:
dest: /etc/sysctl.conf
regexp: '^#* *{{item.key}}.*'
line: "{{item.key}} = {{item.val}}"
create: true
state: present
with_items:
- { key: "fs.file-max", val: "4913709" }
#? - {key: "net.ipv4.ip_forward", val: "0"}
#? not sure about this - see docker check-config.sh
- { key: "kernel.keys.root_maxkeys", val: "1000000" }
check_mode: false
- name: /etc/sysctl.conf.d/20_hugepages.conf
check_mode: false
lineinfile:
dest: /etc/sysctl.conf.d/20_hugepages.conf
regexp: '^#* *{{item.key}}.*'
line: "{{item.key}} = {{item.val}}"
state: present
with_items:
# https://wiki.archlinux.org/title/KVM#Enabling_huge_pages
# boot cmdline hugepages=
- { key: "vm.nr_hugepages", val: "550" }
when:
- ansible_virtualization_role|replace('NA', 'host') == 'host' or
ansible_virtualization_role|replace('NA', 'host') == 'guest'
rescue:
- debug:
verbosity: 1
msg: "Ignoring error"
#libvirt
- block:
# /usr/sbin/dnsmasq --conf-file=/var/lib/libvirt/dnsmasq/network.conf --leasefile-ro --dhcp-script=/usr/libexec/libvirt_leaseshelper
- name: "/bind-dynamic/bind-interfaces/ -i /var/lib/libvirt/dnsmasq/default.conf"
shell: |
[ -f {{item}} ] || exit 0
sed -e 's/bind-dynamic/bind-interfaces/' -i {{item}}
#? virsh net-edit default
# maybe just check ifconfig for ...
# bogus? var/lib?
with_items:
- /var/lib/libvirt/dnsmasq/network.conf
- /var/lib/libvirt/dnsmasq/default.conf
ignore_errors: true
when:
- "'libvirt' in TOXCORE_FEATURES" # or 'lxd' in TOXCORE_FEATURES"
- ( ansible_virtualization_type is not defined or
not (ansible_virtualization_type in ['chroot', 'lxc', 'lxd', 'docker', 'qemu', 'kvm']))

213
roles/toxcore/vars/Devuan5.yml Normal file
View File

@ -0,0 +1,213 @@
# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
---
# TOXCORE_FEATURES: ['virtualbox', 'libvirt', docker']
toxcore_system_users: "{{ BOX_USER_NAME }}"
toxcore_also_users: "{{ BOX_ALSO_USERS }}"
toxcore_standard_users_groups:
- disk
- "{{ 'libvirt-qemu' if 'libvirt' in TOXCORE_FEATURES else '' }}"
- "{{ 'kvm' if ( 'qemu' in TOXCORE_FEATURES or 'libvirt' in TOXCORE_FEATURES ) else '' }}"
- "{{ 'docker' if 'docker' in TOXCORE_FEATURES else '' }}"
toxcore_libvirt_services:
- "{{ 'libvirtd' if 'libvirt' in TOXCORE_FEATURES else '' }}"
- "{{ 'virtlogd' if 'libvirt' in TOXCORE_FEATURES else '' }}"
- "{{ 'docker' if 'docker' in TOXCORE_FEATURES else '' }}"
toxcore_debs_inst:
- gpg
- nbdkit
- nbd-client
- nbd-server
- python3-yaml
- gparted
- ebtables
- dnsmasq-base
# vagrant-libvirt/README.md
- libxslt-dev
- libxml2-dev
- zlib1g-dev
- iptables
- zstd
- bridge-utils
- sdparm
- hdparm
- libxml2
- zerofree
- openssh-sftp-server
- sshfs
- xmlstarlet
- libtss2-tcti-swtpm0
# toxcore
- libconfig-dev
- libgtest-dev
- ninja-build
- pkg-config
- zip
- grep
- file
- ca-certificates
- autotools-dev
- autoconf
- automake
- git
- bc
- wget
- rsync
- cmake
- make
- pkg-config
- libtool
- ssh
- gzip
- tar
- unzip
- libcurl4-gnutls-dev
- coreutils
- libavutil-dev
- libsodium-dev
- libffms2-4
- libgpac10
- libx264-dev
- x264
- libv4lconvert0
- libv4l-dev
- libv4l-dev
- libv4l2rds0
- v4l-conf
- v4l-utils
- libv4l-dev
- libtool
- autotools-dev
- automake
- checkinstall
- check
- git
- yasm
- libv4lconvert0
- libv4l-dev
- libopus-dev
- libvpx-dev
- pkg-config
- libx264-dev
- libavcodec-dev
- libavdevice-dev
# nbd
- genisoimage
- bridge-utils
- guestfsd
- libvirt-clients
- libvirt-daemon
- libvirt-daemon-system
- libvirt-daemon-driver-storage-zfs
- python3-libvirt
- virt-manager
- virtinst
- python3-hacking
# - cloud-init
toxcore_qemu_debs_inst:
- qemu-kvm
- qemu
- qemu-block-extra
- qemu-system-common
- qemu-system-data
- qemu-system-x86
- qemu-utils
- guestfsd
- grub-firmware-qemu
- nbd-client
- aqemu
- seabios
- squashfs-tools
- attr
- zerofree
- nbdkit
- sgabios
- lzop
- firmware-realtek
- firmware-linux-free
proxy_libvirt_debs_inst:
- libvirt-daemon-system
- libvirt-daemon
- libvirt-daemon-driver-vbox
- libvirt-daemon-driver-qemu
- libvirt-clients
- libvirt-daemon-driver-storage-zfs
- python3-libvirt
- virt-manager
- virtinst
- usbutils
- usb.ids
- guestfsd
- libguestfs-perl
- libguestfs-tools
#
- genisoimage
- bridge-utils
- guestfsd
- python3-hacking
#? - cloud-init
toxcore_lxd_debs_absent:
- lxd
- lxd-client
- lxcfs
- liblxc1
- liblxc-common
toxcore_lxd_debs_inst:
- lxcfs
- zfsutils-linux
# see proxy_libvirt_debs_inst
toxcore_libvirt_debs_inst:
- python3-libvirt
- libvirt-dev
- libguestfs
- libguestfs-tools
# OUCH 4G!
- libguestfs-appliance
- virt-top
toxcore_packer_debs_inst:
- jq
toxcore_docker_debs_inst:
- aufs-tools
- cgroup-lite
- linux-image-extra-virtual
- docker.io
# - docker-compose
toxcore_ansible_debs_inst:
- jinja
- pycrypto
- python-ecdsa
- python-httplib2
- python-jinja2
- python-markupsafe
- python-paramiko
toxcore_aws_debs_inst:
# for aws
- boto
- bs4
- awscli
toxcore_pips2_inst: []
toxcore_pips3_inst:
- pycrypto
- pywinrm
- requests-unixsocket
- ovirt-engine-sdk-python
- msgpack_python
- pysha3
- pycryptodomex

101
roles/toxcore/vars/Gentoo2.yml Normal file
View File

@ -0,0 +1,101 @@
# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
---
toxcore_system_users: "{{ base_system_users }}"
toxcore_also_users: "{{ base_also_users }}"
toxcore_standard_users_groups:
- disk
- "{{ 'libvirt' if 'libvirt' in TOXCORE_FEATURES else '' }}"
- "{{ 'qemu' if ( 'qemu' in TOXCORE_FEATURES or 'libvirt' in TOXCORE_FEATURES ) else '' }}"
- "{{ 'kvm' if ( 'qemu' in TOXCORE_FEATURES or 'libvirt' in TOXCORE_FEATURES ) else '' }}"
- "{{ 'docker' if 'docker' in TOXCORE_FEATURES else '' }}"
toxcore_libvirt_services:
- libvirtd
# - qemu-guest-agent
toxcore_pkgs_inst_guest: []
toxcore_pkgs_inst:
- net-firewall/iptables
- app-portage/gentoolkit
- sys-apps/gptfdisk
- app-admin/testdisk
- app-arch/zstd
- net-firewall/ebtables
- net-misc/bridge-utils
- sys-apps/sdparm
- sys-apps/hdparm
### virt-manager
- media-libs/libvpx
- net-libs/libpsl
- x11-libs/libxcb
- x11-libs/libxkbcommon
- dev-libs/libxml2
- dev-python/argcomplete
- dev-python/libvirt-python
- dev-python/pygobject
- dev-python/requests
- gnome-base/dconf
- net-libs/gtk-vnc
- net-misc/spice-gtk
- sys-apps/dbus
- x11-libs/gtk+
- x11-libs/gtksourceview
- x11-libs/vte
toxcore_pkgs_masked:
- x11-drivers/xf86-video-amdgpu
- x11-drivers/xf86-video-mga
- x11-drivers/xf86-video-radeon
- x11-drivers/xf86-video-siliconmotion
- x11-drivers/xf86-video-via
toxcore_zfs_pkgs_inst:
- sys-fs/zfs-kmod
- sys-fs/zfs
toxcore_qemu_pkgs_inst:
- app-emulation/qemu
- app-emulation/aqemu
- app-emulation/spice
- net-misc/spice-gtk
- qemu-guest-agent
toxcore_libvirt_pkgs_inst:
- app-emulation/libvirt
- dev-python/libvirt
- app-emulation/virt-manager
- app-emulation/virt-viewer
- x11-drivers/xf86-video-qxl
# - sys-block/open-iscsi
- sys-firmware/edk2-ovmf
- app-emulation/libguestfs
- sys-block/nbd
#? - app-emulation/libguestfs-appliance
- dev-python/libvirt-python
- app-emulation/virtiofsd
toxcore_docker_pkgs_inst:
- sys-process/tini
- app-emulation/docker
#? - dev-python/docker-pycreds
- dev-python/dockerpty
- dev-python/docker-py
- app-containers/docker
# - app-containers/docker-compose
toxcore_pips2_inst: []
toxcore_pips3_inst:
- pycrypto
- pywinrm
- requests-unixsocket
- ovirt-engine-sdk-python
- msgpack_python
- pysha3
- pycryptodomex
- pyanalyze

37
roles/toxcore/vars/Linux.yml Normal file
View File

@ -0,0 +1,37 @@
# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
---
# from BOX
toxcore_system_users: "{{ base_system_users }}"
toxcore_standard_users: "{{ base_standard_users }}"
toxcore_etc_initd_libvirt_name: libvirtd
toxcore_kmods_in_host:
- "{{ 'kvm' if ( ( 'qemu' in TOXCORE_FEATURES or 'libvirt' in TOXCORE_FEATURES ) and 'virtualbox' not in TOXCORE_FEATURES ) else '' }}"
- "{{ 'qxl' if ( 'qemu' in TOXCORE_FEATURES or 'libvirt' in TOXCORE_FEATURES ) else '' }}"
- "{{ 'nbd' if ( 'qemu' in TOXCORE_FEATURES or 'libvirt' in TOXCORE_FEATURES ) else '' }}"
toxcore_kmods_not_in_host:
# I'm not sure kvm should be missing anymore 5+
# FixMe: how do these conflict? qemu libvirt
- "{{ 'kvm' if 'virtualbox' in TOXCORE_FEATURES else '' }}"
toxcore_gpg_keys_system:
- uid: "30737D12308C9D0C882FC34B57CB0A121BAECB2E"
primary: "70ACBB6BFEE7BC572A8941D19266C4FA11FD00FD"
name: "Daniel Robbins (metro:node) <drobbins@funtoo.org>"
key: "9266C4FA11FD00FD"
toxcore_services_enabled: []
# - qemu-guest-agent
toxcore_services_started:
- "{{ toxcore_libvirt_services if 'libvirt' in TOXCORE_FEATURES else [] }}"
# not on Gentoo 5
#? - "{{ 'docker' if 'docker' in TOXCORE_FEATURES else '' }}"
toxcore_services_stopped:
- "{{ toxcore_libvirt_services if 'libvirt' not in TOXCORE_FEATURES else [] }}"

25
roles/toxcore/vars/Makefile Normal file
View File

@ -0,0 +1,25 @@
# -*-mode: text; fill-column: 75; tab-width: 8; coding: utf-8-dos -*-
test::
update:: funtoo
p_src.yaml:: /p/src/_p_src.yaml
( cd /p/src && $(MAKE) $(MFLAGS) funtoo )
world.lis:: /var/lib/portage/world Makefile
cp -p /var/lib/portage/world world.lis
grep -h '^ *- .*/' main.yml base.yml p_src.yaml \
| sed -e 's@/@\\/@' -e 's@^ *- @/@' -e 's@$$@/d@' > world.sed
sed -f world.sed -i world.lis
funtoo:: world.lis Makefile
cp /dev/null python_world.lis
grep ^dev-python world.lis | while read file ; do \
grep -q "^- $$file" p_src.yaml && continue ; \
echo "$$file" >> python_world.lis ; \
done
clean::
# find * -type f -name \*~ -exec rm '{}' \;
rm -f *~ world.sed

155
roles/toxcore/vars/Ubuntu18.yml Normal file
View File

@ -0,0 +1,155 @@
# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
---
# , 'lxd' 'vagrant'
TOXCORE_FEATURES: ['libvirt'] # 'docker', 'packer'
toxcore_system_users: "{{ base_system_users }}"
toxcore_also_users: "{{ base_also_users }}"
toxcore_libvirt_services:
- libvirtd
toxcore_debs_inst:
- python-yaml
- gparted
- libvncserver
- ebtables
- dnsmasq-base
# vagrant-libvirt/README.md
- libxslt-dev
- libxml2-dev
- zlib1g-dev
- iptables
- zstd
- ebtables
- bridge-utils
- sdparm
- hdparm
- libxml2
- sdparm
- hdparm
- libxml2
- zerofree
- openssh-sftp-server
- sshfs
- xmlstarlet
- libtss2-tcti-swtpm0
# toxcore
- libconfig-dev
- libgtest-dev
- ninja-build
- pkg-config
- zip
- grep
- file
- ca-certificates
- autotools-dev
- autoconf
- automake
- git
- bc
- wget
- rsync
- cmake
- make
- pkg-config
- libtool
- ssh
- gzip
- tar
- unzip
- libcurl4-gnutls-dev
- coreutils
- libavutil-dev
- libsodium-dev
- libffms2-4
- libgpac10
- libx264-dev
- x264
- libv4lconvert0
- libv4l-dev
- libv4l-dev
- libv4l2rds0
- v4l-conf
- v4l-utils
- libv4l-dev
- libtool
- autotools-dev
- automake
- checkinstall
- check
- git
- yasm
- libv4lconvert0
- libv4l-dev
- libopus-dev
- libvpx-dev
- pkg-config
- libx264-dev
- libavcodec-dev
- libavdevice-dev
toxcore_lxd_debs_absent:
- lxd
- lxd-client
- lxcfs
- liblxc1
- liblxc-common
toxcore_lxd_pkgs_inst:
- lxcfs
# - lxd
# - lxd-client
- zfsutils-linux
# or snapd
toxcore_qemu_debs_inst:
- qemu-kvm
toxcore_libvirt_debs_inst:
#? - libvirt
- libvirt-bin
- libvirt-dev
- virt-manager
- libguestfs
- libguestfs-tools
# OUCH 4G!
# - libguestfs-appliance
- python-libvirt
- qemu
toxcore_virtualbox_debs_inst:
- virtualbox
- virtualbox-modules
- virtualbox-ext-pack
# virtualbox bridged networking
toxcore_packer_debs_inst:
- jq
toxcore_docker_debs_inst:
- aufs-tools
- cgroup-lite
- libsystemd-journal0
- linux-image-extra-virtual
- docker.io
toxcore_ansible_debs_inst:
- jinja
- pycrypto
- paramiko
- python-ecdsa
- python-httplib2
- python-jinja2
- python-markupsafe
- python-paramiko
toxcore_aws_debs_inst:
# for aws
- boto
- bs4
- requests_ntlm
- awscli

17
src/ansible_gentooimgr/CONFIG.md Normal file
View File

@ -0,0 +1,17 @@
GentooIMGR Configuration Specification
======================================
---------------
Getting Started
---------------
If you want your own customized gentoo image, simply copy the gentooimgr/configs/base.json.example file:
```sh
cp gentooimgr/configs/base.json.example gentooimgr/myconfig.json
# modify your config file. It needs to be saved in the directory directly above or within gentooimgr to be
# included in the generated iso file and therefore mounted automatically; otherwise, copy it to the live image
python -m gentooimgr -c gentooimgr/myconfig.json install
```

168
src/ansible_gentooimgr/README.md Normal file
View File

@ -0,0 +1,168 @@
GentooImgr: Gentoo Image Builder for Cloud and Turnkey ISO installers
=====================================================================
**This is a modified version of https://github.com/NucleaPeon/gentooimgr/
where we've modified the code a little to do use Python logging. We can
still use it for the build stage, but we think the install stage is better
done using ansible, hence the libvirt_cloud playbook.
The code is in overlay/Linux/usr/local/src/ansible_gentooimgr
for reasons to do with how the ansible role transfers files.
The code is being supported as an ansible module using
library/ansible_gentooimgr.py which is a work in progress; the idea
is to use it for the build and status actions, but handle the install
tasks using ansible.
**
GentooImgr is a python script system to build cloud images based on Gentoo Linux.
Huge thanks to https://github.com/travisghansen/gentoo-cloud-image-builder for providing a foundation to work from.
**Features:**
* This project enables easy access to building ``systemd`` or ``openrc`` -based images.
* Performs automatic download AND verification of the linux iso, stage3 tarball and portage.
* Caches the iso and stage3 .txt files for at most a day before redownloading and rechecking for new files
* Sane and readable cli commands to build, run and test.
* Step system to enable user to continue off at the same place if a step fails
* No heavy packages like rust included ** TODO
**rename to gentooimgr, upload to pip**
Preface
-------
This project was created so I could spawn off Gentoo OS templates on my Proxmox server for various services while being more efficient than many other Linux OS's and avoiding systemd.
This python module contains all the software needed to download resources, build the image, run the image in qemu, and allow for some extensibility. The built-in functionality includes a base standard gentoo image configuration as well as a cloud-init image that can be run. You can copy the .json config file and optionally a kernel .config file to configure your image to your liking and pass it in with ``--config``.
This software is in beta so please report any issues or feature requests. You are **highly** encouraged to do so.
Thanks!
Roadmap
-------
* [X] Use gentooimgr to configure and Install a Base Gentoo OS using the least amount of configuration
- Successfully built a gentoo qcow2 image that can be run in qemu, but it requires using the --dist-kernel flag
as building from source still requires some work.
* [X] Use gentooimgr to create a usable cloud image (requires --dist-kernel currently)
* [ ] Use gentooimgr to create Gentoo installations on other non-amd64/non-native architectures (ex: ppc64)
Prerequisites
-------------
* [ ] QEMU
* [ ] python3.11
* [ ] Recommended 20GB of space
* [ ] Internet Connection
* [ ] virt-sparsify (for use with `gentooimgr shrink` action)
Usage
-----
```sh
git clone https://github.com/NucleaPeon/gentooimgr.git
python -m gentooimgr build
python -m gentooimgr run
```
Once qemu is running, mount the available gentooimgr iso and run the appropriate command:
```sh
mkdir -p /mnt/gi
mount /dev/disk/by-label/gentooimgr /mnt/gi
cd /mnt/gi
python -m gentooimgr --config-cloud install
```
Configuring the cloud image will automatically bring in appropriate kernel configs (these are defined in ``gentooimgr/configs/cloud.config``).
Then perform any additional procedures, such as shrinking the img from 10G to ~3-4G
```sh
python -m gentooimgr shrink gentoo.qcow2
```
**NOTE** Due to how ``gentooimgr`` dynamically finds the most recent portage/stage3 and iso files, if multiples exist in the same directory you may have to specify them using the appropriate flag (ie: ``--iso [path-to-iso]``). Older images can be used in this manner and eventually setting those values in the .json file should be recognized by gentooimgr so there will be no need to specify them on the command line.
Extended Usage
--------------
GentooImgr is flexible in that it can be run on a live running system as well as on a livecd in qemu;
It's possible to automate a new bare-metal Gentoo installation (with some further development) simply by running the ``install`` action or equivalent command to install and set up everything.
Eventually, GentooImgr will be used to build gentoo turnkey OS images automatically.
----
One of the conveniences of this software is that it allows you to continue off where an error last occured.
For example, if there's an issue where a package was renamed and the compile software step fails, you can edit
the package list and rerun ``python -m gentooimgr cloud-cfg`` without any arguments and it will resume the compile
step (albeit at the beginning of that step.)
There are also commands that allow you to quickly enter the livecd chroot or run commands in the chroot:
```sh
python -m gentooimgr chroot
```
Mounts/binds are handled automatically when you chroot, but you will need to ``unchroot`` after to unmount the file systems:
```sh
python -m gentooimgr chroot
# do stuff
exit
python -m gentooimgr unchroot
```
Adding Image to Proxmox
-----------------------
(Use the correct username and address to ssh/scp)
```sh
scp gentoo-[stamp].qcow2 root@proxmox:/tmp
ssh root@proxmox
# Set vmbr to your available bridge, it could be vmbr0 or vmbr1, etc.
qm create 1000 --name gentoo-templ --memory 2048 --net0 virtio,bridge=vmbr0
qm importdisk 1000 /tmp/gentoo-[stamp].qcow2 local -format qcow2
qm set 1000 --scsihw virtio-scsi-pci --scsi0 /var/lib/vz/images/1000/vm-1000-disk-0.qcow2
qm set 1000 --ide2 local:cloudinit --boot c --bootdisk scsi0 --serial0 socket --vga serial0
qm resize 1000 scsi0 +20G
qm set 1000 --ipconfig0 ip=dhcp
qm set 1000 --sshkey ~/.ssh/id_rsa.pub
qm template 1000
(Creating a template from this image requires clicking the "Regenerate Image" button or equivalent cli command,
after you set username and password)
```
Caveats
--------
* [X] Forced use of Rust in cloud images (cloud-init dependency)
Unfortunately, using cloud-init brings in cryptography and oauthlib which pulls in rust. Any cloud images therefore are forced to use it, which is a large compilation target if rust-bin is not used. Some FOSS users, myself included, do not want rust installed on their systems and dislike how it is encroaching on so many FOSS areas.
Work may be done to see if this can be avoided, but for now consider it a requirement.
TODO
----
* [ ] Hash check portage downloads on ``build``
* [ ] have a way to set the iso creation to either ignore things not set in the config file, or have options to include dirs, etc.
* [ ] --skip-update-check : Do not even attempt to download new files in any capacity, simply use the latest ones found.
We could implement a way to find by glob and filter by modified by state and simply use the latest modified file
each and every time so we don't fail on multiple file detections

0
src/ansible_gentooimgr/__init__.py Normal file
View File

5
src/ansible_gentooimgr/gentooimgr/__init__.py Normal file
View File

@ -0,0 +1,5 @@
import os
HERE = os.path.abspath(os.path.dirname(__file__))
import logging
LOG = logging.getLogger('GI ')
logging.basicConfig(level=logging.INFO) # oArgs.loglevel) #

185
src/ansible_gentooimgr/gentooimgr/__main__.py Normal file
View File

@ -0,0 +1,185 @@
import os
import sys
import json
import argparse
import pathlib
import copy
import logging
try:
import coloredlogs
os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red'
except ImportError as e:
logging.log(logging.DEBUG, f"coloredlogs not available: {e}")
coloredlogs = None
from gentooimgr import LOG
import gentooimgr.common
import gentooimgr.config
import gentooimgr.configs
def main(args):
'''Gentoo Cloud Image Builder Utility'''
import gentooimgr.config
configjson = gentooimgr.config.determine_config(args)
prefix = args.temporary_dir
LOG.info(f'Gentoo Cloud Image Builder Utility {args.action}')
if args.action == "build":
import gentooimgr.builder
gentooimgr.builder.build(args, configjson)
elif args.action == "run":
import gentooimgr.run
gentooimgr.run.run(args, configjson)
elif args.action == "test":
# empty
import gentooimgr.test
elif args.action == "clean":
# empty
import gentooimgr.clean
elif args.action == "status":
import gentooimgr.status
gentooimgr.status.print_template(args, configjson)
elif args.action == "install":
import gentooimgr.install
gentooimgr.install.configure(args, configjson)
elif args.action == "command":
import gentooimgr.command
gentooimgr.command.command(configjson)
elif args.action == "chroot":
import gentooimgr.chroot
gentooimgr.chroot.chroot(path=args.mountpoint, shell="/bin/bash")
elif args.action == "unchroot":
import gentooimgr.chroot
gentooimgr.chroot.unchroot(path=args.mountpoint)
elif args.action == "shrink":
import gentooimgr.shrink
fname = gentooimgr.shrink.shrink(args, configjson, stamp=args.stamp)
print(f"Shrunken image at {fname}, {os.path.getsize(fname)}")
elif args.action == "kernel":
import gentooimgr.kernel
gentooimgr.kernel.build_kernel(args, configjson)
return 0
if __name__ == "__main__":
"""Gentoo Cloud Image Builder Utility"""
parser = argparse.ArgumentParser(prog="gentooimgr", description="Gentoo Image Builder Utility")
parser.add_argument("-c", "--config", nargs='?', type=pathlib.Path,
help="Path to a custom conf file")
parser.add_argument("--config-cloud", action="store_const", const="cloud.json", dest="config",
help="Use cloud init configuration")
parser.add_argument("--config-base", action="store_const", const="base.json", dest="config",
help="Use a minimal base Gentoo configuration")
parser.add_argument("-t", "--temporary-dir", nargs='?', type=pathlib.Path,
default=pathlib.Path(os.getcwd()), help="Path to temporary directory for downloading files")
parser.add_argument("-j", "--threads", type=int, default=gentooimgr.config.THREADS,
help="Number of threads to use for building and emerging software")
parser.add_argument("-l", "--loglevel", type=int, default=logging.INFO,
help="python logging level <= 50, INFO=20")
parser.add_argument("-y", "--days", type=int, default=7, # gentooimgr.config.DAYS
help="Number of days before the files are redownloaded")
parser.add_argument("-d", "--download-dir", type=pathlib.Path, default=os.getcwd(),
help="Path to the desired download directory (default: current)")
parser.add_argument("--openrc", dest="profile", action="store_const", const="openrc",
help="Select OpenRC as the Gentoo Init System")
parser.add_argument("--systemd", dest="profile", action="store_const", const="systemd",
help="Select SystemD as the Gentoo Init System")
parser.add_argument("-f", "--force", action="store_true",
help="Let action occur at potential expense of data loss or errors (applies to clean and cloud-cfg)")
parser.add_argument("--format", default="qcow2", help="Image format to generate, default qcow2")
parser.add_argument("--portage", default=None, type=pathlib.Path, nargs='?',
help="Extract the specified portage package onto the filesystem")
parser.add_argument("--stage3", default=None, type=pathlib.Path, nargs='?',
help="Extract the specified stage3 package onto the filesystem")
parser.add_argument("--kernel-dir", default="/usr/src/linux",
help="Where kernel is specified. By default uses the active linux kernel")
subparsers = parser.add_subparsers(help="gentooimgr actions", dest="action")
subparsers.required = True
# Build action
parser_build = subparsers.add_parser('build', help="Download and verify all the downloaded components for cloud image")
parser_build.add_argument("image", default=gentooimgr.config.GENTOO_IMG_NAME, type=str, nargs='?',
help="Specify the exact image (date) you want to build; ex: 20231112T170154Z. Defaults to downloading the latest image. If image exists, will automatically use that one instead of checking online.")
parser_build.add_argument("--size", default="12G", help="Size of image to build")
parser_build.add_argument("--no-verify", dest="verify", action="store_false", help="Do not verify downloaded iso")
parser_build.add_argument("--verify", dest="verify", action="store_true", default=True,
help="Verify downloaded iso")
parser_build.add_argument("--redownload", action="store_true", help="Overwrite downloaded files")
parser_run = subparsers.add_parser('run', help="Run a Gentoo Image in QEMU to process it into a cloud image")
parser_run.add_argument("--iso", default=None, type=pathlib.Path, nargs='?',
help="Mount the specified iso in qemu, should be reserved for live cd images")
parser_run.add_argument("image", default=gentooimgr.config.GENTOO_IMG_NAME,
type=pathlib.Path, nargs="?",
help="Run the specified image in qemu")
parser_run.add_argument("-m", "--mounts", nargs='+', default=[],
help="Path to iso files to mount into the running qemu instance")
parser_test = subparsers.add_parser('test', help="Test whether image is a legitamite cloud configured image")
parser_clean = subparsers.add_parser('clean', help="Remove all downloaded files")
# --force also applies to clean action
parser_status = subparsers.add_parser('status', help="Review information, downloaded images and configurations")
parser_install = subparsers.add_parser("install", help="Install Gentoo on a qemu guest. Defaults to "
"--config-base with --kernel-dist if the respective --config or --kernel options are not provided.")
parser_install.add_argument("--kernel-dist", action="store_true",
help="Use a distribution kernel in the installation. Overrides all other kernel options.")
parser_install.add_argument("--kernel-virtio", action="store_true", help="Include virtio support in non-dist kernels")
parser_install.add_argument("--kernel-g5", action="store_true", help="Include all kernel config options for PowerMac G5 compatibility")
parser_chroot = subparsers.add_parser("chroot", help="Bind mounts and enter chroot with shell on guest. Unmounts binds on shell exit")
parser_chroot.add_argument("mountpoint", nargs='?', default=gentooimgr.config.GENTOO_MOUNT,
help="Point to mount and run the chroot and shell")
parser_unchroot = subparsers.add_parser("unchroot", help="Unmounts chroot filesystems")
parser_unchroot.add_argument("mountpoint", nargs='?', default=gentooimgr.config.GENTOO_MOUNT,
help="Point to mount and run the chroot and shell")
parser_cmd = subparsers.add_parser('command', help="Handle bind mounts and run command(s) in guest chroot, then unmount binds")
parser_cmd.add_argument("cmds", nargs='*',
help="Commands to run (quote each command if more than one word, ie: \"grep 'foo'\" \"echo foo\")")
parser_shrink = subparsers.add_parser('shrink', help="Take a finalized Gentoo image and rearrange it for smaller size")
parser_shrink.add_argument("img", type=pathlib.Path, help="Image to shrink")
parser_shrink.add_argument("--stamp", nargs='?', default=None,
help="By default a timestamp will be added to the image name, otherwise provide "
"a hardcoded string to add to the image name. Result: gentoo-[stamp].img")
parser_kernel = subparsers.add_parser('kernel', help="Build the kernel based on configuration and optional --kernel-dist flag")
args = parser.parse_args()
assert args.loglevel < 59
if coloredlogs:
# https://pypi.org/project/coloredlogs/
coloredlogs.install(level=args.loglevel,
logger=LOG,
# %(asctime)s,%(msecs)03d %(hostname)s [%(process)d]
fmt='%(name)s %(levelname)s %(message)s'
)
else:
logging.basicConfig(level=args.loglevel) # logging.INFO
logging.basicConfig(level=args.loglevel)
isos = gentooimgr.common.find_iso(args.download_dir)
if args.action == "run" and args.iso is None and len(isos) > 1:
LOG.error(f"Error: multiple iso files were found in {args.download_dir}, please specify one using `--iso [iso]`")
sys.exit(1)
main(args)

26
src/ansible_gentooimgr/gentooimgr/builder.py Normal file
View File

@ -0,0 +1,26 @@
import os
import argparse
from gentooimgr import LOG
import gentooimgr.config as config
import gentooimgr.download as download
import gentooimgr.qemu as qemu
import gentooimgr.common
import requests
def build(args: argparse.Namespace, config: dict) -> None:
LOG.info(": build")
iso = config.get("iso") or download.download(args)
stage3 = config.get("stage3") or download.download_stage3(args)
portage = config.get("portage") or download.download_portage(args)
filename = f"{args.image}.{args.format}"
image = qemu.create_image(args, config)
if not os.path.exists(image):
raise Exception(f"Image {image} does not exist")
is_default = os.path.basename(image) == filename
LOG.info(image)
LOG.info(f"Image {image} build successfully.\nRun `python -m gentooimgr run{' ' + image if not is_default else ''} --iso {iso}`")
return image

54
src/ansible_gentooimgr/gentooimgr/chroot.py Normal file
View File

@ -0,0 +1,54 @@
import os
import sys
from subprocess import Popen, PIPE
import gentooimgr.config
def bind(mount=gentooimgr.config.GENTOO_MOUNT, verbose=True):
mounts = [
["mount", "--types", "proc", "/proc", os.path.join(mount, "proc")],
["mount", "--rbind", "/sys", os.path.join(mount, "sys")],
["mount", "--make-rslave", os.path.join(mount, "sys")],
["mount", "--rbind", "/dev", os.path.join(mount, "dev")],
["mount", "--make-rslave", os.path.join(mount, "dev")],
["mount", "--bind", "/run", os.path.join(mount, "run")],
["mount", "--make-slave", os.path.join(mount, "run")],
]
for mcmd in mounts:
if verbose:
print(f"\t:: {' '.join(mcmd)}")
proc = Popen(mcmd, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
if proc.returncode != 0:
sys.stderr.write(f"{stderr}\n")
sys.exit(proc.returncode)
def unbind(mount=gentooimgr.config.GENTOO_MOUNT, verbose=True):
os.chdir("/")
if not os.path.exists(mount):
sys.stderr.write(f"Mountpoint {mount} does not exist\n")
return
unmounts = [
["umount", os.path.join(mount, 'dev', 'shm')],
["umount", os.path.join(mount, 'dev', 'pts')],
["umount", "-l", os.path.join(mount, 'dev')],
["umount", "-R", mount]
]
for uncmd in unmounts:
if verbose:
print(f"\t:: {' '.join(uncmd)}")
proc = Popen(uncmd)
stdout, stderr = proc.communicate()
if proc.returncode != 0:
sys.stderr.write(f"{stderr}\n")
continue
def chroot(path=gentooimgr.config.GENTOO_MOUNT, shell="/bin/bash"):
bind(mount=path)
os.chroot(path)
os.chdir(os.sep)
os.system(shell)
unchroot(path=path) # May fail if we do this automatically
def unchroot(path=gentooimgr.config.GENTOO_MOUNT):
unbind(mount=path)

0
src/ansible_gentooimgr/gentooimgr/clean.py Normal file
View File

14
src/ansible_gentooimgr/gentooimgr/command.py Normal file
View File

@ -0,0 +1,14 @@
import sys
from subprocess import Popen, PIPE
import gentooimgr.chroot
def command(config, *args):
gentooimgr.chroot.bind()
for a in args:
proc = Popen(a, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
if proc.returncode != 0:
sys.stderr.write(f"{stderr}\n")
break
gentooimgr.chroot.unbind()

124
src/ansible_gentooimgr/gentooimgr/common.py Normal file
View File

@ -0,0 +1,124 @@
import os
import sys
import time
import copy
import json
from subprocess import Popen, PIPE
import gentooimgr.config
from gentooimgr import LOG
def older_than_a_day(fullpath):
if not os.path.exists(fullpath):
return True # Don't fail on missing files
filetime = os.path.getmtime(fullpath)
return time.time() - filetime > (gentooimgr.config.DAY_IN_SECONDS *
gentooimgr.config.DAYS)
def find_iso(download_dir):
LOG.info(f"Looking for iso in {download_dir}")
name = None
ext = None
found = []
for f in os.listdir(download_dir):
name, ext = os.path.splitext(f)
if ext == ".iso":
found.append(os.path.join(download_dir, f))
return found
def make_iso_from_dir(mydir):
""" Generates an iso with gentooimgr inside it for use inside a live cd guest
:Returns:
path to iso that was created or NoneType if mydir is not found
"""
if not os.path.exists(mydir):
LOG.warn(f"\t:: dir not found {mydir}")
return
LOG.info(f"\t:: Making ISO with dir of {mydir}")
path = os.path.join(mydir, "..", "cloudgen.iso")
proc = Popen(["mkisofs",
"--input-charset", "utf-8",
"-J",
"-r",
"-V", "gentooimgr",
"-m", "*.img",
"-m", "*.iso",
"-o", path,
mydir
], stdout=PIPE, stderr=PIPE)
proc.communicate()
return path
def portage_from_dir(download_dir, filename=None):
"""Find portage file from directory. Will do a check in os.listdir() for portage*.tar.bz2.
If a filename is provided, this function either returns that filename assuming it exists in d,
or return None. If filename is None, this looks through all entries for portage files and if
only one exists, returns it, otherwise None.
"""
assert download_dir, f"empty {download_dir} for for portage"
LOG.info(f"Looking for portage in {download_dir}")
found = []
for f in os.listdir(download_dir):
if filename is not None:
if filename == f:
found.append(f)
elif f.startswith("portage") and f.endswith(".tar.xz"):
found.append(f)
if len(found) > 1:
LOG.error("\tEE: More than one portage file exists, please specify the exact portage file with --portage [file] or remove all others\n")
LOG.error(''.join([f"\t{f}\n" for f in found]))
LOG.error(f"in {download_dir}\n")
sys.exit(1)
return found[0] if found else None
def stage3_from_dir(d, filename=None):
"""Find stage3 file from directory. Will do a check in os.listdir() for stage3*.tar.xz.
If a filename is provided, this function either returns that filename assuming it exists in d,
or return None. If filename is None, this looks through all entries for stage3 files and if
only one exists, returns it, otherwise None.
"""
found = []
for f in os.listdir(d):
if filename is not None:
if filename == f:
found.append(f)
elif f.startswith("stage3") and f.endswith(".tar.xz"):
found.append(f)
if len(found) > 1:
LOG.error("More than one stage3 file exists, please specify the exact stage3 file or remove all others\n")
LOG.error(''.join([f"\t{f}\n" for f in found]))
LOG.error(f"in {d}\n")
return None
return found[0] if found else None
def get_image_name(args, config):
image = config.get("imagename", "gentoo."+args.format)
if image is None:
image = "gentoo."+args.format
return image
#
# def load_config(args):
# cfg = generatecfg(args)
# if args.config:
# override = generatecfg(args, config=args.config)
# cfg.update(cfgoverride)
#
# if cfg.get("portage") is None:
# cfg['portage'] = portage_from_dir(args.download_dir, filename=args.portage or cfg.get("portage"))
# if cfg.get("stage3") is None:
# cfg['stage3'] = stage3_from_dir(args.download_dir, filename=args.stage3 or cfg.get("stage3"))
#
# return cfg
#

132
src/ansible_gentooimgr/gentooimgr/config.py Normal file
View File

@ -0,0 +1,132 @@
import os
import json
import sys
import argparse
from gentooimgr import LOG
import gentooimgr.configs
import multiprocessing
# A day in seconds:
DAY_IN_SECONDS = 60*60*24
# days until the iso is old
DAYS = 1
# Define threads to compile packages with
THREADS = multiprocessing.cpu_count()
# URL to latest image text file, defaults to amd64. This is parsed to find latest iso to download
ARCHITECTURE = "amd64"
GENTOO_BASE_ISO_URL = f"https://distfiles.gentoo.org/releases/{ARCHITECTURE}/autobuilds/current-install-{ARCHITECTURE}-minimal/"
GENTOO_BASE_STAGE_OPENRC_URL = f"https://distfiles.gentoo.org/releases/{ARCHITECTURE}/autobuilds/current-stage3-{ARCHITECTURE}-openrc/"
GENTOO_BASE_STAGE_SYSTEMD_URL = f"https://distfiles.gentoo.org/releases/{ARCHITECTURE}/autobuilds/current-stage3-{ARCHITECTURE}-systemd/"
GENTOO_LATEST_ISO_FILE = f"latest-install-{ARCHITECTURE}-minimal.txt"
GENTOO_LATEST_STAGE_OPENRC_FILE = f"latest-stage3-{ARCHITECTURE}-openrc.txt"
GENTOO_LATEST_STAGE_SYSTEMD_FILE = f"latest-stage3-{ARCHITECTURE}-systemd.txt"
GENTOO_PORTAGE_FILE = "http://distfiles.gentoo.org/snapshots/portage-latest.tar.xz" # No architecture, no txt files to determine latest.
GENTOO_MOUNT = "/mnt/gentoo"
GENTOO_IMG_NAME = "gentoo.qcow2"
GENTOO_FILE_HASH_RE = r"^Hash\: ([\w]*)$"
GENTOO_FILE_ISO_RE = r"^(install-[\w\-_\.]*.iso) ([\d]*)"
GENTOO_FILE_ISO_HASH_RE = r"^([\w]*) (install-[\w\-_\.]*.iso)$"
GENTOO_FILE_STAGE3_RE = r"^(stage3-[\w\-_\.]*.tar.*) ([\d]*)"
GENTOO_FILE_STAGE3_HASH_RE = r"^([\w]*) (stage3-[\w\-_\.]*.tar.*)$"
# TODO: Repo regex to replace attributes, use function to do so as find key will change.
def replace_repos_conf(key, value):
pass
CLOUD_MODULES = [
"iscsi_tcp"
]
def load_config(path):
assert path, "load config called with nothing"
if os.path.exists(path):
with open(path, 'r') as f:
try:
return json.loads(f.read())
except Exception as e:
LOG.error(f"ERROR loading {path}")
raise
return {}
def load_default_config(config_name):
"""This is called when a --config option is set. --kernel options update the resulting config, whether
it be 'base' or other.
If user is supplying their own configuration, this is not called.
"""
name, ext = os.path.splitext(config_name)
if not name in gentooimgr.configs.KNOWN_CONFIGS:
return {}
json_file = os.path.join(gentooimgr.configs.CONFIG_DIR, config_name)
ret = {}
with open(json_file, 'r') as f:
try:
ret = json.loads(f.read())
except Exception as e:
LOG.error(f"loading {json_file} {e}")
return ret
def inherit_config(config: dict) -> dict:
"""Returns the json file that the inherit key specifies; will recursively update if inherit values are set.
"""
configuration = load_default_config(config.get("inherit"))
if not configuration:
configuration = load_config(config.get("inherit"))
if not configuration:
sys.stderr.write(f"\tWW: Warning: Inherited configuration {config.get('inherit')} is not found.\n")
return {}
if configuration.get("inherit"):
configuration.update(inherit_config(configuration.get("inherit")))
return configuration
def determine_config(args: argparse.Namespace) -> dict:
"""Check argparser options and return the most valid configuration
The "package" key/value object overrides everything that is set, it does not update() them.
If you override "base" package set, it's exactly what you set. It makes more sense to do it this way.
For example, if you have a dist kernel config, you don't want the base.json to update and include all
non-dist kernel options as it would add a lot of used space for unused functionality.
The package set is only overridden in the top level json configuration file though;
If you have multiple inherits, those package sets will be combined before the parent package set overrides
with the keys that are set.
If you have base.json and base2.json that contain multiple layers of "base" packages, ie: base: ['foo'] and base2: ['bar']
then you will have in yours.json: packages { base: ['foo', 'bar'] } and unless you set "base", that is what you'll get.
If you check `status` action, it will flatten all configurations into one, so the "inherit" key will always be null.
:Returns:
- configuration from json to dict
"""
# Check custom configuration
configuration = load_default_config(args.config or 'base.json')
if not configuration and args.config:
configuration = load_config(args.config)
if not configuration:
LOG.error(f"\tWW: Warning: Configuration {args.config} is empty\n")
else:
if configuration.get("inherit"):
# newpkgs = configuration.get("packages", {})
inherited = inherit_config(configuration)
new_packages = configuration.get("packages", {})
old_packages = inherited.get("packages", {})
inherited.update(configuration)
# Set back old package dict and then update only what is set in new:
inherited['packages'] = old_packages
for key, pkgs in new_packages.items():
if pkgs:
inherited['packages'][key] = pkgs
return inherited
return configuration

153
src/ansible_gentooimgr/gentooimgr/configs/__init__.py Normal file
View File

@ -0,0 +1,153 @@
import os
CONFIG_DIR = os.path.abspath(os.path.dirname(__file__))
__all__ = ["CONFIG_DIR", "CLOUD_YAML", "HOST_TMPL", "HOSTNAME", "KNOWN_CONFIGS"]
# List of configurations that end in '.json' within the configs/ directory
KNOWN_CONFIGS = [
"base",
"cloud"
]
# Currently we handle the writing of additional files by having data defined here and checking options.
# this isn't ideal. TODO: Make this better.
CLOUD_YAML = """
# The top level settings are used as module
# and system configuration.
# A set of users which may be applied and/or used by various modules
# when a 'default' entry is found it will reference the 'default_user'
# from the distro configuration specified below
users:
- default
# If this is set, 'root' will not be able to ssh in and they
# will get a message to login instead as the above $user (ubuntu)
disable_root: true
ssh_pwauth: false
# This will cause the set+update hostname module to not operate (if true)
preserve_hostname: false
# this may be helpful in certain scenarios
# resize_rootfs_tmp: /dev
syslog_fix_perms: root:root
ssh_deletekeys: false
ssh_genkeytypes: [rsa, dsa]
# This can be 'template'
# which would look for /etc/cloud/templates/hosts.gentoo.tmpl
# or 'localhost'
# or False / commented out to disable altogether
manage_etc_hosts: template
# Example datasource config
# datasource:
# Ec2:
# metadata_urls: [ 'blah.com' ]
# timeout: 5 # (defaults to 50 seconds)
# max_wait: 10 # (defaults to 120 seconds)
# The modules that run in the 'init' stage
cloud_init_modules:
- seed_random
- bootcmd
- write-files
- growpart
- resizefs
- set_hostname
- update_hostname
- update_etc_hosts
- ca-certs
- users-groups
- ssh
# The modules that run in the 'config' stage
cloud_config_modules:
# Emit the cloud config ready event
# this can be used by upstart jobs for 'start on cloud-config'.
- disk_setup
- mounts
- ssh-import-id
- set-passwords
- package-update-upgrade-install
- timezone
- puppet
- chef
- salt-minion
- mcollective
- disable-ec2-metadata
- runcmd
# The modules that run in the 'final' stage
cloud_final_modules:
- scripts-vendor
- scripts-per-once
- scripts-per-boot
- scripts-per-instance
- scripts-user
- ssh-authkey-fingerprints
- keys-to-console
- phone-home
- final-message
- power-state-change
# System and/or distro specific settings
# (not accessible to handlers/transforms)
system_info:
# This will affect which distro class gets used
distro: gentoo
# Default user name + that default users groups (if added/used)
default_user:
name: gentoo
lock_passwd: True
gecos: Gentoo
groups: [users, wheel]
primary_group: users
no-user-group: true
sudo: ["ALL=(ALL) NOPASSWD:ALL"]
shell: /bin/bash
# Other config here will be given to the distro class and/or path classes
paths:
cloud_dir: /var/lib/cloud/
templates_dir: /etc/cloud/templates/
"""
HOST_TMPL = """
## template:jinja
{#
This file /etc/cloud/templates/hosts.gentoo.tmpl is only utilized
if enabled in cloud-config. Specifically, in order to enable it
you need to add the following to config:
manage_etc_hosts: template
-#}
# Your system has configured 'manage_etc_hosts' as 'template'.
# As a result, if you wish for changes to this file to persist
# then you will need to either
# a.) make changes to the master file in /etc/cloud/templates/hosts.gentoo.tmpl
# b.) change or remove the value of 'manage_etc_hosts' in
# /etc/cloud/cloud.cfg or cloud-config from user-data
#
# The following lines are desirable for IPv4 capable hosts
127.0.0.1 {{fqdn}} {{hostname}}
127.0.0.1 localhost.localdomain localhost
127.0.0.1 localhost4.localdomain4 localhost4
# The following lines are desirable for IPv6 capable hosts
::1 {{fqdn}} {{hostname}}
::1 localhost.localdomain localhost
::1 localhost6.localdomain6 localhost6
"""
HOSTNAME = """
# Set to the hostname of this machine
if [ -f /etc/hostname ];then
hostname=$(cat /etc/hostname 2> /dev/null | cut -d"." -f1 2> /dev/null)
else
hostname="localhost"
fi
"""

82
src/ansible_gentooimgr/gentooimgr/configs/base.json Normal file
View File

@ -0,0 +1,82 @@
{
"inherit": null,
"imgsize": "20G",
"memory": 4096,
"mountpoint": "/mnt/gentoo",
"imagename": null,
"initsys": "openrc",
"licensefiles": {
"kernel": ["sys-kernel/linux-firmware linux-fw-redistributable"]
},
"kernel": {
"path": "/etc/kernels/config.d/gentooimgr-base.config"
},
"repos": {
"/etc/portage/repos.conf/gentoo.conf": {
"gentoo": {
"sync-uri": "rsync://192.168.254.20/gentoo-portage"
}
}
},
"packages": {
"base": [
"acpid",
"dmidecode",
"syslog-ng",
"cronie",
"dhcpcd",
"mlocate",
"xfsprogs",
"dosfstools",
"sudo",
"postfix",
"parted",
"portage-utils",
"gentoo-bashcomp",
"tmux",
"app-misc/screen",
"dev-vcs/git",
"net-misc/curl",
"usbutils",
"pciutils",
"logrotate",
"gptfdisk",
"sys-block/gpart",
"net-misc/ntp",
"net-fs/nfs-utils",
"app-emulation/qemu-guest-agent",
"linux-firmware"
],
"additional": ["app-editors/vim"],
"oneshots": [
"portage"
],
"singles": [
"app-portage/eix",
"dev-util/cmake"
],
"keepgoing": [
"openssh"
],
"bootloader": [
"grub:2"
],
"kernel": [
"sys-kernel/genkernel",
"gentoo-sources",
"gentoolkit"
]
},
"services": {
"syslog-ng": "default",
"cronie": "default",
"acpid": "default",
"ntp": "default",
"qemu-guest-agent": "default"
},
"iso": null,
"portage": null,
"stage3": null,
"disk": "/dev/sda",
"partition": 1
}

77
src/ansible_gentooimgr/gentooimgr/configs/base.json.example Normal file
View File

@ -0,0 +1,77 @@
{
"inherit": null,
"imgsize": "12G",
"memory": 4096,
"mountpoint": "/mnt/gentoo",
"imagename": null,
"initsys": "openrc",
"licensefiles": {
"kernel": ["sys-kernel/linux-firmware linux-fw-redistributable"]
},
"repos": {
"/etc/portage/repos.conf/gentoo.conf": {
"sync-uri": "rsync://192.168.254.20/gentoo-portage"
}
},
"packages": {
"base": [
"acpid",
"dmidecode",
"syslog-ng",
"cronie",
"dhcpcd",
"mlocate",
"xfsprogs",
"dosfstools",
"sudo",
"postfix",
"app-editors/vim",
"parted",
"portage-utils",
"bash-completion",
"gentoo-bashcomp",
"tmux",
"app-misc/screen",
"dev-vcs/git",
"net-misc/curl",
"usbutils",
"pciutils",
"logrotate",
"gptfdisk",
"sys-block/gpart",
"net-misc/ntp",
"net-fs/nfs-utils",
"linux-firmware"
],
"additional": [],
"oneshots": [
"portage"
],
"singles": [
"app-portage/eix",
"dev-util/cmake"
],
"keepgoing": [
"openssh"
],
"bootloader": [
"grub:2"
],
"kernel": [
"sys-kernel/genkernel",
"gentoo-sources",
"gentoolkit"
]
},
"services": {
"syslog-ng": "default",
"cronie": "default",
"acpid": "default",
"ntp": "default"
},
"iso": null,
"portage": null,
"stage3": null,
"disk": null,
"partition": 1
}

5183
src/ansible_gentooimgr/gentooimgr/configs/cloud.config Normal file
View File

File diff suppressed because it is too large Load Diff

17
src/ansible_gentooimgr/gentooimgr/configs/cloud.json Normal file
View File

@ -0,0 +1,17 @@
{
"inherit": "base.json",
"packages": {
"additional": [
"app-emulation/cloud-init",
"sys-block/open-iscsi",
"app-editors/mg",
"net-analyzer/openbsd-netcat"
]
},
"disk": "/dev/vda",
"kernel": {
"path": "/etc/kernels/config.d/gentooimgr-cloud.config",
"config": "cloud.config"
},
"partition": 1
}

195
src/ansible_gentooimgr/gentooimgr/download.py Normal file
View File

@ -0,0 +1,195 @@
"""Module to handle downloading and verification of Gentoo images
To ensure accuracy, we re-download every .txt file if it's older than one day.
We assume that people building a cloud configured image want what is most up to date.
If you have a specific image you want built over and over regardless, create a config
file and load it in using -c/--config that points GENTOO_* values to the files you want.
"""
import os
import re
import sys
from datetime import date
import hashlib
import progressbar
from urllib.request import urlretrieve
import tempfile
from gentooimgr import LOG
import gentooimgr.config as config
from gentooimgr.common import older_than_a_day
hashpattern = re.compile(config.GENTOO_FILE_HASH_RE, re.MULTILINE)
isopattern = re.compile(config.GENTOO_FILE_ISO_RE, re.MULTILINE)
isohashpattern = re.compile(config.GENTOO_FILE_ISO_HASH_RE, re.MULTILINE)
stage3pattern = re.compile(config.GENTOO_FILE_STAGE3_RE, re.MULTILINE)
stage3hashpattern = re.compile(config.GENTOO_FILE_STAGE3_HASH_RE, re.MULTILINE)
class DownloadProgressBar():
def __init__(self):
self.progress = None
def __call__(self, block_num, block_size, total_size):
if not self.progress:
self.progress = progressbar.ProgressBar(maxval=total_size)
self.progress.start()
downloaded = block_num * block_size
if downloaded < total_size:
self.progress.update(downloaded)
else:
self.progress.finish()
def parse_latest_iso_text(fullpath) -> tuple:
"""Returns a tuple of (hash type, iso name, iso bytes)"""
with open(fullpath) as f:
content = f.read()
m_hash = hashpattern.search(content)
m_iso = isopattern.search(content)
return (m_hash.group(1) if not m_hash is None else None,
m_iso.group(1) if not m_iso is None else None,
m_iso.group(2) if not m_iso is None else None,)
def parse_latest_stage3_text(fullpath) -> tuple:
"""Returns a tuple of (hash type, iso name, iso bytes)
"""
with open(fullpath) as f:
content = f.read()
m_hash = hashpattern.search(content)
m_stage3 = stage3pattern.search(content)
return (m_hash.group(1) if not m_hash is None else None,
m_stage3.group(1) if not m_stage3 is None else None,
m_stage3.group(2) if not m_stage3 is None else None,)
def verify(args, _type: str, baseurl: str, hashpattern, filename: str) -> bool:
"""Downloads hash file and run a hash check on the file
:Parameters:
- args: Namespace of parsed arguments
- _type: str hash type
- baseurl: (remote) folder where hashsum file is contained
- hashpattern:
- filename: str name of file to check (used to download corresponding hash file)
A install-amd64-minimal-2023111iso2T170154Z.iso file will have a
install-amd64-minimal-20231112T170154Z.iso.sha256 for example.
:Returns:
Whether iso was verified using the specified hash
"""
thefile = os.path.join(args.download_dir, filename)
LOG.info(f"verifying hash of {thefile}")
digest = hashlib.file_digest(open(thefile, 'rb'), _type.lower())
filename = filename+f".{_type.lower()}" # Update to hash file
hashfile = os.path.join(baseurl, filename)
fullpath = os.path.join(args.download_dir, os.path.basename(hashfile))
if not os.path.exists(fullpath) or args.redownload or older_than_a_day(fullpath):
LOG.info(f"Downloading {filename}")
urlretrieve(hashfile, fullpath, DownloadProgressBar())
hd = digest.hexdigest()
with open(fullpath, 'r') as f:
content = f.read()
m_hash = hashpattern.search(content)
_hash = m_hash.group(1)
assert hd == _hash, f"Hash mismatch {hd} != {_hash}"
def download_stage3(args, url=None) -> str:
if url is None:
if args.profile == "systemd":
url = os.path.join(config.GENTOO_BASE_STAGE_SYSTEMD_URL, config.GENTOO_LATEST_STAGE_SYSTEMD_FILE)
else:
url = os.path.join(config.GENTOO_BASE_STAGE_OPENRC_URL, config.GENTOO_LATEST_STAGE_OPENRC_FILE)
filename = os.path.basename(url)
fullpath = os.path.join(args.download_dir, filename)
if not os.path.exists(fullpath) or args.redownload or older_than_a_day(fullpath):
print(f"Downloading {filename}")
urlretrieve(url, fullpath, DownloadProgressBar())
hashtype, latest, size = parse_latest_stage3_text(fullpath)
size = int(size)
filename = latest
fullpath = os.path.join(args.download_dir, filename)
if not os.path.exists(fullpath) or args.redownload:
LOG.info(f"Downloading {filename}")
url = os.path.join(
config.GENTOO_BASE_STAGE_SYSTEMD_URL if args.profile == "systemd" else \
config.GENTOO_BASE_STAGE_OPENRC_URL,
filename)
urlretrieve(url, fullpath, DownloadProgressBar())
# Verify byte size
stage3size = os.path.getsize(fullpath)
assert size == stage3size, f"Stage 3 size {size} does not match expected value {stage3size}."
verify(args, hashtype, config.GENTOO_BASE_STAGE_SYSTEMD_URL if args.profile == "systemd" else \
config.GENTOO_BASE_STAGE_OPENRC_URL, stage3hashpattern, filename)
return fullpath
def download_portage(args, url=None) -> str:
"""Handle downloading of portage system for installation into cloud image
We always download the latest portage package and rename it to today's date.
If using today's date to grab portage, sometimes depending on timezone, the
package won't be available. If always using latest, worst case scenario is you
have a portage package a day late.
"""
if url is None:
url = config.GENTOO_PORTAGE_FILE
base = os.path.basename(url) # Uses 'latest' filename
today = date.today()
# Write latest to today's date so we don't constantly redownload, but
filename = base.replace("latest", "%d%d%d" % (today.year, today.month, today.day))
fullpath = os.path.join(args.download_dir, filename)
# Portage is always "latest" in this case, so definitely check if older than a day and redownload.
if not os.path.exists(fullpath) or args.redownload or older_than_a_day(fullpath):
LOG.info(f"Downloading {filename} ({base})")
urlretrieve(url, fullpath, DownloadProgressBar())
return fullpath
def download(args, url=None) -> str:
"""Download txt file with iso name and hash type
:Parameters:
- args: Namespace with parsed arguments
- url: str or None. If None, will generate a url to the latest minimal install iso
:Returns:
Full path to the downloaded iso file
Will cause program to exit if iso byte size fails to match expected value.
"""
if url is None:
url = os.path.join(config.GENTOO_BASE_ISO_URL, config.GENTOO_LATEST_ISO_FILE)
# Download the latest txt file
filename = os.path.basename(url)
fullpath = os.path.join(args.download_dir, filename)
if not os.path.exists(fullpath) or args.redownload or older_than_a_day(fullpath):
LOG.info(f"Downloading {fullpath}")
urlretrieve(url, fullpath, DownloadProgressBar())
hashtype, latest, size = parse_latest_iso_text(fullpath)
size = int(size)
# Download the iso file
filename = latest
fullpath = os.path.join(args.download_dir, filename)
if not os.path.exists(fullpath) or args.redownload:
LOG.info(f"Downloading {filename}")
url = os.path.join(config.GENTOO_BASE_ISO_URL, filename)
urlretrieve(url, fullpath, DownloadProgressBar())
# Verify byte size
isosize = os.path.getsize(fullpath)
assert size == isosize, f"ISO size {size} does not match expected value {isosize}."
verify(args, hashtype, config.GENTOO_BASE_ISO_URL, isohashpattern, filename)
return fullpath

359
src/ansible_gentooimgr/gentooimgr/install.py Normal file
View File

@ -0,0 +1,359 @@
"""Configure a Gentoo guest with cloud image settings
This step keeps track of how far it's gotten, so re-running this command
will continue on if an error was to occur, unless --start-over flag is given.
"""
import os
import sys
import shutil
import configparser
from subprocess import Popen, PIPE
import logging
import traceback
import gentooimgr.config
import gentooimgr.configs
import gentooimgr.common
import gentooimgr.chroot
import gentooimgr.kernel
from gentooimgr import LOG
from gentooimgr import HERE
from gentooimgr.configs import *
FILES_DIR = os.path.join(HERE, "..")
def step1_diskprep(args, cfg):
LOG.info("\t:: Step 1: Disk Partitioning")
# http://rainbow.chard.org/2013/01/30/how-to-align-partitions-for-best-performance-using-parted/
# http://honglus.blogspot.com/2013/06/script-to-automatically-partition-new.html
cmds = [
['parted', '-s', f'{cfg.get("disk")}', 'mklabel', 'msdos'],
['parted', '-s', f'{cfg.get("disk")}', 'mkpart', 'primary', '2048s', '100%'],
['partprobe'],
['mkfs.ext4', '-FF', f'{cfg.get("disk")}{cfg.get("partition", 1)}']
]
for c in cmds:
proc = Popen(c, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
completestep(1, "diskprep")
def step2_mount(args, cfg):
LOG.info(f'\t:: Step 2: Mounting {gentooimgr.config.GENTOO_MOUNT}')
proc = Popen(["mount", f'{cfg.get("disk")}{cfg.get("partition")}', cfg.get("mountpoint")])
proc.communicate()
completestep(2, "mount")
def step3_stage3(args, cfg):
LOG.info(f'\t:: Step 3: Stage3 Tarball')
stage3 = cfg.get("stage3") or args.stage3 # FIXME: auto detect stage3 images in mountpoint and add here
if not stage3:
stage3 = gentooimgr.common.stage3_from_dir(FILES_DIR)
proc = Popen(["tar", "xpf", os.path.abspath(stage3), "--xattrs-include='*.*'", "--numeric-owner", "-C",
f'{cfg.get("mountpoint")}'])
proc.communicate()
completestep(3, "stage3")
def step4_binds(args, cfg):
LOG.info(f'\t:: Step 4: Binding Filesystems')
gentooimgr.chroot.bind(verbose=False)
completestep(4, "binds")
def step5_portage(args, cfg):
LOG.info(f'\t:: Step 5: Portage')
portage = cfg.get("portage") or args.portage
if not portage:
portage = gentooimgr.common.portage_from_dir(FILES_DIR)
proc = Popen(["tar", "xpf", portage, "-C", f"{cfg.get('mountpoint')}/usr/"])
proc.communicate()
# Edit portage
portage_env = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'env')
os.makedirs(portage_env, exist_ok=True)
with open(os.path.join(portage_env, 'singlejob.conf'), 'w') as f:
f.write('MAKEOPTS="-j1"\n')
env_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'package.env')
with open(env_path, 'w') as f:
f.write("app-portage/eix singlejob.conf\ndev-util/maturin singlejob.conf\ndev-util/cmake singlejob.conf")
completestep(5, "portage")
def step6_licenses(args, cfg):
LOG.info(f'\t:: Step 6: Licenses')
license_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'package.license')
os.makedirs(license_path, exist_ok=True)
for f, licenses in cfg.get("licensefiles", {}).items():
with open(os.path.join(license_path, f), 'w') as f:
f.write('\n'.join(licenses))
completestep(6, "license")
def step7_repos(args, cfg):
LOG.info(f'\t:: Step 7: Repo Configuration')
repo_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'repos.conf')
os.makedirs(repo_path, exist_ok=True)
# Copy from template
repo_file = os.path.join(repo_path, 'gentoo.conf')
shutil.copyfile(
os.path.join(cfg.get("mountpoint"), 'usr', 'share', 'portage', 'config', 'repos.conf'),
repo_file)
# Regex replace lines
cp = configparser.ConfigParser()
for repofile, data in cfg.get("repos", {}).items():
cp.read(cfg.get("mountpoint") + repofile) # repofile should be absolute path, do not use os.path.join.
for section, d in data.items():
if section in cp:
for key, val in d.items():
# Replace everything after the key with contents of value.
# Sed is simpler than using regex for this purpose.
cp.set(section, key, val)
else:
sys.stderr.write(f"\tWW No section {section} in {repofile}\n")
cp.write(open(cfg.get("mountpoint") + repofile, 'w'))
completestep(7, "repos")
def step8_resolv(args, cfg):
LOG.info(f'\t:: Step 8: Resolv')
proc = Popen(["cp", "--dereference", "/etc/resolv.conf", os.path.join(cfg.get("mountpoint"), 'etc')])
proc.communicate()
# Copy all step files and python module to new chroot
os.system(f"cp /tmp/*.step {cfg.get('mountpoint')}/tmp")
os.system(f"cp -r . {cfg.get('mountpoint')}/mnt/")
completestep(8, "resolv")
def step9_sync(args, cfg):
LOG.info(f"\t:: Step 9: sync")
LOG.info("\t\t:: Entering chroot")
os.chroot(cfg.get("mountpoint"))
os.chdir(os.sep)
os.system("source /etc/profile")
proc = Popen(["emerge", "--sync", "--quiet"])
proc.communicate()
LOG.info("\t\t:: Emerging base")
proc = Popen(["emerge", "--update", "--deep", "--newuse", "--keep-going", "@world"])
proc.communicate()
completestep(9, "sync")
def step10_emerge_pkgs(args, cfg):
LOG.info(f"\t:: Step 10: emerge pkgs")
packages = cfg.get("packages", {})
for oneshot_up in packages.get("oneshots", []):
proc = Popen(["emerge", "--oneshot", "--update", oneshot_up])
proc.communicate()
for single in packages.get("singles", []):
proc = Popen(["emerge", "-j1", single])
proc.communicate()
LOG.info(f"KERNEL PACKAGES {packages.get('kernel')}")
if packages.get("kernel", []):
cmd = ["emerge", "-j", str(args.threads)] + packages.get("kernel", [])
proc = Popen(cmd)
proc.communicate()
cmd = ["emerge", "-j", str(args.threads), "--keep-going"]
cmd += packages.get("keepgoing", [])
proc = Popen(cmd)
proc.communicate()
cmd = ["emerge", "-j", str(args.threads)]
cmd += packages.get("base", [])
cmd += packages.get("additional", [])
cmd += packages.get("bootloader", [])
LOG.info(cmd)
proc = Popen(cmd)
proc.communicate()
completestep(10, "pkgs")
def step11_kernel(args, cfg):
# at this point, genkernel will be installed
LOG.info(f"\t:: Step 11: kernel")
proc = Popen(["eselect", "kernel", "set", "1"])
proc.communicate()
if not args.kernel_dist:
os.chdir(args.kernel_dir)
threads = str(gentooimgr.config.THREADS)
gentooimgr.kernel.build_kernel(args, cfg)
completestep(11, "kernel")
def step12_grub(args, cfg):
LOG.info(f"\t:: Step 12: kernel")
proc = Popen(["grub-install", cfg.get('disk')])
proc.communicate()
code = proc.returncode
if code != 0:
sys.stderr.write(f"Failed to install grub on {cfg.get('disk')}\n")
sys.exit(code)
with open("/etc/default/grub", 'w') as f:
f.write(f"{gentooimgr.kernel.GRUB_CFG}")
proc = Popen(["grub-mkconfig", "-o", "/boot/grub/grub.cfg"])
proc.communicate()
completestep(12, "grub")
def step13_serial(args, cfg):
LOG.info(f"\t:: Step 13: Serial")
os.system("sed -i 's/^#s0:/s0:/g' /etc/inittab")
os.system("sed -i 's/^#s1:/s1:/g' /etc/inittab")
completestep(13, "serial")
def step14_services(args, cfg):
LOG.info(f"\t:: Step 14: Services")
for service in ["acpid", "syslog-ng", "cronie", "sshd", "cloud-init-local", "cloud-init", "cloud-config",
"cloud-final", "ntpd", "nfsclient"]:
if args.profile == "systemd":
proc = Popen(["systemctl", "enable", service])
else:
proc = Popen(["rc-update", "add", service, "default"])
proc.communicate()
completestep(14, "services")
def step15_ethnaming(args, cfg):
LOG.info(f"\t:: Step 15: Eth Naming")
completestep(15, "networking")
def step16_sysconfig(args, cfg):
LOG.info(f"\t:: Step 16: Sysconfig")
with open("/etc/timezone", "w") as f:
f.write("UTC")
proc = Popen(["emerge", "--config", "sys-libs/timezone-data"])
proc.communicate()
with open("/etc/locale.gen", "a") as f:
f.write("en_US.UTF-8 UTF-8\nen_US ISO-8859-1\n")
proc = Popen(["locale-gen"])
proc.communicate()
proc = Popen(["eselect", "locale", "set", "en_US.utf8"])
proc.communicate()
proc = Popen(["env-update"])
proc.communicate()
with open('/etc/sysctl.d/swappiness.conf', 'w') as f:
f.write("vm.swappiness = 0\n")
modloadpath = os.path.join(os.sep, 'etc', 'modules-load.d')
os.makedirs(modloadpath, exist_ok=True)
with open(os.path.join(modloadpath, 'cloud-modules.conf'), 'w') as f:
f.write('\n'.join(gentooimgr.config.CLOUD_MODULES))
cloudcfg = os.path.join(os.sep, 'etc', 'cloud')
if not os.path.exists(cloudcfg):
os.makedirs(cloudcfg, exist_ok=True)
os.makedirs(os.path.join(cloudcfg, 'templates'), exist_ok=True)
with open(os.path.join(cloudcfg, 'cloud.cfg'), 'w') as cfg:
cfg.write(f"{CLOUD_YAML}")
os.chmod(os.path.join(cloudcfg, "cloud.cfg"), 0o644)
with open(os.path.join(cloudcfg, "templates", "hosts.gentoo.tmpl"), 'w') as tmpl:
tmpl.write(f"{HOST_TMPL}") # FIXME:
os.chmod(os.path.join(cloudcfg, "templates", "hosts.gentoo.tmpl"), 0o644)
proc = Popen("sed -i 's/domain_name\,\ domain_search\,\ host_name/domain_search/g' /etc/dhcpcd.conf", shell=True)
proc.communicate()
hostname = os.path.join(os.sep, 'etc', 'conf.d', 'hostname')
with open(hostname, 'w') as f:
f.write(f"{HOSTNAME}\n")
os.chmod(hostname, 0o644)
proc = Popen(["eix-update"])
proc.communicate()
os.remove(os.path.join(os.sep, 'etc', 'resolv.conf'))
completestep(16, "sysconfig")
def step17_fstab(args, cfg):
LOG.info(f"\t:: Step 17: fstab")
with open(os.path.join(os.sep, 'etc', 'fstab'), 'a') as fstab:
fstab.write(f"{cfg.get('disk')}\t/\text4\tdefaults,noatime\t0 1\n")
completestep(17, "fstab")
def completestep(step, stepname, prefix='/tmp'):
with open(os.path.join(prefix, f"{step}.step"), 'w') as f:
f.write("done.") # text in this file is not currently used.
def getlaststep(prefix='/tmp'):
i = 1
found = False
while not found:
if os.path.exists(f"{i}.step"):
i += 1
else:
found = True
return i
def stepdone(step, prefix='/tmp'):
return os.path.exists(os.path.join(prefix, f"{step}.step"))
def configure(args, config: dict):
# Load configuration
if not os.path.exists(gentooimgr.config.GENTOO_MOUNT):
if not args.force:
# We aren't in a gentoo live cd are we?
sys.stderr.write("Your system doesn't look like a gentoo live cd, exiting for safety.\n"
"If you want to continue, use --force option and re-run `python -m gentooimgr install` with your configuration\n")
sys.exit(1)
else:
# Assume we are root as per live cd, otherwise user should run this as root as a secondary confirmation
os.makedirs(gentooimgr.config.GENTOO_MOUNT)
# disk prep
cfg = config
if not stepdone(1): step1_diskprep(args, cfg)
# mount root
if not stepdone(2): step2_mount(args, cfg)
# extract stage
if not stepdone(3): step3_stage3(args, cfg)
# mount binds
if not stepdone(4): step4_binds(args, cfg)
# extract portage
if not stepdone(5): step5_portage(args, cfg)
# Set licenses
if not stepdone(6): step6_licenses(args, cfg)
# repos.conf
if not stepdone(7): step7_repos(args, cfg)
# portage env files and resolv.conf
if not stepdone(8): step8_resolv(args, cfg)
# emerge --sync
if not stepdone(9): step9_sync(args, cfg)
# bindist
if not stepdone(10): step10_emerge_pkgs(args, cfg)
# emerge packages
# configure & emerge kernel (use cloud configuration too)
if not stepdone(11): step11_kernel(args, cfg)
# grub
if not stepdone(12): step12_grub(args, cfg)
# enable serial console
if not stepdone(13): step13_serial(args, cfg)
# services
if not stepdone(14): step14_services(args, cfg)
# eth0 naming
# timezone
if not stepdone(15): step15_ethnaming(args, cfg)
# locale
# set some sysctl things
# set some dhcp things
# hostname
if not stepdone(16): step16_sysconfig(args, cfg)
# fstab
if not stepdone(17): step17_fstab(args, cfg)
# copy cloud cfg?
gentooimgr.chroot.unbind()
# Finish install processes like emaint and eix-update and news read

128
src/ansible_gentooimgr/gentooimgr/kernel.py Normal file
View File

@ -0,0 +1,128 @@
import os
import sys
import re
import shutil
import datetime
from subprocess import Popen, PIPE
import time
import gentooimgr.configs
DEFAULT_KERNEL_CONFIG_PATH = os.path.join(os.sep, 'etc', 'kernel', 'default.config')
def kernel_conf_apply(args, config):
"""Kernel configuration is a direct copy of a full complete config file"""
fname, ext = os.path.splitext(args.config)
# Default is the json file's name but with .config extension.
kernelconfig = os.path.join(gentooimgr.configs.CONFIG_DIR,
config.get("kernel", {}).get("config", f"{fname}.config"))
kernelpath = config.get("kernel", {}).get("path", DEFAULT_KERNEL_CONFIG_PATH)
if os.path.exists(kernelpath):
os.remove(kernelpath)
else:
# Ensure if we have directories specified that they exist
os.makedirs(os.path.dirname(kernelpath), exist_ok=True)
shutil.copyfile(kernelconfig, kernelpath)
def build_kernel(args, config):
if config.get("kernel", {}).get("config") is None:
kernel_default_config(args, config)
os.chdir(args.kernel_dir)
kernelpath = config.get("kernel", {}).get("path", DEFAULT_KERNEL_CONFIG_PATH)
kernel_conf_apply(args, config)
proc = Popen(['genkernel', f'--kernel-config={kernelpath}', '--save-config', '--no-menuconfig', 'all'])
proc.communicate()
kernel_save_config(args, config)
def kernel_default_config(args, config):
os.chdir(args.kernel_dir)
proc = Popen(["make", "defconfig"])
proc.communicate()
def kernel_save_config(args, config):
os.chdir(args.kernel_dir)
"""Saves the current .config file"""
proc = Popen(["make", "savedefconfig"])
proc.communicate()
GRUB_CFG = """
# Copyright 1999-2015 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
# $Header: /var/cvsroot/gentoo-x86/sys-boot/grub/files/grub.default-3,v 1.5 2015/03/25 01:58:00 floppym Exp $
#
# To populate all changes in this file you need to regenerate your
# grub configuration file afterwards:
# 'grub2-mkconfig -o /boot/grub/grub.cfg'
#
# See the grub info page for documentation on possible variables and
# their associated values.
GRUB_DISTRIBUTOR="Gentoo"
# Default menu entry
#GRUB_DEFAULT=0
# Boot the default entry this many seconds after the menu is displayed
#GRUB_TIMEOUT=5
#GRUB_TIMEOUT_STYLE=menu
# Append parameters to the linux kernel command line
# openrc only spits to the last console=tty
GRUB_CMDLINE_LINUX="net.ifnames=0 vga=791 console=tty0 console=ttyS0,115200"
#
# Examples:
#
# Boot with network interface renaming disabled
# GRUB_CMDLINE_LINUX="net.ifnames=0"
#
# Boot with systemd instead of sysvinit (openrc)
# GRUB_CMDLINE_LINUX="init=/usr/lib/systemd/systemd"
# Append parameters to the linux kernel command line for non-recovery entries
#GRUB_CMDLINE_LINUX_DEFAULT=""
# Uncomment to disable graphical terminal (grub-pc only)
GRUB_TERMINAL="serial console"
GRUB_SERIAL_COMMAND="serial --speed=115200"
#GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"
# The resolution used on graphical terminal.
# Note that you can use only modes which your graphic card supports via VBE.
# You can see them in real GRUB with the command `vbeinfo'.
#GRUB_GFXMODE=640x480
# Set to 'text' to force the Linux kernel to boot in normal text
# mode, 'keep' to preserve the graphics mode set using
# 'GRUB_GFXMODE', 'WIDTHxHEIGHT'['xDEPTH'] to set a particular
# graphics mode, or a sequence of these separated by commas or
# semicolons to try several modes in sequence.
#GRUB_GFXPAYLOAD_LINUX=
# Path to theme spec txt file.
# The starfield is by default provided with use truetype.
# NOTE: when enabling custom theme, ensure you have required font/etc.
#GRUB_THEME="/boot/grub/themes/starfield/theme.txt"
# Background image used on graphical terminal.
# Can be in various bitmap formats.
#GRUB_BACKGROUND="/boot/grub/mybackground.png"
# Uncomment if you don't want GRUB to pass "root=UUID=xxx" parameter to kernel
#GRUB_DISABLE_LINUX_UUID=true
# Uncomment to disable generation of recovery mode menu entries
#GRUB_DISABLE_RECOVERY=true
# Uncomment to disable generation of the submenu and put all choices on
# the top-level menu.
# Besides the visual affect of no sub menu, this makes navigation of the
# menu easier for a user who can't see the screen.
GRUB_DISABLE_SUBMENU=y
# Uncomment to play a tone when the main menu is displayed.
# This is useful, for example, to allow users who can't see the screen
# to know when they can make a choice on the menu.
#GRUB_INIT_TUNE="60 800 1"
"""

101
src/ansible_gentooimgr/gentooimgr/qemu.py Normal file
View File

@ -0,0 +1,101 @@
"""Qemu commands to run and handle the image"""
import os
import sys
import argparse
from subprocess import Popen, PIPE
from gentooimgr import LOG
import gentooimgr.config
import gentooimgr.common
def create_image(args, config: dict, overwrite: bool = False) -> str:
"""Creates an image (.img) file using qemu that will be used to create the cloud image
:Parameters:
- config: dictionary/json configuration containing required information
- overwrite: if True, run_image() will call this and re-create.
:Returns:
Full path to image file produced by qemu
"""
image = gentooimgr.common.get_image_name(args, config)
name, ext = os.path.splitext(image)
if os.path.exists(image) and not overwrite:
return os.path.abspath(image)
cmd = ['qemu-img', 'create', '-f', ext[1:], image, str(config.get("imgsize", "12G"))]
proc = Popen(cmd, stderr=PIPE, stdout=PIPE)
stdout, stderr = proc.communicate()
return os.path.abspath(image)
def run_image(
args: argparse.Namespace,
config: dict,
mounts=[]):
"""Handle mounts and run the live cd image
- mount_isos: list of iso paths to mount in qemu as disks.
"""
iso = config.get("iso")
prefix = args.temporary_dir
LOG.info(f"iso from config {iso}")
if iso is None:
iso = gentooimgr.common.find_iso(
os.path.join(
os.path.abspath(os.path.dirname(__file__)),
".."
)
)
LOG.info(f"iso from cwd {iso}")
if not iso:
prefix = config.get('temporary_dir')
iso = gentooimgr.common.find_iso(prefix)
LOG.info(f"iso from {prefix} {iso}")
assert iso, f"iso not found {iso}"
if isinstance(iso, list):
assert len(iso), f"iso list is empty {iso}"
iso = iso[0]
image = gentooimgr.common.get_image_name(args, config)
qmounts = []
mounts.extend(args.mounts)
for i in mounts:
qmounts.append("-drive")
qmounts.append(f"file={i},media=cdrom")
assert image, f"image is empty {image}"
if not os.path.exists(image):
if os.path.exists(os.path.join(prefix, image)):
image = os.path.join(prefix, image)
assert os.path.exists(image), f"image not found {image}"
threads = args.threads
cmd = [
"qemu-system-x86_64",
"-enable-kvm",
"-boot", "d",
"-m", str(config.get("memory", 2048)),
"-smp", str(threads),
"-drive", f"file={image},if=virtio,index=0",
"-cdrom", iso,
"-net", "nic,model=virtio",
"-vga", "virtio",
"-cpu", "kvm64",
"-chardev", "file,id=charserial0,path=gentoo.log",
"-device", "isa-serial,chardev=charserial0,id=serial0",
"-chardev", "pty,id=charserial1",
"-device", "isa-serial,chardev=charserial1,id=serial1"
]
# "-net", "user",
# -net user: network backend 'user' is not compiled into this binary"
cmd += qmounts
LOG.info(cmd)
proc = Popen(cmd, stderr=PIPE, stdout=PIPE)
stdout, stderr = proc.communicate()
if stderr:
LOG.error(str(stderr))

37
src/ansible_gentooimgr/gentooimgr/run.py Normal file
View File

@ -0,0 +1,37 @@
import os
from gentooimgr import LOG
import gentooimgr.config
import gentooimgr.qemu
import gentooimgr.common
def run(args, config: dict) -> None:
LOG.info(": run")
mounts = args.mounts
# Specified image or look for gentoo.{img,qcow2}
image = config.get("imagename") or args.image or gentooimgr.qemu.create_image()
# We need to package up our gentooimgr package into an iso and mount it to the running image
# Why? basic gentoo livecd has no git and no pip installer. We want install to be simple
# and use the same common codebase.
# This will require a couple mount commands to function though.
main_iso = gentooimgr.common.make_iso_from_dir(os.path.join(
os.path.abspath(os.path.dirname(__file__)),
".."
))
assert os.path.isfile(main_iso), f"iso not found {main_iso}"
LOG.info(args)
LOG.info(f'iso={args.iso}')
if args.iso != config['iso']:
LOG.warn(f'iso={args.iso}')
config['iso'] = args.iso
else:
LOG.info(f'iso={args.iso}')
gentooimgr.qemu.run_image(
args,
config,
# Add our generated mount and livecd (assumed)
mounts=[main_iso]
)
LOG.info("done")

15
src/ansible_gentooimgr/gentooimgr/shrink.py Normal file
View File

@ -0,0 +1,15 @@
import os
import datetime
from subprocess import PIPE, Popen
def shrink(args, config, stamp=None):
if stamp is None:
dt = datetime.datetime.utcnow()
# 0 padded month and day timestamp
stamp = f"{dt.year}-{dt.month:02d}-{dt.day:02d}"
name, ext = os.path.splitext(config.get("imagename") or args.img)
# ext includes the .
filename = f"{name}-{stamp}{ext}"
proc = Popen(["virt-sparsify", "--compress", args.img, filename])
proc.communicate()
return filename

54
src/ansible_gentooimgr/gentooimgr/status.py Normal file
View File

@ -0,0 +1,54 @@
"""Step 1: Disk Partitioning
Step 2: Mounting {gentooimgr.config.GENTOO_MOUNT}
Step 3: Stage3 Tarball
Step 4: Binding Filesystems
Step 5: Portage
Step 6: Licenses
Step 7: Repo Configuration
Step 8: Resolv
Step 9: sync
Step 10: emerge pkgs
Step 11: kernel
Step 12: kernel
Step 13: Serial
Step 14: Services
Step 15: Eth Naming
Step 16: Sysconfig
Step 17: fstab
"""
import os
import sys
import json
# from gentooimgr import LOG
import gentooimgr.config
import gentooimgr.configs
from gentooimgr import install
def print_template(args, configjson, prefix='/tmp'):
print(__doc__)
sys.stderr.write(f"the last step to succeed is {install.getlaststep(prefix)}\n")
print(f"the last step to succeed is {install.getlaststep(prefix)}\n")
print(f"""------------------------ STATUS ------------------------
CPU_THREADS = {args.threads or 1}
TEMPORARY_DIRECTORY = {args.temporary_dir}
PROFILE = {args.profile}
""")
print(f"CONFIG {args.config}")
print(json.dumps(configjson, sort_keys=True, indent=4))
# inherit = configjson.get("inherit")
# if inherit:
# print(f"CONFIG {inherit}")
# j = gentooimgr.config.load_default_config(inherit)
# if not j:
# j = gentooimgr.config.load_config(inherit)
#
# print(json.dumps(j, sort_keys=True, indent=4))
# print(f"""------------------------ PACKAGES ------------------------""")
# for k, v in configjson.get("packages").items():
# print(k.upper())
# print("\t" + '\n\t'.join(v))
# print()

0
src/ansible_gentooimgr/gentooimgr/test.py Normal file
View File

189
src/ansible_gentooimgr/library/ansible-keepassxc Executable file
View File

@ -0,0 +1,189 @@
#!/usr/bin/python
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import os
import traceback
from ansible.module_utils.basic import AnsibleModule, missing_required_lib
IMPORT_ERR = None
try:
# import _argon2_xffi_bindings
import pykeepass as keepass
except ImportError:
IMPORT_ERR = traceback.format_exc()
DOCUMENTATION = r'''
---
module: ansible-keepassxc
short_description: Module to read credentials from KeePassXC
version_added: "0.0.1"
description: Module to read credentials from KeePassXC
options:
database:
description: Path to database file
required: true
type: str
password:
description: Database Password
required: true
type: str
keyfile:
description: Path to key file
required: false
type: str
entry:
description: Entry name for the attribute to fetch
required: true
type: str
group:
decription: Group name that the Entry belongs to
required: false
type: str
author:
- Jeremy Lumley (@jlumley)
'''
EXAMPLES = r'''
# Fetch the credentials for the server_1 entry in any group
- name: Fetch server_1 credentials
jlumley.jlumley.ansible-keepassxc:
database: "/secrets/db.kdbx"
password: "s3cure_p4550rd"
entry: "server_1"
# Fetch the reddit entry in the social group
- name: Fetching reddit credentials
jlumley.jlumley.ansible-keepassxc:
database: "/secrets/db.kdbx"
password: "sup3r_s3cure_p4550rd"
entry: "reddit"
group: "social"
# Fetch a custom strig attribute from the github entry
- name: Fetch Github API Token
jlumley.jlumley.ansible-keepassxc:
database: "/secrets/db.kdbx"
password: "d0pe_s3cure_p4550rd"
keyfile: "/secrets/top_secret_key"
entry: "github"
group: "development"
'''
RETURN = r'''
# Return values
username:
description: Username of entry if present
type: str
returned: always
sample: 's3cr3t_us3r'
password:
description: Password of entry if present
type: str
returned: always
sample: 's3cr3t_p455word'
url:
description: Url of entry if present
type: str
returned: always
sample: 'http://reddit.com'
custom_fields:
description: dictionary containing all custom fields
type: dict
returned: always
sample: False
no_log:
description: suppress logging of password
type: bool
returned: never
sample: False
'''
def run_module():
# define available arguments/parameters a user can pass to the module
module_args = dict(
database = dict(type='str', required=True),
password = dict(type='str', required=False,
default=os.environ.get('ANSIBLE_KEEPASSXC_PASSWORD')),
keyfile = dict(type='str', required=False, default=None),
entry = dict(type='str', required=True),
group = dict(type='str', required=False),
no_log = dict(type='bool', required=False, default=False),
)
# seed the result dict in the object
result = dict(
changed=False,
username='',
password='',
url='',
custom_fields={}
)
# Currently no support for a check_mode this maybe added later if
# functionality to modify the database is added later
module = AnsibleModule(
argument_spec=module_args,
supports_check_mode=False,
)
if IMPORT_ERR:
module.fail_json(
msg=missing_required_lib("pykeepass"),
exception=IMPORT_ERR
)
# unlock local keepass database
try:
kp = keepass.PyKeePass(
module.params['database'],
password=module.params['password'],
keyfile=module.params['keyfile'])
except keepass.exceptions.CredentialsError:
module.fail_json(msg='Invalid Credentials')
# find entry
entry = kp.find_entries(
title=module.params['entry'],
group=module.params['group']
)
# fail is entry is not present
if not entry:
module.fail_json(msg=f"Unable to find entry: {module.params['entry']}")
else:
entry = entry[0]
custom_field_keys = entry._get_string_field_keys(exclude_reserved=True)
custom_fields = dict()
for key in custom_field_keys:
custom_fields[key] = entry.get_custom_property(key)
result = dict (
changed=False,
username=entry.username,
password=entry.password,
url=entry.url,
custom_fields=custom_fields
)
# in the event of a successful module execution, you will want to
# simple AnsibleModule.exit_json(), passing the key/value results
module.exit_json(**result)
def main():
run_module()
if __name__ == '__main__':
main()

189
src/ansible_gentooimgr/library/ansible-keepassxc.py Executable file
View File

@ -0,0 +1,189 @@
#!/usr/bin/python
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import os
import traceback
from ansible.module_utils.basic import AnsibleModule, missing_required_lib
IMPORT_ERR = None
try:
# import _argon2_xffi_bindings
import pykeepass as keepass
except ImportError:
IMPORT_ERR = traceback.format_exc()
DOCUMENTATION = r'''
---
module: ansible-keepassxc
short_description: Module to read credentials from KeePassXC
version_added: "0.0.1"
description: Module to read credentials from KeePassXC
options:
database:
description: Path to database file
required: true
type: str
password:
description: Database Password
required: true
type: str
keyfile:
description: Path to key file
required: false
type: str
entry:
description: Entry name for the attribute to fetch
required: true
type: str
group:
decription: Group name that the Entry belongs to
required: false
type: str
author:
- Jeremy Lumley (@jlumley)
'''
EXAMPLES = r'''
# Fetch the credentials for the server_1 entry in any group
- name: Fetch server_1 credentials
jlumley.jlumley.ansible-keepassxc:
database: "/secrets/db.kdbx"
password: "s3cure_p4550rd"
entry: "server_1"
# Fetch the reddit entry in the social group
- name: Fetching reddit credentials
jlumley.jlumley.ansible-keepassxc:
database: "/secrets/db.kdbx"
password: "sup3r_s3cure_p4550rd"
entry: "reddit"
group: "social"
# Fetch a custom strig attribute from the github entry
- name: Fetch Github API Token
jlumley.jlumley.ansible-keepassxc:
database: "/secrets/db.kdbx"
password: "d0pe_s3cure_p4550rd"
keyfile: "/secrets/top_secret_key"
entry: "github"
group: "development"
'''
RETURN = r'''
# Return values
username:
description: Username of entry if present
type: str
returned: always
sample: 's3cr3t_us3r'
password:
description: Password of entry if present
type: str
returned: always
sample: 's3cr3t_p455word'
url:
description: Url of entry if present
type: str
returned: always
sample: 'http://reddit.com'
custom_fields:
description: dictionary containing all custom fields
type: dict
returned: always
sample: False
no_log:
description: suppress logging of password
type: bool
returned: never
sample: False
'''
def run_module():
# define available arguments/parameters a user can pass to the module
module_args = dict(
database = dict(type='str', required=True),
password = dict(type='str', required=False,
default=os.environ.get('ANSIBLE_KEEPASSXC_PASSWORD')),
keyfile = dict(type='str', required=False, default=None),
entry = dict(type='str', required=True),
group = dict(type='str', required=False),
no_log = dict(type='bool', required=False, default=False),
)
# seed the result dict in the object
result = dict(
changed=False,
username='',
password='',
url='',
custom_fields={}
)
# Currently no support for a check_mode this maybe added later if
# functionality to modify the database is added later
module = AnsibleModule(
argument_spec=module_args,
supports_check_mode=False,
)
if IMPORT_ERR:
module.fail_json(
msg=missing_required_lib("pykeepass"),
exception=IMPORT_ERR
)
# unlock local keepass database
try:
kp = keepass.PyKeePass(
module.params['database'],
password=module.params['password'],
keyfile=module.params['keyfile'])
except keepass.exceptions.CredentialsError:
module.fail_json(msg='Invalid Credentials')
# find entry
entry = kp.find_entries(
title=module.params['entry'],
group=module.params['group']
)
# fail is entry is not present
if not entry:
module.fail_json(msg=f"Unable to find entry: {module.params['entry']}")
else:
entry = entry[0]
custom_field_keys = entry._get_string_field_keys(exclude_reserved=True)
custom_fields = dict()
for key in custom_field_keys:
custom_fields[key] = entry.get_custom_property(key)
result = dict (
changed=False,
username=entry.username,
password=entry.password,
url=entry.url,
custom_fields=custom_fields
)
# in the event of a successful module execution, you will want to
# simple AnsibleModule.exit_json(), passing the key/value results
module.exit_json(**result)
def main():
run_module()
if __name__ == '__main__':
main()

233
src/ansible_gentooimgr/library/ansible_gentooimgr.py Executable file
View File

@ -0,0 +1,233 @@
#!/usr/bin/python3
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import os
import sys
import logging
from argparse import Namespace
import pathlib
import traceback
# in the library
if os.environ.get('PLAY_ANSIBLE_SRC',''):
# running from source
mod_path = os.environ.get('PLAY_ANSIBLE_SRC','')
mod_path = os.path.join(mod_path, 'src', 'ansible_gentooimgr')
assert os.path.isdir(mod_path), f"parent {mod_path}"
assert os.path.isfile(os.path.join(mod_path, '__init__.py')),f"index {mod_path}"
assert os.path.isdir(os.path.join(mod_path, 'gentooimgr')), f"sub {mod_path}"
sys.path.append(mod_path)
else:
# in the library
mod_path = os.path.dirname(os.path.realpath('__file__'))
mod_path = os.path.join(mod_path, 'src', 'ansible_gentooimgr')
assert os.path.isdir(mod_path), f"parent {mod_path}"
assert os.path.isfile(os.path.join(mod_path, '__init__.py')),f"index {mod_path}"
assert os.path.isdir(os.path.join(mod_path, 'gentooimgr')), f"sub {mod_path}"
sys.path.append(mod_path)
try:
import gentooimgr
except Exception as e:
sys.stderr.write(f"{mod_path} {sys.path} {traceback.print_exc()}")
raise
import ansible
DOCUMENTATION = rf'''
---
module: gentooimgr
short_description: Gentoo Image Builder for Cloud and Turnkey ISO installers
version_added: "1.0.0"
description:
* This project enables easy access to building ``systemd`` or ``openrc`` -based images.
* Performs automatic download AND verification of the linux iso, stage3 tarball and portage.
* Caches the iso and stage3 .txt files for at most a day before redownloading and rechecking for new files
* Sane and readable cli commands to build, run and test.
* Step system to enable user to continue off at the same place if a step fails
* No heavy packages like rust included ** TODO
options:
action:
description: The action to be run by the image builder
choices:
- build
- run
- status
- install
- chroot
- unchroot
- command
- shrink
- kernel
required: true
# clean test
config:
default: cloud.json
description: init configuration file or or base.json or cloud.json
required: false
loglevel:
default: {logging.INFO}
description: python logging level <= 50, INFO=20
required: false
threads:
default: 1
description: Number of threads to use
required: false
profile:
default: openrc
description: The init system
choices:
- openrc
- systemd
required: false
kernel_dir:
default: /usr/src/linux
description: Where kernel is specified. By default uses the active linux kernel
required: false
portage:
description: Extract the specified portage tarball onto the filesystem
required: false
stage3:
description: Extract the specified stage3 package onto the filesystema
required: false
action_args:
default: []
description: Arguments for some of the actions - UNUSED!
required: false
temporary_dir:
description: Path to temporary directory for downloading files (20G)
required: false
qcow:
description: Path to file to serve as the base image
required: false
# Specify this value according to your collection
# in format of namespace.collection.doc_fragment_name
# extends_documentation_fragment:
# - my_namespace.my_collection.my_doc_fragment_name
author:
- Your Name (@yourGitHubHandle)
'''
#[-y DAYS]
# [-d DOWNLOAD_DIR]
# [-f]
# [--format FORMAT]
EXAMPLES = r'''
# Pass in a message
- name: Test with a message
my_namespace.my_collection.my_test:
name: hello world
# pass in a message and have changed true
- name: Test with a message and changed output
my_namespace.my_collection.my_test:
name: hello world
new: true
# fail the module
- name: Test failure of the module
my_namespace.my_collection.my_test:
name: fail me
'''
RETURN = r'''
# These are examples of possible return values, and in general should use other names for return values.
message:
description: The output message that the test module generates.
type: str
returned: always
sample: 'goodbye'
'''
from ansible.module_utils.basic import AnsibleModule
def run_module():
# define available arguments/parameters a user can pass to the module
module_args = dict(
action=dict(type='str', required=True),
loglevel=dict(type='int', required=False, default=logging.INFO),
threads=dict(type='int', required=False, default=1),
config=dict(type='str', default='cloud.json', required=False),
profile=dict(type='str', required=False),
kernel_dir=dict(type='path', required=False),
portage=dict(type='path', required=False),
stage3=dict(type='path', required=False),
temporary_dir=dict(type='path', required=False, default=pathlib.Path(os.getcwd())),
download_dir=dict(type='path', required=False, default=pathlib.Path(os.getcwd())),
qcow=dict(type='path', required=False),
)
# seed the result dict in the object
# we primarily care about changed and state
# changed is if this module effectively modified the target
# state will include any data that you want your module to pass back
# for consumption, for example, in a subsequent task
result = dict(
changed=False,
original_message='',
message=''
)
# the AnsibleModule object will be our abstraction working with Ansible
# this includes instantiation, a couple of common attr would be the
# args/params passed to the execution, as well as if the module
# supports check mode
module = AnsibleModule(
argument_spec=module_args,
supports_check_mode=True
)
# if the user is working with this module in only check mode we do not
# want to make any changes to the environment, just return the current
# state with no modifications
if module.check_mode:
module.exit_json(**result)
# manipulate or modify the state as needed (this is going to be the
# part where your module will do what it needs to do)
# if module.params.get('thirsty'):
oargs = Namespace(**module.params)
# during the execution of the module, if there is an exception or a
# conditional state that effectively causes a failure, run
# AnsibleModule.fail_json() to pass in the message and the result
result['original_message'] = ""
try:
from gentooimgr.__main__ import main
retval = main(oargs)
except Exception as e:
result['message'] = str(e)
e = traceback.print_exc()
if e: result['original_message'] += f"{e}"
module.fail_json(msg='Exception', **result)
else:
result['message'] = str(retval)
# use whatever logic you need to determine whether or not this module
# made any modifications to your target
if dArgs['action'] in ['status']:
result['changed'] = False
else:
result['changed'] = True
# in the event of a successful module execution, you will want to
# simple AnsibleModule.exit_json(), passing the key/value results
module.exit_json(**result)
def main():
run_module()
if __name__ == '__main__':
main()

5
src/ansible_gentooimgr/requirements.txt Normal file
View File

@ -0,0 +1,5 @@
pydantic
typing
urllib
progressbar
requests

0
src/ansible_gentooimgr/setup.py Normal file
View File