add roles/toxcore/

Makefile

@@ -209,3 +209,4 @@ veryclean:: clean
 
 clean::
 	find . -name \*~ -delete
+	rm roles/*/vars/*.txt

README.md

@@ -151,9 +151,22 @@ On Ansibles from 2.10 and later, you will need the community plugins installed.
 
 ## Advanced Usage
 
+### GentooImgr: Gentoo Image Builder for Cloud and Turnkey ISO installers
+
+There is a modified version of https://github.com/NucleaPeon/gentooimgr/
+where we've modified the code a little to do use Python logging. We can
+still use it for the build stage, but we think the install stage is better
+done using ansible, hence the libvirt_cloud playbook.
+
+The code is in src/ansible_gentooimgr The code is being supported as
+an ansible module using library/ansible_gentooimgr.py which is a work
+in progress; the idea is to use it for the build and status actions,
+but handle the install tasks using ansible.
+
 ### ansible_local.bash
 
 We have a script that calls ansible to run our play: ansible_local.yml
+but sets some defaults
 
 [ -l limit ]
 [ -c connection ]

library/ansible_gentooimgr.py

@@ -0,0 +1,233 @@
+#!/usr/bin/python3
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import sys
+import logging
+from argparse import Namespace
+import pathlib
+import traceback
+
+# in the library
+if os.environ.get('PLAY_ANSIBLE_SRC',''):
+    # running from source
+    mod_path = os.environ.get('PLAY_ANSIBLE_SRC','')
+    mod_path = os.path.join(mod_path, 'src', 'ansible_gentooimgr')
+    assert os.path.isdir(mod_path), f"parent {mod_path}"
+    assert os.path.isfile(os.path.join(mod_path, '__init__.py')),f"index {mod_path}"
+    assert os.path.isdir(os.path.join(mod_path, 'gentooimgr')), f"sub {mod_path}"
+    sys.path.append(mod_path)
+else:
+    # in the library
+    mod_path = os.path.dirname(os.path.realpath('__file__'))
+    mod_path = os.path.join(mod_path, 'src', 'ansible_gentooimgr')
+    assert os.path.isdir(mod_path), f"parent {mod_path}"
+    assert os.path.isfile(os.path.join(mod_path, '__init__.py')),f"index {mod_path}"
+    assert os.path.isdir(os.path.join(mod_path, 'gentooimgr')), f"sub {mod_path}"
+    sys.path.append(mod_path)
+try:
+    import gentooimgr
+except Exception as e:
+    sys.stderr.write(f"{mod_path} {sys.path} {traceback.print_exc()}")
+    raise
+import ansible
+
+DOCUMENTATION = rf'''
+---
+module: gentooimgr
+
+short_description: Gentoo Image Builder for Cloud and Turnkey ISO installers
+
+
+version_added: "1.0.0"
+
+description:
+* This project enables easy access to building ``systemd`` or ``openrc`` -based images.
+* Performs automatic download AND verification of the linux iso, stage3 tarball and portage.
+* Caches the iso and stage3 .txt files for at most a day before redownloading and rechecking for new files
+* Sane and readable cli commands to build, run and test.
+* Step system to enable user to continue off at the same place if a step fails
+* No heavy packages like rust included ** TODO
+
+options:
+    action:
+        description: The action to be run by the image builder
+        choices:
+            - build
+            - run
+            - status
+            - install
+            - chroot
+            - unchroot
+            - command
+            - shrink
+            - kernel
+        required: true
+        # clean test
+    config:
+        default: cloud.json
+        description: init configuration file or or base.json or cloud.json
+        required: false
+    loglevel:
+        default: {logging.INFO}
+        description: python logging level <= 50, INFO=20
+        required: false
+    threads:
+        default: 1
+        description: Number of threads to use
+        required: false
+    profile:
+        default: openrc
+        description: The init system
+        choices:
+            - openrc
+            - systemd
+        required: false
+    kernel_dir:
+        default: /usr/src/linux
+        description: Where kernel is specified. By default uses the active linux kernel
+        required: false
+    portage:
+        description: Extract the specified portage tarball onto the filesystem
+        required: false
+    stage3:
+        description: Extract the specified stage3 package onto the filesystema
+        required: false
+    action_args:
+        default: []
+        description: Arguments for some of the actions - UNUSED!
+        required: false
+    temporary_dir:
+        description: Path to temporary directory for downloading files (20G)
+        required: false
+    qcow:
+        description: Path to file to serve as the base image
+        required: false
+
+# Specify this value according to your collection
+# in format of namespace.collection.doc_fragment_name
+# extends_documentation_fragment:
+#     - my_namespace.my_collection.my_doc_fragment_name
+
+author:
+    - Your Name (@yourGitHubHandle)
+'''
+
+#[-y DAYS]
+#                  [-d DOWNLOAD_DIR]
+# [-f]
+#                  [--format FORMAT]
+
+EXAMPLES = r'''
+# Pass in a message
+- name: Test with a message
+  my_namespace.my_collection.my_test:
+    name: hello world
+
+# pass in a message and have changed true
+- name: Test with a message and changed output
+  my_namespace.my_collection.my_test:
+    name: hello world
+    new: true
+
+# fail the module
+- name: Test failure of the module
+  my_namespace.my_collection.my_test:
+    name: fail me
+'''
+
+RETURN = r'''
+# These are examples of possible return values, and in general should use other names for return values.
+message:
+    description: The output message that the test module generates.
+    type: str
+    returned: always
+    sample: 'goodbye'
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def run_module():
+    # define available arguments/parameters a user can pass to the module
+    module_args = dict(
+        action=dict(type='str', required=True),
+        loglevel=dict(type='int', required=False, default=logging.INFO),
+        threads=dict(type='int', required=False, default=1),
+        config=dict(type='str', default='cloud.json', required=False),
+        profile=dict(type='str', required=False),
+        kernel_dir=dict(type='path', required=False),
+        portage=dict(type='path', required=False),
+        stage3=dict(type='path', required=False),
+        temporary_dir=dict(type='path', required=False, default=pathlib.Path(os.getcwd())),
+        download_dir=dict(type='path', required=False, default=pathlib.Path(os.getcwd())),
+        qcow=dict(type='path', required=False),
+    )
+
+    # seed the result dict in the object
+    # we primarily care about changed and state
+    # changed is if this module effectively modified the target
+    # state will include any data that you want your module to pass back
+    # for consumption, for example, in a subsequent task
+    result = dict(
+        changed=False,
+        original_message='',
+        message=''
+    )
+
+    # the AnsibleModule object will be our abstraction working with Ansible
+    # this includes instantiation, a couple of common attr would be the
+    # args/params passed to the execution, as well as if the module
+    # supports check mode
+    module = AnsibleModule(
+        argument_spec=module_args,
+        supports_check_mode=True
+    )
+
+    # if the user is working with this module in only check mode we do not
+    # want to make any changes to the environment, just return the current
+    # state with no modifications
+    if module.check_mode:
+        module.exit_json(**result)
+
+    # manipulate or modify the state as needed (this is going to be the
+    # part where your module will do what it needs to do)
+    # if module.params.get('thirsty'):
+
+    oargs = Namespace(**module.params)
+    # during the execution of the module, if there is an exception or a
+    # conditional state that effectively causes a failure, run
+    # AnsibleModule.fail_json() to pass in the message and the result
+    result['original_message'] = ""
+    try:
+        from gentooimgr.__main__ import main
+        retval = main(oargs)
+    except Exception as e:
+        result['message'] = str(e)
+        e = traceback.print_exc()
+        if e: result['original_message'] += f"{e}"
+        module.fail_json(msg='Exception', **result)
+    else:
+        result['message'] = str(retval)
+
+    # use whatever logic you need to determine whether or not this module
+    # made any modifications to your target
+    if dArgs['action'] in ['status']:
+        result['changed'] = False
+    else:
+        result['changed'] = True
+
+    # in the event of a successful module execution, you will want to
+    # simple AnsibleModule.exit_json(), passing the key/value results
+    module.exit_json(**result)
+
+
+def main():
+    run_module()
+
+
+if __name__ == '__main__':
+    main()
+

roles/ansible-gentoo_install/tasks/disk.yml

@@ -18,15 +18,21 @@
     
 - name: create disklabel
   command: parted -s {{ AGI_install_disk }} mklabel {{ AGI_install_disklabel }}
+  register: disklabel_out
+  #   stderr: 'Warning: Error fsyncing/closing /dev/nbd1: Input/output error'
+  failed_when: false
+
+- name: disklabel_out
+  debug:
+    var: disklabel_out
   
 # We need to leave a small gap at the beginning of the disk, or grub won't be
 # able to install to the MBR
 - name: create boot partition
   shell: |
     parted -s {{ AGI_install_disk }} mkpart primary ext2 1M 200M
-    e2label  {{ AGI_install_disk }}p1 boot
   args:
-    creates: "{{ AGI_install_disk }}p2"
+    creates: "{{ AGI_install_disk }}p1"
 
 - name: mark boot partition as active
   shell: |
@@ -35,14 +41,12 @@
 - name: create swap partition
   shell: |
     parted -s {{ AGI_install_disk }} -- mkpart primary linux-swap 201M 2200M
-    mkswap -L swap  "{{ AGI_install_disk }}p2"
   args:
     creates: "{{ AGI_install_disk }}p2"
  
 - name: create root partition
   shell: |
     parted -s {{ AGI_install_disk }} -- mkpart primary ext4 2201M 20070M
-    e2label  {{ AGI_install_disk }}p3 root
   args:
     creates: "{{ AGI_install_disk }}p3"
 
@@ -61,3 +65,8 @@
   check_mode: false
   when: not ansible_check_mode
 
+- name: label partitions
+  shell: |
+    e2label  {{ AGI_install_disk }}p3 root
+    e2label  {{ AGI_install_disk }}p1 boot
+    mkswap -L swap  "{{ AGI_install_disk }}p2"

roles/toxcore/Makefile

@@ -0,0 +1,24 @@
+# -*-mode: makefile; fill-column: 75; tab-width: 8; coding: utf-8-unix -*-
+
+ROLE=hostvms
+USER=`grep PRIV_SKEL_USERS_LIST /usr/local/etc/testforge/testforge.bash|sed -e 's/.*=//' -e 's/"//g'`
+
+VAGRANT_DOCTEST=python3.sh ../../../src/ansible/bin/vagrant_doctest-ssh.py
+PYTHON_DOCTEST=python3.sh -m doctest
+SSH_DOCTEST=python3.sh ../../../src/ansible/bin/vagrant_doctest-ssh.py -p 2222 --box="vagrant@127.0.0.1"
+LOCAL_DOCTEST=/var/local/bin/testforge_run_doctest2.bash
+
+DOCTEST=$(LOCAL_DOCTEST)
+
+default:: local
+
+test:: local # hourly
+	sudo -u ${USER} /var/local/src/var_local_$(ROLE).bash $@
+
+hourly::
+	/var/local/bin/$(ROLE)_hourly.bash
+
+local::
+	 $(DOCTEST) overlay/Linux/var/local/share/doc/txt/$(ROLE)3.txt
+#	 $(DOCTEST) overlay/Linux/var/local/share/doc/txt/proxy2.txt
+

roles/toxcore/defaults/main.yml

@@ -0,0 +1,30 @@
+# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
+---
+
+# be careful - these are running on the guest which may have no relationship with the other roles
+
+# "ansible_virtualization_role": ["host", "guest"]
+# "ansible_virtualization_type": ["kvm", "virtualbox", "chroot",  ...]
+
+TOXCORE_USR_LOCAL: "{{ USR_LOCAL }}"
+TOXCORE_VAR_LOCAL: "{{ USR_LOCAL }}"
+TOXCORE_LOG_DIR: "{{ TOXCORE_VAR_LOCAL }}/var/log"
+
+TOXCORE_HOURLY_LOG: "{{TOXCORE_LOG_DIR}}/hourly"
+TOXCORE_DAILY_LOG: "{{TOXCORE_LOG_DIR}}/daily"
+TOXCORE_WEEKLY_LOG: "{{TOXCORE_LOG_DIR}}/weekly"
+TOXCORE_MONTHLY_LOG: "{{TOXCORE_LOG_DIR}}/monthly"
+
+TOXCORE_VMS_DIR: "{{TOXCORE_VAR_LOCAL}}/data/Vms"
+TOXCORE_LOCAL_SRC: "{{TOXCORE_VAR_LOCAL}}/src"
+
+TOXCORE_GPG_SERVER: "{{BASE_GPG_SERVER}}"
+
+TOXCORE_GENTOO_FROM_MP: "{{BOX_GENTOO_FROM_MP}}"
+
+# TOXCORE_CTOXCORE_CMAKE:
+
+TOXCORE_FEATURES:
+  - libvirt
+  - docker
+  

roles/toxcore/meta/main.yml

@@ -0,0 +1,15 @@
+# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
+
+---
+galaxy_info:
+  author: Funtoo
+  company: Remote
+  description: Toxcore provisioning
+  license: MIT
+  min_ansible_version: 2.9
+  platforms:
+    - name: Gentoo
+  categories:
+    - system
+
+dependencies: []

roles/toxcore/tasks/Devuan.yml

@@ -0,0 +1,33 @@
+# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
+
+---
+
+- name: "toxcore Devuan.yml"
+  debug:
+    verbosity: 1
+    msg: "toxcore Devuan.yml"
+
+- name: "install toxcore_debs_inst packages"
+  environment:
+    - "RUNLEVEL": 1
+  apt:
+    force_apt_get: true
+    name: "{{ item }}"
+    state: latest
+    update_cache: no
+  when:
+    - item != '' and item != []
+    - not ansible_check_mode
+    - "BASE_ARE_CONNECTED|default('') != ''"
+  with_items:
+    - "{{ toxcore_debs_inst }}"
+    - "{{ toxcore_qemu_debs_inst    if 'qemu' in TOXCORE_FEATURES or 'libvirt' in TOXCORE_FEATURES else [] }}"
+    - "{{ proxy_libvirt_debs_inst    if 'libvirt' in TOXCORE_FEATURES else [] }}"
+    - "{{ toxcore_libvirt_debs_inst    if 'libvirt' in TOXCORE_FEATURES else [] }}"
+    - "{{ toxcore_virtualbox_debs_inst if 'virtualbox' in TOXCORE_FEATURES else [] }}"
+    - "{{ toxcore_vagrant_debs_inst    if 'vagrant' in TOXCORE_FEATURES else [] }}"
+    - "{{ toxcore_packer_debs_inst     if 'packer' in TOXCORE_FEATURES else [] }}"
+    - "{{ toxcore_docker_debs_inst     if 'docker' in TOXCORE_FEATURES else [] }}"
+  ignore_errors: "{{ BASE_PKG_IGNORE_ERRORS }}"
+
+

roles/toxcore/tasks/Gentoo.yml

@@ -0,0 +1,171 @@
+# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
+
+---
+- name: "DEBUG: toxcore Gentoo.yml"
+  debug:
+    verbosity: 1
+    msg: "DEBUG: Including toxcore Gentoo.ym"
+
+#    - "{{ ansible_distribution }}/{{ BOX_SERVICE_MGR }}"
+
+- name: install toxcore packages
+  environment: "{{ portage_proxy_env }}"
+  shell:  |
+    role=toxcore
+    cd {{ BASE_ROOT_LOG_DIR }} || exit 2
+    /usr/local/bin/usr_local_base.bash box_gentoo_emerge {{item}} || exit $?
+  with_items:
+    - "{{ toxcore_pkgs_inst }}"
+    - "{{ toxcore_qemu_pkgs_inst if 'qemu' in TOXCORE_FEATURES }}"
+    - "{{ toxcore_qemu_pkgs_inst if 'libvirt' in TOXCORE_FEATURES }}"
+    - "{{ toxcore_libvirt_pkgs_inst if 'libvirt' in TOXCORE_FEATURES }}"
+    - "{{ toxcore_docker_pkgs_inst if 'DOCKER' in TOXCORE_FEATURES }}"
+  ignore_errors: "{{ BASE_PKG_IGNORE_ERRORS }}"
+  when:
+    - item != '' and item != []
+    - BASE_ARE_CONNECTED|default('') != ''
+    - "{{ ansible_virtualization_role|replace('NA', 'host') == 'host' }}"
+
+- name: install toxcore packages GUEST
+  environment: "{{ portage_proxy_env }}"
+  shell:  |
+    cd {{ BASE_ROOT_LOG_DIR }} || exit 2
+    /usr/local/bin/usr_local_base.bash box_gentoo_emerge {{item}} || exit $?
+  with_items:
+    - "{{ toxcore_pkgs_inst_guest }}"
+  ignore_errors: "{{ BASE_PKG_IGNORE_ERRORS }}"
+  when:
+    - item != '' and item != []
+    - BASE_ARE_CONNECTED|default('') != ''
+    - "{{ ansible_virtualization_role|replace('NA', 'host') != 'host' }}"
+
+- name: /etc/conf.d/consolefont
+  blockinfile:
+    dest: "/etc/{{ETC_CONF_D}}/consolefont"
+    marker: "# {mark} ANSIBLE MANAGED BLOCK proxy Gentoo"
+    mode: 0644
+    owner: "{{BOX_ROOT_USER}}"
+    group: "{{BOX_ROOT_GROUP}}"
+    create: yes
+    block: |
+      # for 80x24
+      consolefont="ter-v24b"
+
+- name: /etc/local.d/rc.local.start later
+  shell: |
+    # not systemd
+    [ -d /etc/local.d ] || exit 0
+    [ -f "/etc/local.d/rc.local.start" ] && exit 0
+    echo /etc/rc.local > /etc/local.d/rc.local.start
+    chmod 755  /etc/local.d/rc.local.start
+    cat >> /etc/rc.local << EOF
+    /etc/init.d/consolefont stop; /etc/init.d/consolefont start
+    stty -F /dev/tty1 cols 80 rows 24
+    grep vda /proc/partitions && \
+        e2label /dev/vda3 root && \
+        e2label /dev/vda1 boot
+    sed -e 's/^#L/L/' -i /etc/fstab
+    EOF
+    bash /etc/rc.local
+    exit 0
+  when:
+    - not ansible_check_mode
+    - BOX_SERVICE_MGR != 'systemd' # maybe
+
+# safe ones
+- block:
+
+  - name: "/etc/portage/make.conf base Gentoo PORTAGE_ELOG"
+    blockinfile:
+      dest: /etc/portage/make.conf
+      create: false
+      marker: "# {mark} ANSIBLE MANAGED BLOCK base Gentoo.yml [PORTAGE_ELOG]"
+      block: |
+        PORTAGE_ELOG_CLASSES="warn error"
+        # NOT syslog
+        PORTAGE_ELOG_SYSTEM="save"
+  
+  - name: "/etc/portage/make.conf base Gentoo2.yml CFLAGS"
+    blockinfile:
+      dest: /etc/portage/make.conf
+      create: false
+      marker: "# {mark} ANSIBLE MANAGED BLOCK base Gentoo.yml [CFLAGS]"
+      block: |
+        # -pipe
+        CFLAGS="-mtune=generic -O2"
+        CXXFLAGS="-mtune=generic -O2"
+  
+  - name: "/etc/portage/make.conf base Gentoo.yml PORT_LOGDIR"
+    blockinfile:
+      dest: /etc/portage/make.conf
+      create: false
+      marker: "# {mark} ANSIBLE MANAGED BLOCK base Gentoo.yml [PORT_LOGDIR]"
+      block: |
+        PORT_LOGDIR="/var/log/portage"
+        PORTAGE_ELOG_CLASSES="log warn error info"
+        PORTAGE_ELOG_SYSTEM="echo:log,warn save:log,warn,error,info syslog:error"
+  
+  - name: "/etc/portage/make.conf base Gentoo.yml ACCEPT_KEYWORDS"
+    blockinfile:
+      dest: /etc/portage/make.conf
+      create: false
+      marker: "# {mark} ANSIBLE MANAGED BLOCK base Gentoo.yml [ACCEPT_KEYWORDS]"
+      block: |
+        # In ACCEPT_KEYWORDS, ~amd64 is used for current 64-bit builds;
+        # There is no tilde for the stable build.
+        ACCEPT_KEYWORDS="~amd64"
+  
+  - name: "/etc/portage/make.conf base Gentoo.yml GENTOO_MIRRORS"
+    blockinfile:
+      dest: /etc/portage/make.conf
+      create: false
+      marker: "# {mark} ANSIBLE MANAGED BLOCK base Gentoo.yml [GENTOO_MIRRORS]"
+      block: |
+        GENTOO_MIRRORS="http://distfiles.gentoo.org"
+  
+  - name: "/etc/portage/make.conf Gentoo2 base"
+    blockinfile:
+      dest: /etc/portage/make.conf
+      create: false
+      marker: "# {mark} ANSIBLE MANAGED BLOCK base Gentoo2.yml [COLLISION_IGNORE]"
+      block: |
+        # 2016-01 added for some py packages
+        COLLISION_IGNORE="/lib/modules/* *.py[co] __init__.py"
+  
+  - name: "/etc/portage/make.conf Gentoo2 UNINSTALL_IGNORE"
+    blockinfile:
+      dest: /etc/portage/make.conf
+      create: false
+      marker: "# {mark} ANSIBLE MANAGED BLOCK base Gentoo2.yml [UNINSTALL_IGNORE]"
+      block: |
+        # 2017-01 added for kernel compiling
+        UNINSTALL_IGNORE="/usr/src /opt /usr/lib32 /usr/lib64/python2.7"
+  
+  - name: "/etc/portage/make.conf Gentoo2 FEATURES-preserve-libs"
+    blockinfile:
+      dest: /etc/portage/make.conf
+      create: false
+      marker: "# {mark} ANSIBLE MANAGED BLOCK base Gentoo2.yml [FEATURES-preserve-libs]"
+      block: |
+        # https://wiki.gentoo.org/wiki/Project:Toolchain/libcrypt_implementation
+        FEATURES="${FEATURES} preserve-libs"
+  
+  - name: "/etc/portage/make.conf Gentoo2 FEATURES"
+    blockinfile:
+      dest: /etc/portage/make.conf
+      create: false
+      marker: "# {mark} ANSIBLE MANAGED BLOCK base Gentoo2.yml [CFLAGS]"
+      block: |
+        #Please adjust your CFLAGS as desired, information can be found here: https://wiki.gentoo.org/wiki/CFLAGS
+        #Do not modify these FLAGS unless you know what you are doing, always check the defaults first with "portageq envvar CFLAGS"
+        #This is the default for pentoo at the time of build:
+          #CFLAGS="-Os -mtune=nocona -pipe  -frecord-gcc-switches"
+          #A safe choice would be to keep whatever Pentoo defaults are, but optimize for your specific machine:
+          CFLAGS="-mtune=generic -O2 -pipe"
+          #If you do change your CFLAGS, it is best for all the compile flags to match so uncomment the following three lines:
+          CXXFLAGS="-mtune=generic -O2 -pipe"
+          FCFLAGS="${CFLAGS}"
+          FFLAGS="${CFLAGS}"
+  
+  when: true
+

roles/toxcore/tasks/Ubuntu.yml

@@ -0,0 +1,48 @@
+# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
+
+---
+
+- name: "toxcore Ubuntu.yml"
+  debug:
+    verbosity: 1
+    msg: "toxcore Ubuntu.yml"
+
+- name: uninstall toxcore_lxd_debs_absent packages
+  environment:
+    - "RUNLEVEL": 1
+  apt:
+    force_apt_get: true
+    name: "{{item}}"
+    state: absent
+    update_cache: no
+  when:
+    - item != ''
+    - not ansible_check_mode
+    - "'lxd' not in TOXCORE_FEATURES"
+  with_items: "{{ toxcore_lxd_debs_absent }}"
+
+- name: "install toxcore_debs_inst packages"
+  environment:
+    - "RUNLEVEL": 1
+  shell: |
+    apt-get install -y \
+     "{{ toxcore_debs_inst|join('') }}" \
+     "{{ toxcore_qemu_debs_inst|join('') if 'qemu' in TOXCORE_FEATURES else '' }}" \
+     "{{ toxcore_libvirt_debs_inst|join('') if 'libvirt' in TOXCORE_FEATURES else '' }}" \
+     "{{ toxcore_docker_debs_inst|join('') if 'docker' in TOXCORE_FEATURES else '' }}" \
+      {{ '--print-uris' if BASE_ARE_CONNECTED|default('') == '' else '' }}
+  when:
+    - "item != ''"
+    - not ansible_check_mode
+    - "BASE_ARE_CONNECTED|default('') != ''"
+  ignore_errors: "{{ BASE_PKG_IGNORE_ERRORS }}"
+
+- block:
+
+  - name: "toxcore Ubuntu18.yml"
+    debug:
+      verbosity: 1
+      msg: "toxcore Ubuntu18.yml"
+
+  when: ansible_distribution_major_version == 18
+

roles/toxcore/tasks/Ubuntu18.yml

@@ -0,0 +1,7 @@
+# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
+---
+
+- debug:
+    verbosity: 1
+    msg: "hostvms/Ubuntu18"
+

roles/toxcore/tasks/bootstrap_nbd.yml

@@ -0,0 +1,83 @@
+# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
+
+---
+
+- name: "DEBUG: base bootstrap_nbd.yml nbd_out={{nbd_disk}}"
+  debug:
+    verbosity: 1
+    msg: "DEBUG: Including bootstrap_nbd nbd_out={{nbd_disk}}"
+
+- block:
+
+  - name: "partition the disk"
+    shell: |
+      cat /proc/partitions | grep {{BASE_NBD_DEV}}   || exit 1
+      cat /proc/partitions | grep {{BASE_NBD_DEV}}p1 || exit 0
+      [ {{BASE_NBD_DEV}}p1 = {{nbd_disk}} ]
+      parted -s {{nbd_disk}} mklabel msdos
+      parted -s {{nbd_disk}} mkpart primary 2048s 100%
+      partprobe
+      mkfs.ext4 -FF {{nbd_disk}}p1
+  
+  - name: "mount the disk"
+    shell: |
+      df | grep {{nbd_disk}}p1 && exit 0
+      [ -d {{BASE_NBD_MP}} ] || mkdir  {{BASE_NBD_MP}}
+      mount {{nbd_disk}}p1 {{BASE_NBD_MP}}
+      [ -d {{BASE_NBD_MP}}/lost+found ]
+  
+  - name: "check the disk"
+    shell: |
+      df | grep {{nbd_disk}}p1 || exit 1
+      [ -d {{BASE_NBD_MP}}/lost+found ] || exit 2
+      
+   
+  - name: check the downloads step5
+    shell: |
+      stage3_asc=stage3-amd64-openrc-20231217T170203Z.tar.xz.sha256
+      stage3_xz=stage3-amd64-openrc-20231217T170203Z.tar.xz
+      sha256sum -c $stage3_asc
+      tar xJpf $stage3 --xattrs-include='*.*' --numeric-owner -C {{BASE_NBD_MP}
+      
+      portage_xz=portage-20231221.tar.xz
+      tar xpJf $portage -C {{BASE_NBD_MP}}/usr
+
+      [ -d etc/portage/package.env ] || mkdir -p etc/portage/package.env
+      cat >> etc/portage/package.env/singlejob.txt << EOF
+      app-portage/eix singlejob.conf
+      dev-util/maturin singlejob.conf
+      dev-util/cmake singlejob.conf
+
+      [ -d etc/portage/package.license ] || mkdir -p etc/portage/package.license
+      EOF
+        
+    chdir: "{{BASE_NBD_MP}}"
+    creates: "{{BASE_NBD_MP}}/etc"
+    
+  - name: chroot into the partition - step4
+    shell: |
+      df | grep {{nbd_disk}}p1 || exit 1
+      [ -d {{BASE_NBD_MP}}/lost+found ] || exit 2
+      [ -d {{BASE_NBD_MP}}/proc ] && exit 0
+      /usr/local/sbin/base_chroot.bash {{BASE_NBD_MP}}
+    register: base_bootstrap_chroot_rc
+
+  - name: check the sources
+    shell: |
+      true
+
+  - name: install the downloads
+    shell: |
+      /usr/local/sbin/base_chroot_unbind.bash  {{BASE_NBD_MP}}
+    when: base_bootstrap_chroot_rc.rc == 0
+
+  - name: stop here
+    shell: |
+      exit 1
+    
+  delegate_to: localhost
+  when:
+    - nbd_disk != ''
+
+#gpg --keyserver hkps://keys.gentoo.org --recv-keys 0xBB572E0E2D182910
+

roles/toxcore/tasks/daily.yml

@@ -0,0 +1,37 @@
+# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
+---
+
+- name: "toxcore daily.yml"
+  debug:
+    msg: "toxcore daily.yml ansible_virtualization_role {{ansible_virtualization_role|replace('NA', 'host')}}"
+
+# these should test on the host
+
+# delegate_to: localhost? - no - per test
+- name: "usr_local_toxcore.bash"
+  environment: "{{ shell_proxy_env }}"
+  shell: |
+    umask 0002
+    bash {{TOXCORE_USR_LOCAL}}/src/usr_local_toxcore.bash \
+              {{ 'test' if not ansible_check_mode }}
+    exit 0
+  args:
+    chdir: "{{TOXCORE_USR_LOCAL}}/src"
+  become: yes
+  become_user: "{{ BOX_USER_NAME }}"
+  ignore_errors: true
+  check_mode: false
+  
+- name: "run toxcore_daily.bash"
+  environment: "{{ shell_proxy_env }}"
+  shell: |
+    umask 0027
+    echo "INFO: toxcore_log_daily {{HARDEN_LOG_DIR}}"
+    cd {{USR_LOCAL}}/bin
+    toxcore_daily.bash
+  register: toxcore_log_daily
+  notify: summary of logs
+  ignore_errors: true
+  when:
+    - not ansible_check_mode
+    - false

roles/toxcore/tasks/hourly.yml

@@ -0,0 +1,58 @@
+# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
+---
+
+- name: "toxcore hourly.yml"
+  debug:
+    msg: "toxcore hourly.yml ansible_virtualization_role {{ansible_virtualization_role|replace('NA', 'host')}}"
+
+# these should test on the host
+- block:
+  - name: check ifconfig
+    # was toxcore_bin.bash
+    shell: |
+      echo WARN: this maybe vagrant only
+      ifconfig eth0 | grep 'inet ' || \
+          ifconfig eth0 netmask 255.255.255.0 192.168.122.22
+    when:
+      - false # this is vagrant only
+      - "ansible_virtualization_role|replace('NA', 'host') != 'host'"
+      
+  - name: check guest-agent
+    # was toxcore_bin.bash
+    shell: |
+      UNIT=qemu-guest-agent
+      S=/etc/init.d/$UNIT
+      $S $UNIT status || {
+          retval=$?
+          echo WARN: $UNIT not running
+          exit 0 # $retval
+          }
+      exit 0
+      
+  when: "ansible_virtualization_role|replace('NA', 'host') != 'host'"
+
+- name: run toxcore_check_modules.bash
+  # was toxcore_bin.bash
+  shell: |
+    bash toxcore_check_modules.bash {{TOXCORE_FEATURES|join(' ')}}
+  args:
+    chdir: "{{ TOXCORE_VAR_LOCAL }}/bin"
+  environment: "{{ shell_proxy_env }}"
+  register: toxcore_log_bin
+  notify: summary of logs
+  # FixMe?
+  ignore_errors: true
+
+- name: "run toxcore_hourly.bash"
+  environment: "{{ shell_proxy_env }}"
+  shell: |
+    umask 0027
+    echo "INFO: toxcore_log_hourly {{HARDEN_LOG_DIR}}"
+    cd {{VAR_LOCAL}}/bin
+    toxcore_hourly.bash
+  register: toxcore_log_hourly
+  notify: summary of logs
+  ignore_errors: true
+  when:
+    - not ansible_check_mode
+    - false

roles/toxcore/tasks/main.yml

@@ -0,0 +1,342 @@
+# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
+
+---
+
+- name: "toxcore main.yml"
+  debug:
+    verbosity: 1
+    msg: "toxcore main.yml BOX_OS_FAMILY={{BOX_OS_FAMILY}} {{BOX_GENTOO_FROM_MP}} {{ansible_virtualization_role|replace('NA', 'host')}}"  
+  
+- name: toxcore include_vars
+  include_vars: "{{item}}.yml"
+  with_items:
+    - Linux
+    - "{{ ansible_distribution }}{{ ansible_distribution_major_version }}"
+  tags: always
+
+- name: "rsync toxcore root_overlay"
+  synchronize:
+    src: "roles/toxcore/overlay/{{item}}/"
+    dest: /
+    compress: no
+    copy_links: yes
+    archive: no
+    recursive: yes
+    links: no
+    owner: no
+    perms: no
+    times: yes
+    rsync_opts: "{{base_rsync_opts}}"
+  with_items:
+    - Linux
+    - "{{ ansible_distribution }}"
+  notify:
+    - chmod /var/local
+  when:
+    - not ansible_check_mode
+    # FixAns: This remote host is being accessed via chroot instead so it cannot work
+    - ansible_connection|default('') not in PLAY_NORSYNC_CONNECTIONS
+  tags:
+    - always
+
+- name: rsync root_overlay - tar before running
+  unarchive:
+    dest: /
+    src: "{{item}}.tar"
+    keep_newer: true
+    owner: "{{BOX_ROOT_USER}}"
+    # extra_opts: "{{ BASE_UNTAR_ARGS }}"
+  with_items:
+    - Linux
+    - "{{ ansible_distribution }}"
+  notify:
+    - chmod /usr/local
+  ignore_errors: true
+  when:
+    - not ansible_check_mode
+    # FixAns: This remote host is being accessed via chroot instead so it cannot work
+    - ansible_connection|default('') in PLAY_NORSYNC_CONNECTIONS
+
+- name: "make /var/local/var/log/testforge toxcore"
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0775
+  with_items:
+    - "{{ TOXCORE_LOG_DIR }}"
+    - "{{ TOXCORE_DAILY_LOG }}"
+#    - "{{ TOXCORE_WEEKLY_LOG }}"
+#    - "{{ TOXCORE_MONTHLY_LOG }}"
+  tags:
+    - hourly
+    - daily
+    - weekly
+    - monthly
+
+- name: "/var/local/bin/toxcore_sign_modules.bash modules needed toxcore_kmods_in_host"
+  shell: |
+    KVER=`cat /proc/cmdline |sed -e 's/ .*//' -e 's/BOOT_IMAGE=//' -e 's/kernel-[a-z]*-x86_64-//'`
+    KDIR=/lib/modules/$KVER
+    DIR=$KDIR/misc
+    [ -d $DIR ] || { echo WARN: no directory $DIR ; exit 0 ; }
+    /var/local/bin/toxcore_sign_modules.bash {{ toxcore_kmods_in_host|flatten|join(' ') }}
+    
+  when: false
+  # this changes with kernel 4 -> 5
+
+- name: modprobe VM modules toxcore_kmods_in_host
+  modprobe:
+    name: "{{ item }}"
+    state: "{{ 'absent' if ansible_virtualization_role|replace('NA', 'host') != 'host' else 'present'}}"
+  when:
+    - item != ''
+  with_items: "{{ toxcore_kmods_in_host }}"
+  ignore_errors: true
+
+- name: modprobe VM modules toxcore_kmods_not_in_host
+  modprobe:
+    name: "{{ item }}"
+    state: absent
+  with_items: "{{ toxcore_kmods_not_in_host }}"
+  when: false
+  # I'm not sure kvm should be missing anymore 5+
+  ignore_errors: true
+
+- name: include by-platform tasks
+  include_tasks: "{{ ansible_distribution }}.yml"
+
+- name: grub.cfg from roles/ansible-gentoo_install/tasks/
+  shell: |
+    LINE="rd.skipfsck=1 ipv6.disable=1 console=tty1 lang=en keymap=us"
+    # LINE="$LINE pti=on doscsi iommu=pt  amd_iommu=on debugfs=off efi=disable_early_pci_dma extra_latent_entropy init_on_free=1 kvm.nx_huge_pages=force l1tf=full,force mce=0 mds=full,nosmt nosmt=force page_alloc.shuffle=1 pti=on random.trust_cpu=off slab_nomerge slub_debug=FZ spec_store_bypass_disable=on spectre_v2=on tsx_async_abort=full,nosmt vsyscall=none "
+    LINE="$LINE intel_iommu=on vga=0x315 text"
+    grep  /boot /etc/fstab || exit 1
+    df | grep /boot || mount /boot || exit 2
+    [ -d /boot/grub ] || exit 3
+    [ -f /boot/grub/grub.cfg ] || exit 4
+    [ -f /boot/grub/grub.cfg ] && cp -p /boot/grub/grub.cfg /boot/grub/grub.cfg.dst
+    sed -e 's@ ro *$@ '"$LINE"' ro@' -i /boot/grub/grub.cfg
+  ignore_errors: true
+
+- name: add standard_users to groups
+  user:
+    name: "{{ item.0 }}"
+    append: true
+    groups: "{{ item.1 }}"
+  when:
+    - item != ''
+  # some groups may not be there
+  ignore_errors: true
+  with_nested:
+    - "{{ base_system_users }}"
+    - "{{ toxcore_standard_users_groups }}"
+
+- name: "make a directory for /data/Vms"
+  file:
+    path: "{{item }}"
+    state: directory
+    mode: 0775
+  with_items:
+    - "{{TOXCORE_VMS_DIR}}"
+    - "{{TOXCORE_VMS_DIR}}/Instances"
+    - "{{TOXCORE_VMS_DIR}}/Qemu"
+    - "{{TOXCORE_VMS_DIR}}/Boxes"
+#?    - "{{TOXCORE_VMS_DIR}}/vagrant.d/tmp"
+  check_mode: false
+
+- name: "toxcore gpg keys system"
+  # Option --keyserver must be used to
+  environment: "{{ shell_proxy_env }}"
+  shell: |
+    /usr/bin/gpg --list-keys | grep "{{ item.primary }}" || \
+      /usr/bin/gpg --recv-keys "{{ item.uid }}" # --keyserver "{{ TOXCORE_GPG_SERVER }}"
+  with_items: "{{ toxcore_gpg_keys_system }}"
+  when:
+    - toxcore_gpg_keys_system|length > 0
+    - BASE_ARE_CONNECTED|default('') != ''
+  ignore_errors: true
+
+- name: "toxcore gpg keys gentoo"
+  environment: "{{ shell_proxy_env }}"
+  shell: |
+    [ -f /usr/share/openpgp-keys/gentoo-release.asc ] && exit 0
+    gpg --import /usr/local/share/openpgp-keys/gentoo-release.asc 
+ 
+# this should not run as root
+# delegate_to: localhost? - no - per test
+- name: "usr_local_toxcore.bash"
+  environment: "{{ shell_proxy_env }}"
+  shell: |
+    umask 0002
+    bash {{TOXCORE_USR_LOCAL}}/src/usr_local_toxcore.bash \
+              {{ 'check' if ansible_check_mode }}
+    exit 0
+  args:
+    chdir: "{{TOXCORE_USR_LOCAL}}/src"
+  become: yes
+  become_user: "{{ BOX_USER_NAME }}"
+  ignore_errors: true
+  check_mode: false
+  
+- name: "include_tasks toxcore vms as root"
+  include_tasks:
+    file: "{{LOOP_ITEM}}.yml"
+    apply:
+      environment: "{{ proxy_env }}"
+  when:
+    - LOOP_ITEM  != '' and LOOP_ITEM  != []
+  with_items:
+    - "vms"
+  loop_control:
+    loop_var: LOOP_ITEM
+
+- name: "include_tasks toxcore users on the command host"
+  include_tasks:
+    file: "{{ LOOP_USER_F[1] }}"
+    apply:
+      environment: "{{ proxy_env }}"
+      become_user: "{{ LOOP_USER_F[0] }}"
+  when:
+    - "LOOP_USER_F[1] != ''"
+    - "ansible_virtualization_role|replace('NA', 'host') == 'host'"
+  with_nested:
+    -
+      - "{{ toxcore_system_users }}"
+    -
+      - users.yml
+  loop_control:
+    loop_var: LOOP_USER_F
+
+- name: "include_tasks toxcore users as user"
+  include_tasks:
+    file: "{{ LOOP_USER_F[1] }}.yml"
+    apply:
+      environment: "{{ proxy_env }}"
+      become_user: "{{ LOOP_USER_F[0] }}"
+  when:
+    - "LOOP_USER_F[1] != ''"
+    - "ansible_virtualization_role|replace('NA', 'host') == 'host'"
+    - false
+  with_nested:
+    - "{{ toxcore_system_users }}"
+    -
+#no      - users
+      - "{{ 'libvirt_users' if 'libvirt' in TOXCORE_FEATURES else '' }}"
+#      - "{{ 'qemu_users' if 'qemu' in TOXCORE_FEATURES else '' }}"
+#      - "{{ 'docker_users' if 'docker' in TOXCORE_FEATURES else '' }}"
+  loop_control:
+    loop_var: LOOP_USER_F
+    
+- name: enable and start toxcore services
+  service:
+    name: "{{ item }}"
+    enabled: yes
+    state: started
+  when:
+    - "item != ''"
+    - ansible_connection|default('') not in PLAY_CHROOT_CONNECTIONS
+    - "toxcore_services_enabled|length > 0"
+  with_items: "{{ toxcore_services_enabled }}"
+  ignore_errors: true
+
+- name: start toxcore services
+  service:
+    name: "{{ item }}"
+    enabled: no
+    state: started
+  when:
+    - "item != ''"
+    - ansible_connection|default('') not in PLAY_CHROOT_CONNECTIONS
+    - false
+  with_items: "{{ toxcore_services_started }}"
+  ignore_errors: true
+
+- name: stop toxcore services
+  service:
+    name: "{{ item }}"
+    enabled: no
+    state: stopped
+  failed_when: false
+  when:
+    - "item != ''"
+    - ansible_connection|default('') not in PLAY_CHROOT_CONNECTIONS
+    - false
+  with_items: "{{ toxcore_services_stopped }}"
+
+- name: run ansible-gentoo_install
+  include_role:
+    name: ansible-gentoo_install
+  when:
+    - "ansible_virtualization_role|replace('NA', 'host') == 'host'"
+#    - BOX_OS_FAMILY == 'Gentoo' or BOX_GENTOO_FROM_MP not in ['/', '']
+
+  # Set BOX_NBD_OVERLAY_DIR environment variable to override storage location for VMs
+  #   -b      Bridge interface to use (defaults to virbr0)
+  #   -m      MAC address to use (default is to use a randomly-generated MAC)
+  #   -p      ansible users plaintext password
+  #   -k      Full path and name of the ansible user's public key file (required)
+ 
+- name: run toxcore_create-vm.bash
+  delegate_to: localhost
+  shell: |
+    export BOX_NBD_OVERLAY_DIR=${BOX_NBD_OVERLAY_DIR:-"${HOME}/vms/virsh"}
+    [ '{{BOX_NBD_OVERLAY_NAME}}' != '' ] || exit 1
+    [ '{{BOX_NBD_OVERLAY_DIR}}' != '' ] || exit 2
+    [ '{{BOX_NBD_BASE_PUBKEY}}' != '' ] || exit 3
+    [ '{{BOX_NBD_BASE_QCOW}}' != '' ] || exit 4
+    [ -f '{{BOX_NBD_BASE_QCOW}}' ] || exit 5
+    toxcore_create-vm.bash \
+             -n {{BOX_NBD_OVERLAY_NAME}} \
+             -k {{BOX_NBD_BASE_PUBKEY}} \
+             -i {{BOX_NBD_BASE_QCOW}} \
+             -s {{BOX_NBD_OVERLAY_GB}} \
+             -c {{BOX_NBD_OVERLAY_CPUS}} \
+             -r {{BOX_NBD_OVERLAY_RAM}} \
+             -d {{BOX_NBD_OVERLAY_DIR}} \
+             -b virbr0 \
+             -p gentoo \
+             -o gentoo 
+  ignore_errors: true
+  when:
+    - "ansible_virtualization_role|replace('NA', 'host') == 'host'"
+
+# run this last
+
+# works with apply tags hourly
+- name: toxcore hourly include_tasks
+  include_tasks:
+    file: "hourly.yml"
+    apply:
+      tags:
+        - always
+  with_items:
+    - "{{ toxcore_system_users }}"
+  loop_control:
+    loop_var: LOOP_USER
+  tags:
+    - always
+
+- name: toxcore daily include_tasks
+  include_tasks:
+    file: "daily.yml"
+    apply:
+      environment: "{{ proxy_env }}"
+      tags:
+        - daily
+  tags:
+    - daily
+
+- name: toxcore weekly include_tasks
+  include_tasks:
+    file: "weekly.yml"
+    apply:
+      environment: "{{ proxy_env }}"
+      tags:
+        - weekly
+  tags:
+    - weekly
+  when: false
+
+
+# replace this ^A with a control-A to provoke an error in the error handler of yamlint
+# yaml.reader.ReaderError: unacceptable character #x0001: special characters are not allowed

roles/toxcore/tasks/users.yml

@@ -0,0 +1,118 @@
+# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
+
+---
+
+- name: "toxcore users.yml LOOP_USER_F[0]"
+  debug:
+    verbosity: 1
+    msg: "toxcore users.yml LOOP_USER_F={{LOOP_USER_F[0]}}"
+
+- block:
+    
+  - name: /etc/sysconfig/docker
+    blockinfile:
+      dest: /etc/sysconfig/docker
+      create: yes
+      mode:  0644
+      marker: "# {mark} ANSIBLE MANAGED BLOCK proxy proxy_post.yml"
+      state: "{{ 'present' if HTTP_PROXYHOST != '' else 'absent' }}"
+      block: |
+        HTTP_PROXY={{HTTP_PROXYTYPE}}://{{HTTP_PROXYHOST}}:{{HTTP_PROXYPORT}}
+        HTTPS_PROXY={{HTTPS_PROXYTYPE}}://{{HTTPS_PROXYHOST}}:{{HTTPS_PROXYPORT}}
+
+  when:
+    - "'docker' in HOSTVMS_FEATURES|default([])"
+
+- name: Run c-toxcore ctest on the tester
+  delegate_to: localhost
+  shell: |
+    [ -d {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build ] || exit 0
+    cd {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build || exit 1
+    ctest
+    exit 0
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_TCP_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_announce_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_bootstrap_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_conference_av_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_conference_double_invite_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_conference_invite_merge_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_conference_peer_nick_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_conference_simple_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_conference_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_conference_two_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_crypto_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_dht_getnodes_api_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_encryptsave_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_file_saving_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_file_transfer_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_forwarding_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_friend_connection_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_friend_request_spam_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_friend_request_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_group_general_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_group_invite_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_group_message_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_group_moderation_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_group_save_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_group_state_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_group_sync_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_group_tcp_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_group_topic_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_invalid_tcp_proxy_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_invalid_udp_proxy_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_lan_discovery_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_lossless_packet_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_lossy_packet_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_network_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_onion_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_overflow_recvq_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_overflow_sendq_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_reconnect_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_save_compatibility_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_save_friend_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_save_load_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_send_message_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_set_name_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_set_status_message_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_tcp_relay_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_tox_dispatch_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_tox_events_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_tox_many_tcp_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_tox_many_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_tox_strncasecmp_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_toxav_basic_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_toxav_many_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_typing_test
+# {{TOXCORE_LOCAL_SRC}}/c-toxcore/_build/auto_tests/auto_version_test
+
+- name: Run c-toxcore dockerfiles on the tester
+  delegate_to: localhost
+  shell: |
+    [ -d {{TOXCORE_LOCAL_SRC}}/dockerfiles ] || exit 0
+    cd {{TOXCORE_LOCAL_SRC}}/dockerfiles || exit 1
+    exit 0
+
+# - alpine-s390x
+# - alpine-x86_64
+# - buildfarm
+# - compcert
+# - flutter
+# - frama-c
+# - freebsd
+# - ghc-android
+# - ghc
+# - haskell
+# - infer
+# - kythe
+# - nacl
+# - windows-qt5
+# - windows
+
+    
+- name: Run toxygen_wrapper on the tester
+  delegate_to: localhost
+  shell: |
+    [ -d {{TOXCORE_LOCAL_SRC}}/toxygen_wrapper ] || exit 0
+    cd {{TOXCORE_LOCAL_SRC}}/toxygen_wrapper || exit 1
+    exit 0
+

roles/toxcore/tasks/vms.yml

@@ -0,0 +1,144 @@
+# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
+
+---
+
+- name: "toxcore vms.yml"
+  debug:
+    verbosity: 1
+    msg: "toxcore vms.yml TOXCORE_FEATURES={{TOXCORE_FEATURES}}"
+
+- name: "assert /dev/kvm"
+  shell: |
+    [ -e /dev/kvm ]
+  delegate_to: localhost
+  
+- name: "make a directory for /etc/pki/qemu"
+  file:
+    path: "{{item }}"
+    state: directory
+    owner: "{{BOX_ROOT_USER}}"
+    group: "{{BOX_ROOT_GROUP}}"
+    mode: 0770
+  when:
+    - "item != ''"
+  with_items:
+    # see /etc/libvirt/qemu.conf
+    - "{{ '/etc/pki/qemu' if ( 'qemu' in TOXCORE_FEATURES or 'libvirt' in TOXCORE_FEATURES ) else '' }}"
+
+- block:
+
+  - name: increase fs.inotify.max_user_instances (default 128)
+    blockinfile:
+      dest: /etc/sysctl.d/70_toxcore_libvirt.conf
+      marker: "# {mark} ANSIBLE MANAGED BLOCK toxcore lxd"
+      mode: 0440
+      group: "{{BOX_ALSO_GROUP}}"
+      create: yes
+      block: |
+        net.ipv4.ip_forward = 1
+
+  # NB this is per user
+  - name: check ulimit
+    shell: |
+      [ `ulimit -n` -lt 4913709 ]
+    register: ulimit_retval
+
+  # Kernel ulimit is less than the expected value! This might induce RC test
+  - name: /etc/security/limits.conf
+    blockinfile:
+      dest: /etc/security/limits.conf
+      create: yes
+      marker: "# {mark} ANSIBLE MANAGED BLOCK harden vms.yml lxd"
+      backup: 'yes'
+      mode: 0644
+      owner: "{{BOX_ROOT_USER}}"
+      group: "{{BOX_ROOT_GROUP}}"
+      block: |
+        root    soft    nofile  1048576
+        root    hard    nofile  1048576
+        # libvirt/whonix seems to run out at 1024 user tor in the contrainer
+        *       soft    nofile  1048576
+        *       hard    nofile  1048576
+        *       soft    memlock unlimited
+        *       hard    memlock unlimited
+    # FixMe: either this or @ docker break su with a cryptic
+    # pam_open_session: Permission denied
+    # with dmesg entries of
+    # elogind-daemon[7069]: Failed to open pin file: No such file or
+    # elogind-daemon[7069]: Failed to allocate manager object: No such file or
+    # and then later when with no limts.conf but still with a chown error on ~/
+    # su: Error in services module
+
+    # need this in libvirt guest.
+    when:
+      # do this anyway for tor and everybody else
+      - true or ulimit_retval.rc|default(1) == 0
+
+  when:
+    - true or ansible_connection|default('') not in PLAY_CHROOT_CONNECTIONS
+
+# https://github.com/lxc/lxc/issues/3068
+# lxc-create -t download -n my-container -- --keyserver hkp://p80.pool.sks-keyservers.net:80
+# Hope this helps others if it didn't at least help you
+# libvirt lxc docker
+- block:
+
+  # see also https://stackoverflow.com/questions/26415833/docker-ignores-limits-conf-trying-to-solve-too-many-open-files-error
+  # FixMe: conflicts with harden - use sysctl.d
+  # add to your /etc/sysctl.conf : fs.file-max = 64000
+  - name: Add fs.file-max to /etc/sysctl.conf
+    lineinfile:
+      dest: /etc/sysctl.conf
+      regexp: '^#* *{{item.key}}.*'
+      line: "{{item.key}} = {{item.val}}"
+      create: true
+      state: present
+    with_items:
+      - { key: "fs.file-max", val: "4913709" }
+      #? - {key: "net.ipv4.ip_forward", val: "0"}
+      #? not sure about this - see docker check-config.sh
+      - { key: "kernel.keys.root_maxkeys", val: "1000000" }
+    check_mode: false
+
+  - name: /etc/sysctl.conf.d/20_hugepages.conf
+    check_mode: false
+    lineinfile:
+      dest: /etc/sysctl.conf.d/20_hugepages.conf
+      regexp: '^#* *{{item.key}}.*'
+      line: "{{item.key}} = {{item.val}}"
+      state: present
+    with_items:
+# https://wiki.archlinux.org/title/KVM#Enabling_huge_pages
+# boot cmdline hugepages=
+      - { key: "vm.nr_hugepages", val: "550" }
+
+  when:
+    - ansible_virtualization_role|replace('NA', 'host') == 'host' or
+        ansible_virtualization_role|replace('NA', 'host') == 'guest'
+
+  rescue:
+    - debug:
+        verbosity: 1
+        msg: "Ignoring error"
+    
+#libvirt
+- block:
+
+  # /usr/sbin/dnsmasq --conf-file=/var/lib/libvirt/dnsmasq/network.conf --leasefile-ro --dhcp-script=/usr/libexec/libvirt_leaseshelper
+  - name: "/bind-dynamic/bind-interfaces/ -i /var/lib/libvirt/dnsmasq/default.conf"
+    shell: |
+      [ -f {{item}} ] || exit 0
+      sed -e 's/bind-dynamic/bind-interfaces/' -i {{item}}
+      #? virsh net-edit default
+    # maybe just check ifconfig for ...
+    # bogus? var/lib?
+    with_items:
+      - /var/lib/libvirt/dnsmasq/network.conf
+      - /var/lib/libvirt/dnsmasq/default.conf
+    ignore_errors: true
+
+  when:
+    - "'libvirt' in TOXCORE_FEATURES" # or 'lxd' in TOXCORE_FEATURES"
+    - ( ansible_virtualization_type is not defined or
+        not (ansible_virtualization_type in ['chroot', 'lxc', 'lxd', 'docker', 'qemu', 'kvm']))
+

roles/toxcore/vars/Devuan5.yml

@@ -0,0 +1,213 @@
+# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
+
+---
+
+# TOXCORE_FEATURES: ['virtualbox', 'libvirt', docker']
+
+toxcore_system_users:   "{{ BOX_USER_NAME }}"
+toxcore_also_users: "{{ BOX_ALSO_USERS }}"
+
+toxcore_standard_users_groups:
+  - disk
+  - "{{ 'libvirt-qemu' if 'libvirt' in TOXCORE_FEATURES else '' }}"
+  - "{{ 'kvm' if ( 'qemu' in TOXCORE_FEATURES or 'libvirt' in TOXCORE_FEATURES ) else '' }}"
+  - "{{ 'docker' if 'docker' in TOXCORE_FEATURES else '' }}"
+
+toxcore_libvirt_services:
+  - "{{ 'libvirtd' if 'libvirt' in TOXCORE_FEATURES else '' }}"
+  - "{{ 'virtlogd' if 'libvirt' in TOXCORE_FEATURES else '' }}"
+  - "{{ 'docker' if 'docker' in TOXCORE_FEATURES else '' }}"
+
+toxcore_debs_inst:
+  - gpg
+  - nbdkit
+  - nbd-client
+  - nbd-server
+  - python3-yaml
+  - gparted
+  - ebtables
+  - dnsmasq-base
+  # vagrant-libvirt/README.md
+  - libxslt-dev
+  - libxml2-dev
+  - zlib1g-dev
+  - iptables
+  - zstd
+  - bridge-utils
+  - sdparm
+  - hdparm
+  - libxml2
+  - zerofree
+  - openssh-sftp-server
+  - sshfs
+  - xmlstarlet
+  - libtss2-tcti-swtpm0
+# toxcore
+  - libconfig-dev
+  - libgtest-dev
+  - ninja-build
+  - pkg-config
+  - zip
+  - grep
+  - file
+  - ca-certificates
+  - autotools-dev
+  - autoconf
+  - automake
+  - git
+  - bc
+  - wget
+  - rsync
+  - cmake
+  - make
+  - pkg-config
+  - libtool
+  - ssh
+  - gzip
+  - tar
+  - unzip
+  - libcurl4-gnutls-dev
+  - coreutils
+  - libavutil-dev
+  - libsodium-dev
+  - libffms2-4
+  - libgpac10
+  - libx264-dev
+  - x264
+  - libv4lconvert0
+  - libv4l-dev
+  - libv4l-dev
+  - libv4l2rds0
+  - v4l-conf
+  - v4l-utils
+  - libv4l-dev
+  - libtool
+  - autotools-dev
+  - automake
+  - checkinstall
+  - check
+  - git
+  - yasm
+  - libv4lconvert0
+  - libv4l-dev
+  - libopus-dev
+  - libvpx-dev
+  - pkg-config
+  - libx264-dev
+  - libavcodec-dev
+  - libavdevice-dev
+# nbd
+  - genisoimage
+  - bridge-utils
+  - guestfsd
+  - libvirt-clients
+  - libvirt-daemon
+  - libvirt-daemon-system
+  - libvirt-daemon-driver-storage-zfs
+  - python3-libvirt
+  - virt-manager
+  - virtinst
+  - python3-hacking
+#  - cloud-init
+
+toxcore_qemu_debs_inst:
+  - qemu-kvm
+  - qemu
+  - qemu-block-extra
+  - qemu-system-common
+  - qemu-system-data
+  - qemu-system-x86
+  - qemu-utils
+  - guestfsd
+  - grub-firmware-qemu
+  - nbd-client
+  - aqemu
+  - seabios
+  - squashfs-tools
+  - attr
+  - zerofree
+  - nbdkit
+  - sgabios
+  - lzop
+  - firmware-realtek
+  - firmware-linux-free
+
+proxy_libvirt_debs_inst:
+  - libvirt-daemon-system
+  - libvirt-daemon
+  - libvirt-daemon-driver-vbox
+  - libvirt-daemon-driver-qemu
+  - libvirt-clients
+  - libvirt-daemon-driver-storage-zfs
+  - python3-libvirt
+  - virt-manager
+  - virtinst
+  - usbutils
+  - usb.ids
+  - guestfsd
+  - libguestfs-perl
+  - libguestfs-tools
+  #
+  - genisoimage
+  - bridge-utils
+  - guestfsd
+  - python3-hacking
+#?  - cloud-init
+
+toxcore_lxd_debs_absent:
+  - lxd
+  - lxd-client
+  - lxcfs
+  - liblxc1
+  - liblxc-common
+
+toxcore_lxd_debs_inst:
+  - lxcfs
+  - zfsutils-linux
+
+# see proxy_libvirt_debs_inst
+toxcore_libvirt_debs_inst:
+  - python3-libvirt
+  - libvirt-dev
+  - libguestfs
+  - libguestfs-tools
+  # OUCH 4G!
+  - libguestfs-appliance
+  - virt-top
+
+toxcore_packer_debs_inst:
+  - jq
+
+toxcore_docker_debs_inst:
+  - aufs-tools
+  - cgroup-lite
+  - linux-image-extra-virtual
+  - docker.io
+#  - docker-compose
+
+toxcore_ansible_debs_inst:
+  - jinja
+  - pycrypto
+  - python-ecdsa
+  - python-httplib2
+  - python-jinja2
+  - python-markupsafe
+  - python-paramiko
+
+toxcore_aws_debs_inst:
+# for aws
+  - boto
+  - bs4
+  - awscli
+
+toxcore_pips2_inst: []
+  
+toxcore_pips3_inst:
+  - pycrypto
+  - pywinrm
+  - requests-unixsocket
+  - ovirt-engine-sdk-python
+  - msgpack_python
+  - pysha3
+  - pycryptodomex
+  

roles/toxcore/vars/Gentoo2.yml

@@ -0,0 +1,101 @@
+# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
+---
+
+toxcore_system_users:   "{{ base_system_users }}"
+toxcore_also_users: "{{ base_also_users }}"
+
+toxcore_standard_users_groups:
+  - disk
+  - "{{ 'libvirt' if 'libvirt' in TOXCORE_FEATURES else '' }}"
+  - "{{ 'qemu' if ( 'qemu' in TOXCORE_FEATURES or 'libvirt' in TOXCORE_FEATURES ) else '' }}"
+  - "{{ 'kvm' if ( 'qemu' in TOXCORE_FEATURES or 'libvirt' in TOXCORE_FEATURES ) else '' }}"
+  - "{{ 'docker' if 'docker' in TOXCORE_FEATURES else '' }}"
+
+
+toxcore_libvirt_services:
+  - libvirtd
+#  - qemu-guest-agent
+
+toxcore_pkgs_inst_guest: []
+
+toxcore_pkgs_inst:
+  - net-firewall/iptables
+  - app-portage/gentoolkit
+  - sys-apps/gptfdisk
+  - app-admin/testdisk
+  - app-arch/zstd
+  - net-firewall/ebtables
+  - net-misc/bridge-utils
+  - sys-apps/sdparm
+  - sys-apps/hdparm
+  ### virt-manager
+  - media-libs/libvpx
+  - net-libs/libpsl
+  - x11-libs/libxcb
+  - x11-libs/libxkbcommon
+  - dev-libs/libxml2
+  - dev-python/argcomplete
+  - dev-python/libvirt-python
+  - dev-python/pygobject
+  - dev-python/requests
+  - gnome-base/dconf
+  - net-libs/gtk-vnc
+  - net-misc/spice-gtk
+  - sys-apps/dbus
+  - x11-libs/gtk+
+  - x11-libs/gtksourceview
+  - x11-libs/vte
+
+toxcore_pkgs_masked:
+  - x11-drivers/xf86-video-amdgpu
+  - x11-drivers/xf86-video-mga
+  - x11-drivers/xf86-video-radeon
+  - x11-drivers/xf86-video-siliconmotion
+  - x11-drivers/xf86-video-via
+
+toxcore_zfs_pkgs_inst:
+  - sys-fs/zfs-kmod
+  - sys-fs/zfs
+
+toxcore_qemu_pkgs_inst:
+  - app-emulation/qemu
+  - app-emulation/aqemu
+  - app-emulation/spice
+  - net-misc/spice-gtk
+  - qemu-guest-agent
+
+toxcore_libvirt_pkgs_inst:
+  - app-emulation/libvirt
+  - dev-python/libvirt
+  - app-emulation/virt-manager
+  - app-emulation/virt-viewer
+  - x11-drivers/xf86-video-qxl
+#  - sys-block/open-iscsi
+  - sys-firmware/edk2-ovmf
+  - app-emulation/libguestfs
+  - sys-block/nbd
+#?  - app-emulation/libguestfs-appliance
+  - dev-python/libvirt-python
+  - app-emulation/virtiofsd
+
+toxcore_docker_pkgs_inst:
+  - sys-process/tini
+  - app-emulation/docker
+  #? - dev-python/docker-pycreds
+  - dev-python/dockerpty
+  - dev-python/docker-py
+  - app-containers/docker
+#  - app-containers/docker-compose
+
+toxcore_pips2_inst: []
+
+toxcore_pips3_inst:
+  - pycrypto
+  - pywinrm
+  - requests-unixsocket
+  - ovirt-engine-sdk-python
+  - msgpack_python
+  - pysha3
+  - pycryptodomex
+  - pyanalyze
+

roles/toxcore/vars/Linux.yml

@@ -0,0 +1,37 @@
+# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
+---
+
+# from BOX
+toxcore_system_users: "{{ base_system_users }}"
+toxcore_standard_users: "{{ base_standard_users }}"
+
+toxcore_etc_initd_libvirt_name: libvirtd
+
+toxcore_kmods_in_host:
+  - "{{ 'kvm' if ( ( 'qemu' in TOXCORE_FEATURES or 'libvirt' in TOXCORE_FEATURES ) and 'virtualbox' not in TOXCORE_FEATURES ) else '' }}"
+  - "{{ 'qxl' if ( 'qemu' in TOXCORE_FEATURES or 'libvirt' in TOXCORE_FEATURES ) else '' }}"
+  - "{{ 'nbd' if ( 'qemu' in TOXCORE_FEATURES or 'libvirt' in TOXCORE_FEATURES ) else '' }}"
+
+toxcore_kmods_not_in_host:
+  # I'm not sure kvm should be missing anymore 5+
+  # FixMe: how do these conflict? qemu libvirt
+  - "{{ 'kvm' if 'virtualbox' in TOXCORE_FEATURES else '' }}"
+
+toxcore_gpg_keys_system:
+
+  - uid: "30737D12308C9D0C882FC34B57CB0A121BAECB2E"
+    primary: "70ACBB6BFEE7BC572A8941D19266C4FA11FD00FD"
+    name: "Daniel Robbins (metro:node) <drobbins@funtoo.org>"
+    key: "9266C4FA11FD00FD"
+
+toxcore_services_enabled: []
+#  - qemu-guest-agent
+
+toxcore_services_started:
+  - "{{ toxcore_libvirt_services if 'libvirt' in TOXCORE_FEATURES else [] }}"
+  # not on Gentoo 5
+  #?  - "{{ 'docker'   if 'docker'  in TOXCORE_FEATURES else '' }}"
+
+toxcore_services_stopped:
+  - "{{ toxcore_libvirt_services if 'libvirt' not in TOXCORE_FEATURES else [] }}"
+

roles/toxcore/vars/Makefile

@@ -0,0 +1,25 @@
+# -*-mode: text; fill-column: 75; tab-width: 8; coding: utf-8-dos -*-
+
+test::
+
+update:: funtoo
+
+p_src.yaml:: /p/src/_p_src.yaml
+	( cd /p/src && $(MAKE) $(MFLAGS) funtoo )
+
+world.lis:: /var/lib/portage/world Makefile
+	cp -p /var/lib/portage/world world.lis
+	grep -h '^ *- .*/' main.yml base.yml p_src.yaml \
+	| sed -e 's@/@\\/@' -e 's@^ *- @/@' -e 's@$$@/d@' > world.sed
+	sed -f world.sed -i world.lis
+	
+funtoo:: world.lis Makefile
+	cp /dev/null python_world.lis
+	grep ^dev-python world.lis | while read file ; do \
+		grep -q "^- $$file" p_src.yaml && continue ; \
+		echo "$$file"  >> python_world.lis ; \
+	    done
+
+clean::
+#	find * -type f -name \*~ -exec rm '{}' \;
+	rm -f *~ world.sed

roles/toxcore/vars/Ubuntu18.yml

@@ -0,0 +1,155 @@
+# -*- mode: yaml; indent-tabs-mode: nil; tab-width: 2; coding: utf-8-unix -*-
+
+---
+
+# , 'lxd' 'vagrant'
+TOXCORE_FEATURES: ['libvirt'] # 'docker', 'packer'
+
+toxcore_system_users:   "{{ base_system_users }}"
+toxcore_also_users: "{{ base_also_users }}"
+
+toxcore_libvirt_services:
+  - libvirtd
+
+toxcore_debs_inst:
+  - python-yaml
+  - gparted
+  - libvncserver
+  - ebtables
+  - dnsmasq-base
+  # vagrant-libvirt/README.md
+  - libxslt-dev
+  - libxml2-dev
+  - zlib1g-dev
+  - iptables
+  - zstd
+  - ebtables
+  - bridge-utils
+  - sdparm
+  - hdparm
+  - libxml2
+  - sdparm
+  - hdparm
+  - libxml2
+  - zerofree
+  - openssh-sftp-server
+  - sshfs
+  - xmlstarlet
+  - libtss2-tcti-swtpm0
+# toxcore
+  - libconfig-dev
+  - libgtest-dev
+  - ninja-build
+  - pkg-config
+  - zip
+  - grep
+  - file
+  - ca-certificates
+  - autotools-dev
+  - autoconf
+  - automake
+  - git
+  - bc
+  - wget
+  - rsync
+  - cmake
+  - make
+  - pkg-config
+  - libtool
+  - ssh
+  - gzip
+  - tar
+  - unzip
+  - libcurl4-gnutls-dev
+  - coreutils
+  - libavutil-dev
+  - libsodium-dev
+  - libffms2-4
+  - libgpac10
+  - libx264-dev
+  - x264
+  - libv4lconvert0
+  - libv4l-dev
+  - libv4l-dev
+  - libv4l2rds0
+  - v4l-conf
+  - v4l-utils
+  - libv4l-dev
+  - libtool
+  - autotools-dev
+  - automake
+  - checkinstall
+  - check
+  - git
+  - yasm
+  - libv4lconvert0
+  - libv4l-dev
+  - libopus-dev
+  - libvpx-dev
+  - pkg-config
+  - libx264-dev
+  - libavcodec-dev
+  - libavdevice-dev
+
+toxcore_lxd_debs_absent:
+  - lxd
+  - lxd-client
+  - lxcfs
+  - liblxc1
+  - liblxc-common
+
+toxcore_lxd_pkgs_inst:
+  - lxcfs
+#  - lxd
+#  - lxd-client
+  - zfsutils-linux
+# or snapd
+
+toxcore_qemu_debs_inst:
+  - qemu-kvm
+
+toxcore_libvirt_debs_inst:
+#?  - libvirt
+  - libvirt-bin
+  - libvirt-dev
+  - virt-manager
+  - libguestfs
+  - libguestfs-tools
+  # OUCH 4G!
+#  - libguestfs-appliance
+  - python-libvirt
+  - qemu
+
+toxcore_virtualbox_debs_inst:
+  - virtualbox
+  - virtualbox-modules
+  - virtualbox-ext-pack
+# virtualbox bridged networking
+
+toxcore_packer_debs_inst:
+  - jq
+
+toxcore_docker_debs_inst:
+  - aufs-tools
+  - cgroup-lite
+  - libsystemd-journal0
+  - linux-image-extra-virtual
+  - docker.io
+
+toxcore_ansible_debs_inst:
+  - jinja
+  - pycrypto
+  - paramiko
+  - python-ecdsa
+  - python-httplib2
+  - python-jinja2
+  - python-markupsafe
+  - python-paramiko
+
+toxcore_aws_debs_inst:
+# for aws
+  - boto
+  - bs4
+  - requests_ntlm
+  - awscli
+

src/ansible_gentooimgr/CONFIG.md

@@ -0,0 +1,17 @@
+GentooIMGR Configuration Specification
+======================================
+
+
+---------------
+Getting Started
+---------------
+
+If you want your own customized gentoo image, simply copy the gentooimgr/configs/base.json.example file:
+
+```sh
+cp gentooimgr/configs/base.json.example gentooimgr/myconfig.json
+# modify your config file. It needs to be saved in the directory directly above or within gentooimgr to be
+# included in the generated iso file and therefore mounted automatically; otherwise, copy it to the live image
+python -m gentooimgr -c gentooimgr/myconfig.json install
+```
+

src/ansible_gentooimgr/README.md

@@ -0,0 +1,168 @@
+GentooImgr: Gentoo Image Builder for Cloud and Turnkey ISO installers
+=====================================================================
+
+**This is a modified version of https://github.com/NucleaPeon/gentooimgr/
+where we've modified the code a little to do use Python logging. We can
+still use it for the build stage, but we think the install stage is better
+done using ansible, hence the libvirt_cloud playbook.
+
+The code is in overlay/Linux/usr/local/src/ansible_gentooimgr
+for reasons to do with how the ansible role transfers files.
+The code is being supported as an ansible module using
+library/ansible_gentooimgr.py which is a work in progress; the idea
+is to use it for the build and status actions, but handle the install
+tasks using ansible.
+**
+
+
+GentooImgr is a python script system to build cloud images based on Gentoo Linux.
+
+Huge thanks to https://github.com/travisghansen/gentoo-cloud-image-builder for providing a foundation to work from.
+
+
+**Features:**
+
+* This project enables easy access to building ``systemd`` or ``openrc`` -based images.
+* Performs automatic download AND verification of the linux iso, stage3 tarball and portage.
+* Caches the iso and stage3 .txt files for at most a day before redownloading and rechecking for new files
+* Sane and readable cli commands to build, run and test.
+* Step system to enable user to continue off at the same place if a step fails
+* No heavy packages like rust included ** TODO
+
+**rename to gentooimgr, upload to pip**
+
+Preface
+-------
+
+This project was created so I could spawn off Gentoo OS templates on my Proxmox server for various services while being more efficient than many other Linux OS's and avoiding systemd.
+
+This python module contains all the software needed to download resources, build the image, run the image in qemu, and allow for some extensibility. The built-in functionality includes a base standard gentoo image configuration as well as a cloud-init image that can be run. You can copy the .json config file and optionally a kernel .config file to configure your image to your liking and pass it in with ``--config``.
+
+This software is in beta so please report any issues or feature requests. You are **highly** encouraged to do so.
+
+Thanks!
+
+Roadmap
+-------
+
+* [X] Use gentooimgr to configure and Install a Base Gentoo OS using the least amount of configuration
+
+    - Successfully built a gentoo qcow2 image that can be run in qemu, but it requires using the --dist-kernel flag
+      as building from source still requires some work.
+
+* [X] Use gentooimgr to create a usable cloud image (requires --dist-kernel currently)
+* [ ] Use gentooimgr to create Gentoo installations on other non-amd64/non-native architectures (ex: ppc64)
+
+
+Prerequisites 
+-------------
+
+* [ ] QEMU
+* [ ] python3.11
+* [ ] Recommended 20GB of space
+* [ ] Internet Connection
+* [ ] virt-sparsify (for use with `gentooimgr shrink` action)
+
+
+Usage
+-----
+
+```sh
+git clone https://github.com/NucleaPeon/gentooimgr.git
+python -m gentooimgr build
+python -m gentooimgr run
+```
+
+Once qemu is running, mount the available gentooimgr iso and run the appropriate command:
+
+```sh
+mkdir -p /mnt/gi
+mount /dev/disk/by-label/gentooimgr /mnt/gi
+cd /mnt/gi
+python -m gentooimgr --config-cloud install
+```
+
+Configuring the cloud image will automatically bring in appropriate kernel configs (these are defined in ``gentooimgr/configs/cloud.config``).
+
+Then perform any additional procedures, such as shrinking the img from 10G to ~3-4G
+
+```sh
+python -m gentooimgr shrink gentoo.qcow2
+```
+
+**NOTE** Due to how ``gentooimgr`` dynamically finds the most recent portage/stage3 and iso files, if multiples exist in the same directory you may have to specify them using the appropriate flag (ie: ``--iso [path-to-iso]``). Older images can be used in this manner and eventually setting those values in the .json file should be recognized by gentooimgr so there will be no need to specify them on the command line.
+
+
+
+Extended Usage
+--------------
+
+GentooImgr is flexible in that it can be run on a live running system as well as on a livecd in qemu;
+
+It's possible to automate a new bare-metal Gentoo installation (with some further development) simply by running the ``install`` action or equivalent command to install and set up everything.
+
+Eventually, GentooImgr will be used to build gentoo turnkey OS images automatically.
+
+----
+
+One of the conveniences of this software is that it allows you to continue off where an error last occured.
+For example, if there's an issue where a package was renamed and the compile software step fails, you can edit
+the package list and rerun ``python -m gentooimgr cloud-cfg`` without any arguments and it will resume the compile
+step (albeit at the beginning of that step.)
+
+There are also commands that allow you to quickly enter the livecd chroot or run commands in the chroot:
+
+```sh
+python -m gentooimgr chroot
+```
+
+Mounts/binds are handled automatically when you chroot, but you will need to ``unchroot`` after to unmount the file systems:
+
+```sh
+python -m gentooimgr chroot
+# do stuff
+exit
+python -m gentooimgr unchroot
+```
+
+
+Adding Image to Proxmox
+-----------------------
+
+(Use the correct username and address to ssh/scp)
+
+```sh
+scp gentoo-[stamp].qcow2 root@proxmox:/tmp
+ssh root@proxmox
+# Set vmbr to your available bridge, it could be vmbr0 or vmbr1, etc.
+qm create 1000 --name gentoo-templ --memory 2048 --net0 virtio,bridge=vmbr0
+qm importdisk 1000 /tmp/gentoo-[stamp].qcow2 local -format qcow2
+qm set 1000 --scsihw virtio-scsi-pci --scsi0 /var/lib/vz/images/1000/vm-1000-disk-0.qcow2
+qm set 1000 --ide2 local:cloudinit --boot c --bootdisk scsi0 --serial0 socket --vga serial0
+qm resize 1000 scsi0 +20G
+qm set 1000 --ipconfig0 ip=dhcp
+qm set 1000 --sshkey ~/.ssh/id_rsa.pub
+qm template 1000
+
+(Creating a template from this image requires clicking the "Regenerate Image" button or equivalent cli command,
+after you set username and password)
+```
+
+Caveats
+--------
+
+* [X] Forced use of Rust in cloud images (cloud-init dependency)
+
+Unfortunately, using cloud-init brings in cryptography and oauthlib which pulls in rust. Any cloud images therefore are forced to use it, which is a large compilation target if rust-bin is not used. Some FOSS users, myself included, do not want rust installed on their systems and dislike how it is encroaching on so many FOSS areas.
+
+Work may be done to see if this can be avoided, but for now consider it a requirement.
+
+
+TODO
+----
+
+* [ ] Hash check portage downloads on ``build``
+* [ ] have a way to set the iso creation to either ignore things not set in the config file, or have options to include dirs, etc.
+* [ ] --skip-update-check : Do not even attempt to download new files in any capacity, simply use the latest ones found.
+        We could implement a way to find by glob and filter by modified by state and simply use the latest modified file
+        each and every time so we don't fail on multiple file detections

src/ansible_gentooimgr/gentooimgr/__init__.py

@@ -0,0 +1,5 @@
+import os
+HERE = os.path.abspath(os.path.dirname(__file__))
+import logging
+LOG = logging.getLogger('GI ')
+logging.basicConfig(level=logging.INFO) # oArgs.loglevel) #

src/ansible_gentooimgr/gentooimgr/__main__.py

@@ -0,0 +1,185 @@
+import os
+import sys
+import json
+import argparse
+import pathlib
+import copy
+import logging
+
+try:
+    import coloredlogs
+    os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red'
+except ImportError as e:
+    logging.log(logging.DEBUG, f"coloredlogs not available:  {e}")
+    coloredlogs = None
+
+from gentooimgr import LOG
+import gentooimgr.common
+import gentooimgr.config
+import gentooimgr.configs
+
+
+def main(args):
+    '''Gentoo Cloud Image Builder Utility'''
+    import gentooimgr.config
+    configjson = gentooimgr.config.determine_config(args)
+    prefix = args.temporary_dir
+
+    LOG.info(f'Gentoo Cloud Image Builder Utility {args.action}')
+    if args.action == "build":
+        import gentooimgr.builder
+        gentooimgr.builder.build(args, configjson)
+
+    elif args.action == "run":
+        import gentooimgr.run
+        gentooimgr.run.run(args, configjson)
+
+    elif args.action == "test":
+        # empty
+        import gentooimgr.test
+
+    elif args.action == "clean":
+        # empty
+        import gentooimgr.clean
+
+    elif args.action == "status":
+        import gentooimgr.status
+        gentooimgr.status.print_template(args, configjson)
+
+    elif args.action == "install":
+        import gentooimgr.install
+        gentooimgr.install.configure(args, configjson)
+
+    elif args.action == "command":
+        import gentooimgr.command
+        gentooimgr.command.command(configjson)
+
+    elif args.action == "chroot":
+        import gentooimgr.chroot
+        gentooimgr.chroot.chroot(path=args.mountpoint, shell="/bin/bash")
+
+    elif args.action == "unchroot":
+        import gentooimgr.chroot
+        gentooimgr.chroot.unchroot(path=args.mountpoint)
+
+    elif args.action == "shrink":
+        import gentooimgr.shrink
+        fname = gentooimgr.shrink.shrink(args, configjson, stamp=args.stamp)
+        print(f"Shrunken image at {fname}, {os.path.getsize(fname)}")
+
+    elif args.action == "kernel":
+        import gentooimgr.kernel
+        gentooimgr.kernel.build_kernel(args, configjson)
+    return 0
+
+if __name__ == "__main__":
+    """Gentoo Cloud Image Builder Utility"""
+    parser = argparse.ArgumentParser(prog="gentooimgr", description="Gentoo Image Builder Utility")
+    parser.add_argument("-c", "--config", nargs='?', type=pathlib.Path,
+                        help="Path to a custom conf file")
+    parser.add_argument("--config-cloud", action="store_const", const="cloud.json", dest="config",
+                        help="Use cloud init configuration")
+    parser.add_argument("--config-base", action="store_const", const="base.json", dest="config",
+                        help="Use a minimal base Gentoo configuration")
+
+    parser.add_argument("-t", "--temporary-dir", nargs='?', type=pathlib.Path,
+                        default=pathlib.Path(os.getcwd()), help="Path to temporary directory for downloading files")
+    parser.add_argument("-j", "--threads", type=int, default=gentooimgr.config.THREADS,
+                        help="Number of threads to use for building and emerging software")
+    parser.add_argument("-l", "--loglevel", type=int, default=logging.INFO,
+                        help="python logging level <= 50, INFO=20")
+    parser.add_argument("-y", "--days", type=int, default=7, # gentooimgr.config.DAYS
+                        help="Number of days before the files are redownloaded")
+    parser.add_argument("-d", "--download-dir", type=pathlib.Path, default=os.getcwd(),
+                        help="Path to the desired download directory (default: current)")
+    parser.add_argument("--openrc", dest="profile", action="store_const", const="openrc",
+                        help="Select OpenRC as the Gentoo Init System")
+    parser.add_argument("--systemd", dest="profile", action="store_const", const="systemd",
+                        help="Select SystemD as the Gentoo Init System")
+    parser.add_argument("-f", "--force", action="store_true",
+                        help="Let action occur at potential expense of data loss or errors (applies to clean and cloud-cfg)")
+    parser.add_argument("--format", default="qcow2", help="Image format to generate, default qcow2")
+    parser.add_argument("--portage", default=None, type=pathlib.Path, nargs='?',
+                            help="Extract the specified portage package onto the filesystem")
+    parser.add_argument("--stage3", default=None, type=pathlib.Path, nargs='?',
+                            help="Extract the specified stage3 package onto the filesystem")
+    parser.add_argument("--kernel-dir", default="/usr/src/linux",
+                               help="Where kernel is specified. By default uses the active linux kernel")
+    subparsers = parser.add_subparsers(help="gentooimgr actions", dest="action")
+    subparsers.required = True
+
+    # Build action
+    parser_build = subparsers.add_parser('build', help="Download and verify all the downloaded components for cloud image")
+    parser_build.add_argument("image", default=gentooimgr.config.GENTOO_IMG_NAME, type=str, nargs='?',
+                              help="Specify the exact image (date) you want to build; ex: 20231112T170154Z. Defaults to downloading the latest image. If image exists, will automatically use that one instead of checking online.")
+    parser_build.add_argument("--size", default="12G", help="Size of image to build")
+    parser_build.add_argument("--no-verify", dest="verify", action="store_false", help="Do not verify downloaded iso")
+    parser_build.add_argument("--verify", dest="verify", action="store_true", default=True,
+                              help="Verify downloaded iso")
+    parser_build.add_argument("--redownload", action="store_true", help="Overwrite downloaded files")
+    parser_run = subparsers.add_parser('run', help="Run a Gentoo Image in QEMU to process it into a cloud image")
+
+    parser_run.add_argument("--iso", default=None, type=pathlib.Path, nargs='?',
+                            help="Mount the specified iso in qemu, should be reserved for live cd images")
+    parser_run.add_argument("image", default=gentooimgr.config.GENTOO_IMG_NAME,
+                            type=pathlib.Path, nargs="?",
+                            help="Run the specified image in qemu")
+    parser_run.add_argument("-m", "--mounts", nargs='+', default=[],
+                            help="Path to iso files to mount into the running qemu instance")
+
+    parser_test = subparsers.add_parser('test', help="Test whether image is a legitamite cloud configured image")
+
+    parser_clean = subparsers.add_parser('clean', help="Remove all downloaded files")
+    # --force also applies to clean action
+
+    parser_status = subparsers.add_parser('status', help="Review information, downloaded images and configurations")
+
+    parser_install = subparsers.add_parser("install", help="Install Gentoo on a qemu guest. Defaults to "
+                                           "--config-base with --kernel-dist if the respective --config or --kernel options are not provided.")
+    parser_install.add_argument("--kernel-dist", action="store_true",
+                              help="Use a distribution kernel in the installation. Overrides all other kernel options.")
+    parser_install.add_argument("--kernel-virtio", action="store_true", help="Include virtio support in non-dist kernels")
+    parser_install.add_argument("--kernel-g5", action="store_true", help="Include all kernel config options for PowerMac G5 compatibility")
+
+    parser_chroot = subparsers.add_parser("chroot", help="Bind mounts and enter chroot with shell on guest. Unmounts binds on shell exit")
+    parser_chroot.add_argument("mountpoint", nargs='?', default=gentooimgr.config.GENTOO_MOUNT,
+                               help="Point to mount and run the chroot and shell")
+
+    parser_unchroot = subparsers.add_parser("unchroot", help="Unmounts chroot filesystems")
+    parser_unchroot.add_argument("mountpoint", nargs='?', default=gentooimgr.config.GENTOO_MOUNT,
+                               help="Point to mount and run the chroot and shell")
+
+    parser_cmd = subparsers.add_parser('command', help="Handle bind mounts and run command(s) in guest chroot, then unmount binds")
+    parser_cmd.add_argument("cmds", nargs='*',
+                            help="Commands to run (quote each command if more than one word, ie: \"grep 'foo'\" \"echo foo\")")
+
+    parser_shrink = subparsers.add_parser('shrink', help="Take a finalized Gentoo image and rearrange it for smaller size")
+    parser_shrink.add_argument("img", type=pathlib.Path, help="Image to shrink")
+    parser_shrink.add_argument("--stamp", nargs='?', default=None,
+                               help="By default a timestamp will be added to the image name, otherwise provide "
+                               "a hardcoded string to add to the image name. Result: gentoo-[stamp].img")
+
+    parser_kernel = subparsers.add_parser('kernel', help="Build the kernel based on configuration and optional --kernel-dist flag")
+
+
+    args = parser.parse_args()
+
+    assert args.loglevel < 59
+    if coloredlogs:
+        # https://pypi.org/project/coloredlogs/
+        coloredlogs.install(level=args.loglevel,
+                        logger=LOG,
+                        # %(asctime)s,%(msecs)03d %(hostname)s [%(process)d]
+                        fmt='%(name)s %(levelname)s %(message)s'
+                        )
+    else:
+        logging.basicConfig(level=args.loglevel) #  logging.INFO
+    logging.basicConfig(level=args.loglevel)
+
+    isos = gentooimgr.common.find_iso(args.download_dir)
+    if args.action == "run" and args.iso is None and len(isos) > 1:
+        LOG.error(f"Error: multiple iso files were found in {args.download_dir}, please specify one using `--iso [iso]`")
+        sys.exit(1)
+
+
+    main(args)

src/ansible_gentooimgr/gentooimgr/builder.py

@@ -0,0 +1,26 @@
+import os
+import argparse
+
+from gentooimgr import LOG
+import gentooimgr.config as config
+import gentooimgr.download as download
+import gentooimgr.qemu as qemu
+import gentooimgr.common
+import requests
+
+def build(args: argparse.Namespace, config: dict) -> None:
+    LOG.info(": build")
+
+    iso = config.get("iso") or download.download(args)
+    stage3 = config.get("stage3") or download.download_stage3(args)
+    portage = config.get("portage") or download.download_portage(args)
+    filename = f"{args.image}.{args.format}"
+    image = qemu.create_image(args, config)
+    if not os.path.exists(image):
+        raise Exception(f"Image {image} does not exist")
+
+    is_default = os.path.basename(image) == filename
+    LOG.info(image)
+    LOG.info(f"Image {image} build successfully.\nRun `python -m gentooimgr run{' ' + image if not is_default else ''} --iso {iso}`")
+
+    return image

src/ansible_gentooimgr/gentooimgr/chroot.py

@@ -0,0 +1,54 @@
+import os
+import sys
+from subprocess import Popen, PIPE
+import gentooimgr.config
+
+def bind(mount=gentooimgr.config.GENTOO_MOUNT, verbose=True):
+    mounts = [
+        ["mount", "--types", "proc", "/proc", os.path.join(mount, "proc")],
+        ["mount", "--rbind", "/sys", os.path.join(mount, "sys")],
+        ["mount", "--make-rslave", os.path.join(mount, "sys")],
+        ["mount", "--rbind", "/dev", os.path.join(mount, "dev")],
+        ["mount", "--make-rslave", os.path.join(mount, "dev")],
+        ["mount", "--bind", "/run", os.path.join(mount, "run")],
+        ["mount", "--make-slave", os.path.join(mount, "run")],
+    ]
+    for mcmd in mounts:
+        if verbose:
+            print(f"\t:: {' '.join(mcmd)}")
+        proc = Popen(mcmd, stdout=PIPE, stderr=PIPE)
+        stdout, stderr = proc.communicate()
+        if proc.returncode != 0:
+            sys.stderr.write(f"{stderr}\n")
+            sys.exit(proc.returncode)
+
+def unbind(mount=gentooimgr.config.GENTOO_MOUNT, verbose=True):
+    os.chdir("/")
+    if not os.path.exists(mount):
+        sys.stderr.write(f"Mountpoint {mount} does not exist\n")
+        return
+
+    unmounts = [
+        ["umount", os.path.join(mount, 'dev', 'shm')],
+        ["umount", os.path.join(mount, 'dev', 'pts')],
+        ["umount", "-l", os.path.join(mount, 'dev')],
+        ["umount", "-R", mount]
+    ]
+    for uncmd in unmounts:
+        if verbose:
+            print(f"\t:: {' '.join(uncmd)}")
+        proc = Popen(uncmd)
+        stdout, stderr = proc.communicate()
+        if proc.returncode != 0:
+            sys.stderr.write(f"{stderr}\n")
+            continue
+
+def chroot(path=gentooimgr.config.GENTOO_MOUNT, shell="/bin/bash"):
+    bind(mount=path)
+    os.chroot(path)
+    os.chdir(os.sep)
+    os.system(shell)
+    unchroot(path=path)  # May fail if we do this automatically
+
+def unchroot(path=gentooimgr.config.GENTOO_MOUNT):
+    unbind(mount=path)

src/ansible_gentooimgr/gentooimgr/command.py

@@ -0,0 +1,14 @@
+import sys
+from subprocess import Popen, PIPE
+
+import gentooimgr.chroot
+
+def command(config, *args):
+    gentooimgr.chroot.bind()
+    for a in args:
+        proc = Popen(a, shell=True, stdout=PIPE, stderr=PIPE)
+        stdout, stderr = proc.communicate()
+        if proc.returncode != 0:
+            sys.stderr.write(f"{stderr}\n")
+            break
+    gentooimgr.chroot.unbind()

src/ansible_gentooimgr/gentooimgr/common.py

@@ -0,0 +1,124 @@
+import os
+import sys
+import time
+import copy
+import json
+from subprocess import Popen, PIPE
+
+import gentooimgr.config
+from gentooimgr import LOG
+
+def older_than_a_day(fullpath):
+    if not os.path.exists(fullpath):
+        return True  # Don't fail on missing files
+    filetime = os.path.getmtime(fullpath)
+    return time.time() - filetime > (gentooimgr.config.DAY_IN_SECONDS *
+                                     gentooimgr.config.DAYS)
+
+def find_iso(download_dir):
+    LOG.info(f"Looking for iso in {download_dir}")
+    name = None
+    ext = None
+    found = []
+    for f in os.listdir(download_dir):
+        name, ext = os.path.splitext(f)
+        if ext == ".iso":
+            found.append(os.path.join(download_dir, f))
+
+    return found
+
+def make_iso_from_dir(mydir):
+    """ Generates an iso with gentooimgr inside it for use inside a live cd guest
+    :Returns:
+        path to iso that was created or NoneType if mydir is not found
+    """
+    if not os.path.exists(mydir):
+        LOG.warn(f"\t:: dir not found {mydir}")
+        return
+
+    LOG.info(f"\t:: Making ISO with dir of {mydir}")
+    path = os.path.join(mydir, "..", "cloudgen.iso")
+    proc = Popen(["mkisofs",
+        "--input-charset", "utf-8",
+        "-J",
+        "-r",
+        "-V", "gentooimgr",
+        "-m", "*.img",
+        "-m", "*.iso",
+        "-o", path,
+        mydir
+        ], stdout=PIPE, stderr=PIPE)
+    proc.communicate()
+
+    return path
+
+def portage_from_dir(download_dir, filename=None):
+    """Find portage file from directory. Will do a check in os.listdir() for portage*.tar.bz2.
+
+    If a filename is provided, this function either returns that filename assuming it exists in d,
+    or return None. If filename is None, this looks through all entries for portage files and if
+    only one exists, returns it, otherwise None.
+    """
+    assert download_dir, f"empty {download_dir} for  for portage"
+    LOG.info(f"Looking for portage in {download_dir}")
+    found = []
+    for f in os.listdir(download_dir):
+        if filename is not None:
+            if filename == f:
+                found.append(f)
+        elif f.startswith("portage") and f.endswith(".tar.xz"):
+            found.append(f)
+
+    if len(found) > 1:
+        LOG.error("\tEE: More than one portage file exists, please specify the exact portage file with --portage [file]  or remove all others\n")
+        LOG.error(''.join([f"\t{f}\n" for f in found]))
+        LOG.error(f"in {download_dir}\n")
+        sys.exit(1)
+
+    return found[0] if found else None
+
+
+def stage3_from_dir(d, filename=None):
+    """Find stage3 file from directory. Will do a check in os.listdir() for stage3*.tar.xz.
+
+    If a filename is provided, this function either returns that filename assuming it exists in d,
+    or return None. If filename is None, this looks through all entries for stage3 files and if
+    only one exists, returns it, otherwise None.
+    """
+    found = []
+    for f in os.listdir(d):
+        if filename is not None:
+            if filename == f:
+                found.append(f)
+        elif f.startswith("stage3") and f.endswith(".tar.xz"):
+            found.append(f)
+
+    if len(found) > 1:
+        LOG.error("More than one stage3 file exists, please specify the exact stage3 file or remove all others\n")
+        LOG.error(''.join([f"\t{f}\n" for f in found]))
+        LOG.error(f"in {d}\n")
+        return None
+
+    return found[0] if found else None
+
+
+def get_image_name(args, config):
+    image = config.get("imagename", "gentoo."+args.format)
+    if image is None:
+        image = "gentoo."+args.format
+    return image
+
+#
+# def load_config(args):
+#     cfg = generatecfg(args)
+#     if args.config:
+#         override = generatecfg(args, config=args.config)
+#         cfg.update(cfgoverride)
+#
+#     if cfg.get("portage") is None:
+#         cfg['portage'] = portage_from_dir(args.download_dir, filename=args.portage or cfg.get("portage"))
+#     if cfg.get("stage3") is None:
+#         cfg['stage3'] = stage3_from_dir(args.download_dir, filename=args.stage3 or cfg.get("stage3"))
+#
+#     return cfg
+#

src/ansible_gentooimgr/gentooimgr/config.py

@@ -0,0 +1,132 @@
+import os
+import json
+import sys
+import argparse
+
+from gentooimgr import LOG
+import gentooimgr.configs
+import multiprocessing
+
+# A day in seconds:
+DAY_IN_SECONDS = 60*60*24
+# days until the iso is old
+DAYS = 1
+# Define threads to compile packages with
+THREADS = multiprocessing.cpu_count()
+# URL to latest image text file, defaults to amd64. This is parsed to find latest iso to download
+ARCHITECTURE = "amd64"
+GENTOO_BASE_ISO_URL = f"https://distfiles.gentoo.org/releases/{ARCHITECTURE}/autobuilds/current-install-{ARCHITECTURE}-minimal/"
+GENTOO_BASE_STAGE_OPENRC_URL = f"https://distfiles.gentoo.org/releases/{ARCHITECTURE}/autobuilds/current-stage3-{ARCHITECTURE}-openrc/"
+GENTOO_BASE_STAGE_SYSTEMD_URL = f"https://distfiles.gentoo.org/releases/{ARCHITECTURE}/autobuilds/current-stage3-{ARCHITECTURE}-systemd/"
+GENTOO_LATEST_ISO_FILE = f"latest-install-{ARCHITECTURE}-minimal.txt"
+GENTOO_LATEST_STAGE_OPENRC_FILE = f"latest-stage3-{ARCHITECTURE}-openrc.txt"
+GENTOO_LATEST_STAGE_SYSTEMD_FILE = f"latest-stage3-{ARCHITECTURE}-systemd.txt"
+GENTOO_PORTAGE_FILE = "http://distfiles.gentoo.org/snapshots/portage-latest.tar.xz"  # No architecture, no txt files to determine latest.
+
+GENTOO_MOUNT = "/mnt/gentoo"
+GENTOO_IMG_NAME = "gentoo.qcow2"
+
+GENTOO_FILE_HASH_RE = r"^Hash\: ([\w]*)$"
+GENTOO_FILE_ISO_RE = r"^(install-[\w\-_\.]*.iso) ([\d]*)"
+GENTOO_FILE_ISO_HASH_RE = r"^([\w]*)  (install-[\w\-_\.]*.iso)$"
+GENTOO_FILE_STAGE3_RE = r"^(stage3-[\w\-_\.]*.tar.*) ([\d]*)"
+GENTOO_FILE_STAGE3_HASH_RE = r"^([\w]*)  (stage3-[\w\-_\.]*.tar.*)$"
+# TODO: Repo regex to replace attributes, use function to do so as find key will change.
+
+def replace_repos_conf(key, value):
+    pass
+
+CLOUD_MODULES = [
+    "iscsi_tcp"
+]
+
+def load_config(path):
+    assert path, "load config called with nothing"
+    if os.path.exists(path):
+        with open(path, 'r') as f:
+            try:
+                return json.loads(f.read())
+            except Exception as e:
+                LOG.error(f"ERROR loading {path}")
+                raise
+    return {}
+
+def load_default_config(config_name):
+    """This is called when a --config option is set. --kernel options update the resulting config, whether
+    it be 'base' or other.
+    If user is supplying their own configuration, this is not called.
+    """
+    name, ext = os.path.splitext(config_name)
+    if not name in gentooimgr.configs.KNOWN_CONFIGS:
+        return {}
+
+    json_file = os.path.join(gentooimgr.configs.CONFIG_DIR, config_name)
+    ret = {}
+    with open(json_file, 'r') as f:
+        try:
+            ret = json.loads(f.read())
+        except Exception as e:
+            LOG.error(f"loading {json_file} {e}")
+    return ret
+
+def inherit_config(config: dict) -> dict:
+    """Returns the json file that the inherit key specifies; will recursively update if inherit values are set.
+    """
+    configuration = load_default_config(config.get("inherit"))
+    if not configuration:
+        configuration = load_config(config.get("inherit"))
+
+    if not configuration:
+        sys.stderr.write(f"\tWW: Warning: Inherited configuration {config.get('inherit')} is not found.\n")
+        return {}
+
+    if configuration.get("inherit"):
+        configuration.update(inherit_config(configuration.get("inherit")))
+
+    return configuration
+
+def determine_config(args: argparse.Namespace) -> dict:
+    """Check argparser options and return the most valid configuration
+
+    The "package" key/value object overrides everything that is set, it does not update() them.
+    If you override "base" package set, it's exactly what you set. It makes more sense to do it this way.
+    For example, if you have a dist kernel config, you don't want the base.json to update and include all
+    non-dist kernel options as it would add a lot of used space for unused functionality.
+
+    The package set is only overridden in the top level json configuration file though;
+    If you have multiple inherits, those package sets will be combined before the parent package set overrides
+    with the keys that are set.
+
+    If you have base.json and base2.json that contain multiple layers of "base" packages, ie: base: ['foo'] and base2: ['bar']
+    then you will have in yours.json: packages { base: ['foo', 'bar'] } and unless you set "base", that is what you'll get.
+
+
+    If you check `status` action, it will flatten all configurations into one, so the "inherit" key will always be null.
+
+    :Returns:
+        - configuration from json to dict
+    """
+
+    # Check custom configuration
+    configuration = load_default_config(args.config or 'base.json')
+    if not configuration and args.config:
+        configuration = load_config(args.config)
+    if not configuration:
+        LOG.error(f"\tWW: Warning: Configuration {args.config} is empty\n")
+    else:
+        if configuration.get("inherit"):
+            # newpkgs = configuration.get("packages", {})
+            inherited = inherit_config(configuration)
+            new_packages = configuration.get("packages", {})
+            old_packages = inherited.get("packages", {})
+            inherited.update(configuration)
+            # Set back old package dict and then update only what is set in new:
+            inherited['packages'] = old_packages
+            for key, pkgs in new_packages.items():
+                if pkgs:
+                    inherited['packages'][key] = pkgs
+
+            return inherited
+
+    return configuration
+

src/ansible_gentooimgr/gentooimgr/configs/__init__.py

@@ -0,0 +1,153 @@
+import os
+
+CONFIG_DIR = os.path.abspath(os.path.dirname(__file__))
+
+__all__ = ["CONFIG_DIR", "CLOUD_YAML", "HOST_TMPL", "HOSTNAME", "KNOWN_CONFIGS"]
+
+# List of configurations that end in '.json' within the configs/ directory
+KNOWN_CONFIGS = [
+    "base",
+    "cloud"
+]
+
+# Currently we handle the writing of additional files by having data defined here and checking options.
+# this isn't ideal. TODO: Make this better.
+CLOUD_YAML = """
+# The top level settings are used as module
+# and system configuration.
+
+# A set of users which may be applied and/or used by various modules
+# when a 'default' entry is found it will reference the 'default_user'
+# from the distro configuration specified below
+users:
+   - default
+
+# If this is set, 'root' will not be able to ssh in and they
+# will get a message to login instead as the above $user (ubuntu)
+disable_root: true
+ssh_pwauth:   false
+
+# This will cause the set+update hostname module to not operate (if true)
+preserve_hostname: false
+
+# this may be helpful in certain scenarios
+# resize_rootfs_tmp: /dev
+
+syslog_fix_perms: root:root
+
+ssh_deletekeys: false
+ssh_genkeytypes: [rsa, dsa]
+
+# This can be 'template'
+# which would look for /etc/cloud/templates/hosts.gentoo.tmpl
+# or 'localhost'
+# or False / commented out to disable altogether
+manage_etc_hosts: template
+
+# Example datasource config
+# datasource:
+#    Ec2:
+#      metadata_urls: [ 'blah.com' ]
+#      timeout: 5 # (defaults to 50 seconds)
+#      max_wait: 10 # (defaults to 120 seconds)
+
+# The modules that run in the 'init' stage
+cloud_init_modules:
+ - seed_random
+ - bootcmd
+ - write-files
+ - growpart
+ - resizefs
+ - set_hostname
+ - update_hostname
+ - update_etc_hosts
+ - ca-certs
+ - users-groups
+ - ssh
+
+# The modules that run in the 'config' stage
+cloud_config_modules:
+# Emit the cloud config ready event
+# this can be used by upstart jobs for 'start on cloud-config'.
+ - disk_setup
+ - mounts
+ - ssh-import-id
+ - set-passwords
+ - package-update-upgrade-install
+ - timezone
+ - puppet
+ - chef
+ - salt-minion
+ - mcollective
+ - disable-ec2-metadata
+ - runcmd
+
+# The modules that run in the 'final' stage
+cloud_final_modules:
+ - scripts-vendor
+ - scripts-per-once
+ - scripts-per-boot
+ - scripts-per-instance
+ - scripts-user
+ - ssh-authkey-fingerprints
+ - keys-to-console
+ - phone-home
+ - final-message
+ - power-state-change
+
+# System and/or distro specific settings
+# (not accessible to handlers/transforms)
+system_info:
+   # This will affect which distro class gets used
+   distro: gentoo
+   # Default user name + that default users groups (if added/used)
+   default_user:
+     name: gentoo
+     lock_passwd: True
+     gecos: Gentoo
+     groups: [users, wheel]
+     primary_group: users
+     no-user-group: true
+     sudo: ["ALL=(ALL) NOPASSWD:ALL"]
+     shell: /bin/bash
+   # Other config here will be given to the distro class and/or path classes
+   paths:
+      cloud_dir: /var/lib/cloud/
+      templates_dir: /etc/cloud/templates/
+"""
+
+HOST_TMPL = """
+## template:jinja
+{#
+This file /etc/cloud/templates/hosts.gentoo.tmpl is only utilized
+if enabled in cloud-config.  Specifically, in order to enable it
+you need to add the following to config:
+  manage_etc_hosts: template
+-#}
+# Your system has configured 'manage_etc_hosts' as 'template'.
+# As a result, if you wish for changes to this file to persist
+# then you will need to either
+# a.) make changes to the master file in /etc/cloud/templates/hosts.gentoo.tmpl
+# b.) change or remove the value of 'manage_etc_hosts' in
+#     /etc/cloud/cloud.cfg or cloud-config from user-data
+#
+# The following lines are desirable for IPv4 capable hosts
+127.0.0.1 {{fqdn}} {{hostname}}
+127.0.0.1 localhost.localdomain localhost
+127.0.0.1 localhost4.localdomain4 localhost4
+
+# The following lines are desirable for IPv6 capable hosts
+::1 {{fqdn}} {{hostname}}
+::1 localhost.localdomain localhost
+::1 localhost6.localdomain6 localhost6
+"""
+
+HOSTNAME = """
+# Set to the hostname of this machine
+if [ -f /etc/hostname ];then
+    hostname=$(cat /etc/hostname 2> /dev/null | cut -d"." -f1 2> /dev/null)
+else
+    hostname="localhost"
+fi
+"""
+

src/ansible_gentooimgr/gentooimgr/configs/base.json

@@ -0,0 +1,82 @@
+{
+    "inherit": null,
+    "imgsize": "20G",
+    "memory": 4096,
+    "mountpoint": "/mnt/gentoo",
+    "imagename": null,
+    "initsys": "openrc",
+    "licensefiles": {
+        "kernel": ["sys-kernel/linux-firmware linux-fw-redistributable"]
+    },
+    "kernel": {
+        "path": "/etc/kernels/config.d/gentooimgr-base.config"
+    },
+    "repos": {
+        "/etc/portage/repos.conf/gentoo.conf": {
+            "gentoo": {
+                "sync-uri": "rsync://192.168.254.20/gentoo-portage"
+            }
+        }
+    },
+    "packages": {
+        "base": [
+            "acpid",
+            "dmidecode",
+            "syslog-ng",
+            "cronie",
+            "dhcpcd",
+            "mlocate",
+            "xfsprogs",
+            "dosfstools",
+            "sudo",
+            "postfix",
+            "parted",
+            "portage-utils",
+            "gentoo-bashcomp",
+            "tmux",
+            "app-misc/screen",
+            "dev-vcs/git",
+            "net-misc/curl",
+            "usbutils",
+            "pciutils",
+            "logrotate",
+            "gptfdisk",
+            "sys-block/gpart",
+            "net-misc/ntp",
+            "net-fs/nfs-utils",
+            "app-emulation/qemu-guest-agent",
+            "linux-firmware"
+        ],
+        "additional": ["app-editors/vim"],
+        "oneshots": [
+            "portage"
+        ],
+        "singles": [
+            "app-portage/eix",
+            "dev-util/cmake"
+        ],
+        "keepgoing": [
+            "openssh"
+        ],
+        "bootloader": [
+            "grub:2"
+        ],
+        "kernel": [
+            "sys-kernel/genkernel",
+            "gentoo-sources",
+            "gentoolkit"
+        ]
+    },
+    "services": {
+        "syslog-ng": "default",
+        "cronie": "default",
+        "acpid": "default",
+        "ntp": "default",
+        "qemu-guest-agent": "default"
+    },
+    "iso": null,
+    "portage": null,
+    "stage3": null,
+    "disk": "/dev/sda",
+    "partition": 1
+}

src/ansible_gentooimgr/gentooimgr/configs/base.json.example

@@ -0,0 +1,77 @@
+{
+    "inherit": null,
+    "imgsize": "12G",
+    "memory": 4096,
+    "mountpoint": "/mnt/gentoo",
+    "imagename": null,
+    "initsys": "openrc",
+    "licensefiles": {
+        "kernel": ["sys-kernel/linux-firmware linux-fw-redistributable"]
+    },
+    "repos": {
+        "/etc/portage/repos.conf/gentoo.conf": {
+            "sync-uri": "rsync://192.168.254.20/gentoo-portage"
+        }
+    },
+    "packages": {
+        "base": [
+            "acpid",
+            "dmidecode",
+            "syslog-ng",
+            "cronie",
+            "dhcpcd",
+            "mlocate",
+            "xfsprogs",
+            "dosfstools",
+            "sudo",
+            "postfix",
+            "app-editors/vim",
+            "parted",
+            "portage-utils",
+            "bash-completion",
+            "gentoo-bashcomp",
+            "tmux",
+            "app-misc/screen",
+            "dev-vcs/git",
+            "net-misc/curl",
+            "usbutils",
+            "pciutils",
+            "logrotate",
+            "gptfdisk",
+            "sys-block/gpart",
+            "net-misc/ntp",
+            "net-fs/nfs-utils",
+            "linux-firmware"
+        ],
+        "additional": [],
+        "oneshots": [
+            "portage"
+        ],
+        "singles": [
+            "app-portage/eix",
+            "dev-util/cmake"
+        ],
+        "keepgoing": [
+            "openssh"
+        ],
+        "bootloader": [
+            "grub:2"
+        ],
+        "kernel": [
+            "sys-kernel/genkernel",
+            "gentoo-sources",
+            "gentoolkit"
+        ]
+    },
+    "services": {
+        "syslog-ng": "default",
+        "cronie": "default",
+        "acpid": "default",
+        "ntp": "default"
+    },
+    "iso": null,
+    "portage": null,
+    "stage3": null,
+    "disk": null,
+    "partition": 1
+}

src/ansible_gentooimgr/gentooimgr/configs/cloud.json

@@ -0,0 +1,17 @@
+{
+    "inherit": "base.json",
+    "packages": {
+        "additional": [
+            "app-emulation/cloud-init",
+            "sys-block/open-iscsi",
+            "app-editors/mg",
+            "net-analyzer/openbsd-netcat"
+        ]
+    },
+    "disk": "/dev/vda",
+    "kernel": {
+        "path": "/etc/kernels/config.d/gentooimgr-cloud.config",
+        "config": "cloud.config"
+    },
+    "partition": 1
+}

src/ansible_gentooimgr/gentooimgr/download.py

@@ -0,0 +1,195 @@
+"""Module to handle downloading and verification of Gentoo images
+
+To ensure accuracy, we re-download every .txt file if it's older than one day.
+We assume that people building a cloud configured image want what is most up to date.
+If you have a specific image you want built over and over regardless, create a config
+file and load it in using -c/--config that points GENTOO_* values to the files you want.
+"""
+
+import os
+import re
+import sys
+from datetime import date
+import hashlib
+import progressbar
+from urllib.request import urlretrieve
+import tempfile
+
+from gentooimgr import LOG
+import gentooimgr.config as config
+from gentooimgr.common import older_than_a_day
+
+hashpattern =    re.compile(config.GENTOO_FILE_HASH_RE, re.MULTILINE)
+isopattern =     re.compile(config.GENTOO_FILE_ISO_RE, re.MULTILINE)
+isohashpattern = re.compile(config.GENTOO_FILE_ISO_HASH_RE, re.MULTILINE)
+stage3pattern  = re.compile(config.GENTOO_FILE_STAGE3_RE, re.MULTILINE)
+stage3hashpattern = re.compile(config.GENTOO_FILE_STAGE3_HASH_RE, re.MULTILINE)
+
+class DownloadProgressBar():
+    def __init__(self):
+        self.progress = None
+
+    def __call__(self, block_num, block_size, total_size):
+        if not self.progress:
+            self.progress = progressbar.ProgressBar(maxval=total_size)
+            self.progress.start()
+
+        downloaded = block_num * block_size
+        if downloaded < total_size:
+            self.progress.update(downloaded)
+        else:
+            self.progress.finish()
+
+def parse_latest_iso_text(fullpath) -> tuple:
+    """Returns a tuple of (hash type, iso name, iso bytes)"""
+    with open(fullpath) as f:
+        content = f.read()
+        m_hash = hashpattern.search(content)
+        m_iso = isopattern.search(content)
+        return (m_hash.group(1) if not m_hash is None else None,
+                m_iso.group(1) if not m_iso is None else None,
+                m_iso.group(2) if not m_iso is None else None,)
+
+def parse_latest_stage3_text(fullpath) -> tuple:
+    """Returns a tuple of (hash type, iso name, iso bytes)
+    """
+    with open(fullpath) as f:
+        content = f.read()
+        m_hash = hashpattern.search(content)
+        m_stage3 = stage3pattern.search(content)
+        return (m_hash.group(1) if not m_hash is None else None,
+                m_stage3.group(1) if not m_stage3 is None else None,
+                m_stage3.group(2) if not m_stage3 is None else None,)
+
+def verify(args, _type: str, baseurl: str, hashpattern, filename: str) -> bool:
+    """Downloads hash file and run a hash check on the file
+    :Parameters:
+        - args: Namespace of parsed arguments
+        - _type: str hash type
+        - baseurl: (remote) folder where hashsum file is contained
+        - hashpattern:
+        - filename: str name of file to check (used to download corresponding hash file)
+
+            A install-amd64-minimal-2023111iso2T170154Z.iso file will have a
+              install-amd64-minimal-20231112T170154Z.iso.sha256 for example.
+
+    :Returns:
+        Whether iso was verified using the specified hash
+
+    """
+    thefile = os.path.join(args.download_dir, filename)
+    LOG.info(f"verifying hash of {thefile}")
+    digest = hashlib.file_digest(open(thefile, 'rb'), _type.lower())
+    filename = filename+f".{_type.lower()}"  # Update to hash file
+    hashfile = os.path.join(baseurl, filename)
+    fullpath = os.path.join(args.download_dir, os.path.basename(hashfile))
+    if not os.path.exists(fullpath) or args.redownload or older_than_a_day(fullpath):
+        LOG.info(f"Downloading {filename}")
+        urlretrieve(hashfile, fullpath, DownloadProgressBar())
+
+    hd = digest.hexdigest()
+    with open(fullpath, 'r') as f:
+        content = f.read()
+        m_hash = hashpattern.search(content)
+        _hash = m_hash.group(1)
+        assert hd == _hash, f"Hash mismatch {hd} != {_hash}"
+
+def download_stage3(args, url=None) -> str:
+    if url is None:
+        if args.profile == "systemd":
+            url = os.path.join(config.GENTOO_BASE_STAGE_SYSTEMD_URL, config.GENTOO_LATEST_STAGE_SYSTEMD_FILE)
+
+        else:
+            url = os.path.join(config.GENTOO_BASE_STAGE_OPENRC_URL, config.GENTOO_LATEST_STAGE_OPENRC_FILE)
+
+    filename = os.path.basename(url)
+    fullpath = os.path.join(args.download_dir, filename)
+    if not os.path.exists(fullpath) or args.redownload or older_than_a_day(fullpath):
+        print(f"Downloading {filename}")
+        urlretrieve(url, fullpath, DownloadProgressBar())
+
+    hashtype, latest, size = parse_latest_stage3_text(fullpath)
+    size = int(size)
+
+    filename = latest
+    fullpath = os.path.join(args.download_dir, filename)
+    if not os.path.exists(fullpath) or args.redownload:
+        LOG.info(f"Downloading {filename}")
+        url = os.path.join(
+                config.GENTOO_BASE_STAGE_SYSTEMD_URL if args.profile == "systemd" else \
+                config.GENTOO_BASE_STAGE_OPENRC_URL,
+                filename)
+        urlretrieve(url, fullpath, DownloadProgressBar())
+
+    # Verify byte size
+    stage3size = os.path.getsize(fullpath)
+    assert size == stage3size, f"Stage 3 size {size} does not match expected value {stage3size}."
+    verify(args, hashtype, config.GENTOO_BASE_STAGE_SYSTEMD_URL if args.profile == "systemd" else \
+           config.GENTOO_BASE_STAGE_OPENRC_URL,  stage3hashpattern, filename)
+    return fullpath
+
+
+def download_portage(args, url=None) -> str:
+    """Handle downloading of portage system for installation into cloud image
+
+    We always download the latest portage package and rename it to today's date.
+    If using today's date to grab portage, sometimes depending on timezone, the
+    package won't be available. If always using latest, worst case scenario is you
+    have a portage package a day late.
+
+
+    """
+    if url is None:
+        url = config.GENTOO_PORTAGE_FILE
+
+    base = os.path.basename(url)  # Uses 'latest' filename
+    today = date.today()
+    # Write latest to today's date so we don't constantly redownload, but
+    filename = base.replace("latest", "%d%d%d" % (today.year, today.month, today.day))
+    fullpath = os.path.join(args.download_dir, filename)
+    # Portage is always "latest" in this case, so definitely check if older than a day and redownload.
+    if not os.path.exists(fullpath) or args.redownload or older_than_a_day(fullpath):
+        LOG.info(f"Downloading {filename} ({base})")
+        urlretrieve(url, fullpath, DownloadProgressBar())
+
+    return fullpath
+
+
+def download(args, url=None) -> str:
+    """Download txt file with iso name and hash type
+    :Parameters:
+        - args: Namespace with parsed arguments
+        - url: str or None. If None, will generate a url to the latest minimal install iso
+
+    :Returns:
+        Full path to the downloaded iso file
+
+    Will cause program to exit if iso byte size fails to match expected value.
+    """
+    if url is None:
+        url = os.path.join(config.GENTOO_BASE_ISO_URL, config.GENTOO_LATEST_ISO_FILE)
+
+    # Download the latest txt file
+    filename = os.path.basename(url)
+    fullpath = os.path.join(args.download_dir, filename)
+    if not os.path.exists(fullpath) or args.redownload or older_than_a_day(fullpath):
+        LOG.info(f"Downloading {fullpath}")
+        urlretrieve(url, fullpath, DownloadProgressBar())
+
+    hashtype, latest, size = parse_latest_iso_text(fullpath)
+    size = int(size)
+
+    # Download the iso file
+    filename = latest
+    fullpath = os.path.join(args.download_dir, filename)
+    if not os.path.exists(fullpath) or args.redownload:
+        LOG.info(f"Downloading {filename}")
+        url = os.path.join(config.GENTOO_BASE_ISO_URL, filename)
+        urlretrieve(url, fullpath, DownloadProgressBar())
+
+    # Verify byte size
+    isosize = os.path.getsize(fullpath)
+    assert size == isosize, f"ISO size {size} does not match expected value {isosize}."
+    verify(args, hashtype, config.GENTOO_BASE_ISO_URL, isohashpattern, filename)
+
+    return fullpath

src/ansible_gentooimgr/gentooimgr/install.py

@@ -0,0 +1,359 @@
+"""Configure a Gentoo guest with cloud image settings
+
+This step keeps track of how far it's gotten, so re-running this command
+will continue on if an error was to occur, unless --start-over flag is given.
+"""
+
+import os
+import sys
+import shutil
+import configparser
+from subprocess import Popen, PIPE
+import logging
+import traceback
+
+import gentooimgr.config
+import gentooimgr.configs
+import gentooimgr.common
+import gentooimgr.chroot
+import gentooimgr.kernel
+from gentooimgr import LOG
+from gentooimgr import HERE
+
+from gentooimgr.configs import *
+
+FILES_DIR = os.path.join(HERE, "..")
+
+def step1_diskprep(args, cfg):
+    LOG.info("\t:: Step 1: Disk Partitioning")
+    # http://rainbow.chard.org/2013/01/30/how-to-align-partitions-for-best-performance-using-parted/
+    # http://honglus.blogspot.com/2013/06/script-to-automatically-partition-new.html
+    cmds = [
+            ['parted', '-s', f'{cfg.get("disk")}', 'mklabel', 'msdos'],
+            ['parted', '-s', f'{cfg.get("disk")}', 'mkpart', 'primary', '2048s', '100%'],
+            ['partprobe'],
+            ['mkfs.ext4', '-FF', f'{cfg.get("disk")}{cfg.get("partition", 1)}']
+    ]
+    for c in cmds:
+        proc = Popen(c, stdout=PIPE, stderr=PIPE)
+        stdout, stderr = proc.communicate()
+
+    completestep(1, "diskprep")
+
+def step2_mount(args, cfg):
+    LOG.info(f'\t:: Step 2: Mounting {gentooimgr.config.GENTOO_MOUNT}')
+    proc = Popen(["mount", f'{cfg.get("disk")}{cfg.get("partition")}', cfg.get("mountpoint")])
+    proc.communicate()
+    completestep(2, "mount")
+
+def step3_stage3(args, cfg):
+    LOG.info(f'\t:: Step 3: Stage3 Tarball')
+
+    stage3 = cfg.get("stage3") or args.stage3  # FIXME: auto detect stage3 images in mountpoint and add here
+    if not stage3:
+        stage3 = gentooimgr.common.stage3_from_dir(FILES_DIR)
+
+
+    proc = Popen(["tar", "xpf", os.path.abspath(stage3), "--xattrs-include='*.*'", "--numeric-owner", "-C",
+                  f'{cfg.get("mountpoint")}'])
+    proc.communicate()
+    completestep(3, "stage3")
+
+def step4_binds(args, cfg):
+    LOG.info(f'\t:: Step 4: Binding Filesystems')
+    gentooimgr.chroot.bind(verbose=False)
+    completestep(4, "binds")
+
+def step5_portage(args, cfg):
+    LOG.info(f'\t:: Step 5: Portage')
+    portage = cfg.get("portage") or args.portage
+    if not portage:
+        portage = gentooimgr.common.portage_from_dir(FILES_DIR)
+    proc = Popen(["tar", "xpf", portage, "-C", f"{cfg.get('mountpoint')}/usr/"])
+    proc.communicate()
+    # Edit portage
+    portage_env = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'env')
+    os.makedirs(portage_env, exist_ok=True)
+    with open(os.path.join(portage_env, 'singlejob.conf'), 'w') as f:
+        f.write('MAKEOPTS="-j1"\n')
+
+    env_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'package.env')
+    with open(env_path, 'w') as f:
+        f.write("app-portage/eix singlejob.conf\ndev-util/maturin singlejob.conf\ndev-util/cmake singlejob.conf")
+
+    completestep(5, "portage")
+
+def step6_licenses(args, cfg):
+    LOG.info(f'\t:: Step 6: Licenses')
+    license_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'package.license')
+    os.makedirs(license_path, exist_ok=True)
+    for f, licenses in cfg.get("licensefiles", {}).items():
+        with open(os.path.join(license_path, f), 'w') as f:
+            f.write('\n'.join(licenses))
+    completestep(6, "license")
+
+def step7_repos(args, cfg):
+    LOG.info(f'\t:: Step 7: Repo Configuration')
+    repo_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'repos.conf')
+    os.makedirs(repo_path, exist_ok=True)
+    # Copy from template
+    repo_file = os.path.join(repo_path, 'gentoo.conf')
+    shutil.copyfile(
+        os.path.join(cfg.get("mountpoint"), 'usr', 'share', 'portage', 'config', 'repos.conf'),
+        repo_file)
+    # Regex replace lines
+    cp = configparser.ConfigParser()
+    for repofile, data in cfg.get("repos", {}).items():
+        cp.read(cfg.get("mountpoint") + repofile)  # repofile should be absolute path, do not use os.path.join.
+        for section, d in data.items():
+            if section in cp:
+                for key, val in d.items():
+                    # Replace everything after the key with contents of value.
+                    # Sed is simpler than using regex for this purpose.
+                    cp.set(section, key, val)
+            else:
+                sys.stderr.write(f"\tWW No section {section} in {repofile}\n")
+
+        cp.write(open(cfg.get("mountpoint") + repofile, 'w'))
+
+    completestep(7, "repos")
+
+def step8_resolv(args, cfg):
+    LOG.info(f'\t:: Step 8: Resolv')
+    proc = Popen(["cp", "--dereference", "/etc/resolv.conf", os.path.join(cfg.get("mountpoint"), 'etc')])
+    proc.communicate()
+    # Copy all step files and python module to new chroot
+    os.system(f"cp /tmp/*.step {cfg.get('mountpoint')}/tmp")
+    os.system(f"cp -r . {cfg.get('mountpoint')}/mnt/")
+    completestep(8, "resolv")
+
+def step9_sync(args, cfg):
+    LOG.info(f"\t:: Step 9: sync")
+    LOG.info("\t\t:: Entering chroot")
+    os.chroot(cfg.get("mountpoint"))
+    os.chdir(os.sep)
+    os.system("source /etc/profile")
+    proc = Popen(["emerge", "--sync", "--quiet"])
+    proc.communicate()
+    LOG.info("\t\t:: Emerging base")
+    proc = Popen(["emerge", "--update", "--deep", "--newuse", "--keep-going", "@world"])
+    proc.communicate()
+    completestep(9, "sync")
+
+def step10_emerge_pkgs(args, cfg):
+    LOG.info(f"\t:: Step 10: emerge pkgs")
+    packages = cfg.get("packages", {})
+    for oneshot_up in packages.get("oneshots", []):
+        proc = Popen(["emerge", "--oneshot", "--update", oneshot_up])
+        proc.communicate()
+
+    for single in packages.get("singles", []):
+        proc = Popen(["emerge", "-j1", single])
+        proc.communicate()
+
+    LOG.info(f"KERNEL PACKAGES {packages.get('kernel')}")
+    if packages.get("kernel", []):
+        cmd = ["emerge", "-j", str(args.threads)] + packages.get("kernel", [])
+        proc = Popen(cmd)
+        proc.communicate()
+
+    cmd = ["emerge", "-j", str(args.threads), "--keep-going"]
+    cmd += packages.get("keepgoing", [])
+    proc = Popen(cmd)
+    proc.communicate()
+
+    cmd = ["emerge", "-j", str(args.threads)]
+    cmd += packages.get("base", [])
+    cmd += packages.get("additional", [])
+    cmd += packages.get("bootloader", [])
+    LOG.info(cmd)
+    proc = Popen(cmd)
+    proc.communicate()
+    completestep(10, "pkgs")
+
+def step11_kernel(args, cfg):
+    # at this point, genkernel will be installed
+    LOG.info(f"\t:: Step 11: kernel")
+    proc = Popen(["eselect", "kernel", "set", "1"])
+    proc.communicate()
+    if not args.kernel_dist:
+        os.chdir(args.kernel_dir)
+        threads = str(gentooimgr.config.THREADS)
+        gentooimgr.kernel.build_kernel(args, cfg)
+
+    completestep(11, "kernel")
+
+def step12_grub(args, cfg):
+    LOG.info(f"\t:: Step 12: kernel")
+    proc = Popen(["grub-install", cfg.get('disk')])
+    proc.communicate()
+    code = proc.returncode
+    if code != 0:
+        sys.stderr.write(f"Failed to install grub on {cfg.get('disk')}\n")
+        sys.exit(code)
+
+    with open("/etc/default/grub", 'w') as f:
+        f.write(f"{gentooimgr.kernel.GRUB_CFG}")
+
+    proc = Popen(["grub-mkconfig", "-o", "/boot/grub/grub.cfg"])
+    proc.communicate()
+    completestep(12, "grub")
+
+def step13_serial(args, cfg):
+    LOG.info(f"\t:: Step 13: Serial")
+    os.system("sed -i 's/^#s0:/s0:/g' /etc/inittab")
+    os.system("sed -i 's/^#s1:/s1:/g' /etc/inittab")
+    completestep(13, "serial")
+
+def step14_services(args, cfg):
+    LOG.info(f"\t:: Step 14: Services")
+    for service in ["acpid", "syslog-ng", "cronie", "sshd", "cloud-init-local", "cloud-init", "cloud-config",
+                    "cloud-final", "ntpd", "nfsclient"]:
+        if args.profile == "systemd":
+            proc = Popen(["systemctl", "enable", service])
+        else:
+            proc = Popen(["rc-update", "add", service, "default"])
+        proc.communicate()
+
+    completestep(14, "services")
+
+def step15_ethnaming(args, cfg):
+    LOG.info(f"\t:: Step 15: Eth Naming")
+    completestep(15, "networking")
+
+def step16_sysconfig(args, cfg):
+    LOG.info(f"\t:: Step 16: Sysconfig")
+    with open("/etc/timezone", "w") as f:
+        f.write("UTC")
+    proc = Popen(["emerge", "--config", "sys-libs/timezone-data"])
+    proc.communicate()
+    with open("/etc/locale.gen", "a") as f:
+        f.write("en_US.UTF-8 UTF-8\nen_US ISO-8859-1\n")
+    proc = Popen(["locale-gen"])
+    proc.communicate()
+    proc = Popen(["eselect", "locale", "set", "en_US.utf8"])
+    proc.communicate()
+    proc = Popen(["env-update"])
+    proc.communicate()
+    with open('/etc/sysctl.d/swappiness.conf', 'w') as f:
+        f.write("vm.swappiness = 0\n")
+
+    modloadpath = os.path.join(os.sep, 'etc', 'modules-load.d')
+    os.makedirs(modloadpath, exist_ok=True)
+    with open(os.path.join(modloadpath, 'cloud-modules.conf'), 'w') as f:
+        f.write('\n'.join(gentooimgr.config.CLOUD_MODULES))
+
+    cloudcfg = os.path.join(os.sep, 'etc', 'cloud')
+    if not os.path.exists(cloudcfg):
+        os.makedirs(cloudcfg, exist_ok=True)
+        os.makedirs(os.path.join(cloudcfg, 'templates'), exist_ok=True)
+    with open(os.path.join(cloudcfg, 'cloud.cfg'), 'w') as cfg:
+        cfg.write(f"{CLOUD_YAML}")
+
+    os.chmod(os.path.join(cloudcfg, "cloud.cfg"), 0o644)
+
+    with open(os.path.join(cloudcfg, "templates", "hosts.gentoo.tmpl"), 'w') as tmpl:
+        tmpl.write(f"{HOST_TMPL}") # FIXME:
+
+    os.chmod(os.path.join(cloudcfg, "templates", "hosts.gentoo.tmpl"), 0o644)
+
+    proc = Popen("sed -i 's/domain_name\,\ domain_search\,\ host_name/domain_search/g' /etc/dhcpcd.conf", shell=True)
+    proc.communicate()
+
+    hostname = os.path.join(os.sep, 'etc', 'conf.d', 'hostname')
+    with open(hostname, 'w') as f:
+        f.write(f"{HOSTNAME}\n")
+
+    os.chmod(hostname, 0o644)
+
+    proc = Popen(["eix-update"])
+    proc.communicate()
+
+    os.remove(os.path.join(os.sep, 'etc', 'resolv.conf'))
+
+    completestep(16, "sysconfig")
+
+def step17_fstab(args, cfg):
+    LOG.info(f"\t:: Step 17: fstab")
+    with open(os.path.join(os.sep, 'etc', 'fstab'), 'a') as fstab:
+        fstab.write(f"{cfg.get('disk')}\t/\text4\tdefaults,noatime\t0 1\n")
+
+    completestep(17, "fstab")
+
+def completestep(step, stepname, prefix='/tmp'):
+    with open(os.path.join(prefix, f"{step}.step"), 'w') as f:
+        f.write("done.")  # text in this file is not currently used.
+
+
+def getlaststep(prefix='/tmp'):
+    i = 1
+    found = False
+    while not found:
+        if os.path.exists(f"{i}.step"):
+            i += 1
+        else:
+            found = True
+
+    return i
+
+
+def stepdone(step, prefix='/tmp'):
+    return os.path.exists(os.path.join(prefix, f"{step}.step"))
+
+def configure(args, config: dict):
+    # Load configuration
+    if not os.path.exists(gentooimgr.config.GENTOO_MOUNT):
+        if not args.force:
+            # We aren't in a gentoo live cd are we?
+            sys.stderr.write("Your system doesn't look like a gentoo live cd, exiting for safety.\n"
+                "If you want to continue, use --force option and re-run `python -m gentooimgr install` with your configuration\n")
+            sys.exit(1)
+
+        else:
+            # Assume we are root as per live cd, otherwise user should run this as root as a secondary confirmation
+            os.makedirs(gentooimgr.config.GENTOO_MOUNT)
+    # disk prep
+    cfg = config
+    if not stepdone(1): step1_diskprep(args, cfg)
+    # mount root
+    if not stepdone(2): step2_mount(args, cfg)
+    # extract stage
+    if not stepdone(3): step3_stage3(args, cfg)
+    # mount binds
+    if not stepdone(4): step4_binds(args, cfg)
+    # extract portage
+    if not stepdone(5): step5_portage(args, cfg)
+    # Set licenses
+    if not stepdone(6): step6_licenses(args, cfg)
+    # repos.conf
+    if not stepdone(7): step7_repos(args, cfg)
+    # portage env files and resolv.conf
+    if not stepdone(8): step8_resolv(args, cfg)
+    # emerge --sync
+    if not stepdone(9): step9_sync(args, cfg)
+    # bindist
+    if not stepdone(10): step10_emerge_pkgs(args, cfg)
+    # emerge packages
+    # configure & emerge kernel (use cloud configuration too)
+    if not stepdone(11): step11_kernel(args, cfg)
+    # grub
+    if not stepdone(12): step12_grub(args, cfg)
+    # enable serial console
+    if not stepdone(13): step13_serial(args, cfg)
+    # services
+    if not stepdone(14): step14_services(args, cfg)
+    # eth0 naming
+    # timezone
+    if not stepdone(15): step15_ethnaming(args, cfg)
+    # locale
+    # set some sysctl things
+    # set some dhcp things
+    # hostname
+    if not stepdone(16): step16_sysconfig(args, cfg)
+    # fstab
+    if not stepdone(17): step17_fstab(args, cfg)
+    # copy cloud cfg?
+    gentooimgr.chroot.unbind()
+    # Finish install processes like emaint and eix-update and news read
+
+

src/ansible_gentooimgr/gentooimgr/kernel.py

@@ -0,0 +1,128 @@
+import os
+import sys
+import re
+import shutil
+import datetime
+from subprocess import Popen, PIPE
+import time
+
+import gentooimgr.configs
+
+DEFAULT_KERNEL_CONFIG_PATH = os.path.join(os.sep, 'etc', 'kernel', 'default.config')
+
+def kernel_conf_apply(args, config):
+    """Kernel configuration is a direct copy of a full complete config file"""
+    fname, ext = os.path.splitext(args.config)
+    # Default is the json file's name but with .config extension.
+    kernelconfig = os.path.join(gentooimgr.configs.CONFIG_DIR,
+                                config.get("kernel", {}).get("config", f"{fname}.config"))
+    kernelpath = config.get("kernel", {}).get("path", DEFAULT_KERNEL_CONFIG_PATH)
+    if os.path.exists(kernelpath):
+        os.remove(kernelpath)
+    else:
+        # Ensure if we have directories specified that they exist
+        os.makedirs(os.path.dirname(kernelpath), exist_ok=True)
+
+    shutil.copyfile(kernelconfig, kernelpath)
+
+def build_kernel(args, config):
+    if config.get("kernel", {}).get("config") is None:
+        kernel_default_config(args, config)
+    os.chdir(args.kernel_dir)
+    kernelpath = config.get("kernel", {}).get("path", DEFAULT_KERNEL_CONFIG_PATH)
+    kernel_conf_apply(args, config)
+    proc = Popen(['genkernel', f'--kernel-config={kernelpath}', '--save-config', '--no-menuconfig', 'all'])
+    proc.communicate()
+    kernel_save_config(args, config)
+
+def kernel_default_config(args, config):
+    os.chdir(args.kernel_dir)
+    proc = Popen(["make", "defconfig"])
+    proc.communicate()
+
+def kernel_save_config(args, config):
+    os.chdir(args.kernel_dir)
+    """Saves the current .config file"""
+    proc = Popen(["make", "savedefconfig"])
+    proc.communicate()
+
+GRUB_CFG = """
+# Copyright 1999-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Header: /var/cvsroot/gentoo-x86/sys-boot/grub/files/grub.default-3,v 1.5 2015/03/25 01:58:00 floppym Exp $
+#
+# To populate all changes in this file you need to regenerate your
+# grub configuration file afterwards:
+#     'grub2-mkconfig -o /boot/grub/grub.cfg'
+#
+# See the grub info page for documentation on possible variables and
+# their associated values.
+
+GRUB_DISTRIBUTOR="Gentoo"
+
+# Default menu entry
+#GRUB_DEFAULT=0
+
+# Boot the default entry this many seconds after the menu is displayed
+#GRUB_TIMEOUT=5
+#GRUB_TIMEOUT_STYLE=menu
+
+# Append parameters to the linux kernel command line
+# openrc only spits to the last console=tty
+GRUB_CMDLINE_LINUX="net.ifnames=0 vga=791 console=tty0 console=ttyS0,115200"
+#
+# Examples:
+#
+# Boot with network interface renaming disabled
+# GRUB_CMDLINE_LINUX="net.ifnames=0"
+#
+# Boot with systemd instead of sysvinit (openrc)
+# GRUB_CMDLINE_LINUX="init=/usr/lib/systemd/systemd"
+
+# Append parameters to the linux kernel command line for non-recovery entries
+#GRUB_CMDLINE_LINUX_DEFAULT=""
+
+# Uncomment to disable graphical terminal (grub-pc only)
+GRUB_TERMINAL="serial console"
+GRUB_SERIAL_COMMAND="serial --speed=115200"
+#GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"
+
+# The resolution used on graphical terminal.
+# Note that you can use only modes which your graphic card supports via VBE.
+# You can see them in real GRUB with the command `vbeinfo'.
+#GRUB_GFXMODE=640x480
+
+# Set to 'text' to force the Linux kernel to boot in normal text
+# mode, 'keep' to preserve the graphics mode set using
+# 'GRUB_GFXMODE', 'WIDTHxHEIGHT'['xDEPTH'] to set a particular
+# graphics mode, or a sequence of these separated by commas or
+# semicolons to try several modes in sequence.
+#GRUB_GFXPAYLOAD_LINUX=
+
+# Path to theme spec txt file.
+# The starfield is by default provided with use truetype.
+# NOTE: when enabling custom theme, ensure you have required font/etc.
+#GRUB_THEME="/boot/grub/themes/starfield/theme.txt"
+
+# Background image used on graphical terminal.
+# Can be in various bitmap formats.
+#GRUB_BACKGROUND="/boot/grub/mybackground.png"
+
+# Uncomment if you don't want GRUB to pass "root=UUID=xxx" parameter to kernel
+#GRUB_DISABLE_LINUX_UUID=true
+
+# Uncomment to disable generation of recovery mode menu entries
+#GRUB_DISABLE_RECOVERY=true
+
+# Uncomment to disable generation of the submenu and put all choices on
+# the top-level menu.
+# Besides the visual affect of no sub menu, this makes navigation of the
+# menu easier for a user who can't see the screen.
+GRUB_DISABLE_SUBMENU=y
+
+# Uncomment to play a tone when the main menu is displayed.
+# This is useful, for example, to allow users who can't see the screen
+# to know when they can make a choice on the menu.
+#GRUB_INIT_TUNE="60 800 1"
+
+"""

src/ansible_gentooimgr/gentooimgr/qemu.py

@@ -0,0 +1,101 @@
+"""Qemu commands to run and handle the image"""
+import os
+import sys
+import argparse
+from subprocess import Popen, PIPE
+
+from gentooimgr import LOG
+import gentooimgr.config
+import gentooimgr.common
+def create_image(args, config: dict, overwrite: bool = False) -> str:
+    """Creates an image (.img) file using qemu that will be used to create the cloud image
+
+    :Parameters:
+        - config: dictionary/json configuration containing required information
+        - overwrite: if True, run_image() will call this and re-create.
+
+
+    :Returns:
+        Full path to image file produced by qemu
+    """
+
+    image = gentooimgr.common.get_image_name(args, config)
+    name, ext = os.path.splitext(image)
+    if os.path.exists(image) and not overwrite:
+        return os.path.abspath(image)
+
+    cmd = ['qemu-img', 'create', '-f', ext[1:], image, str(config.get("imgsize", "12G"))]
+    proc = Popen(cmd, stderr=PIPE, stdout=PIPE)
+    stdout, stderr = proc.communicate()
+    return os.path.abspath(image)
+
+def run_image(
+    args: argparse.Namespace,
+    config: dict,
+    mounts=[]):
+    """Handle mounts and run the live cd image
+
+        - mount_isos: list of iso paths to mount in qemu as disks.
+    """
+    iso = config.get("iso")
+    prefix = args.temporary_dir
+    LOG.info(f"iso from config {iso}")
+    if iso is None:
+        iso = gentooimgr.common.find_iso(
+            os.path.join(
+                os.path.abspath(os.path.dirname(__file__)),
+                ".."
+            )
+        )
+        LOG.info(f"iso from cwd {iso}")
+    if not iso:
+        prefix = config.get('temporary_dir')
+        iso = gentooimgr.common.find_iso(prefix)
+        LOG.info(f"iso from {prefix} {iso}")
+
+    assert iso, f"iso not found {iso}"
+
+    if isinstance(iso, list):
+        assert len(iso), f"iso list is empty {iso}"
+        iso = iso[0]
+
+    image = gentooimgr.common.get_image_name(args, config)
+    qmounts = []
+    mounts.extend(args.mounts)
+    for i in mounts:
+        qmounts.append("-drive")
+        qmounts.append(f"file={i},media=cdrom")
+
+    assert image, f"image is empty {image}"
+    if not os.path.exists(image):
+        if os.path.exists(os.path.join(prefix, image)):
+            image = os.path.join(prefix, image)
+    assert os.path.exists(image), f"image not found {image}"
+    threads = args.threads
+    cmd = [
+        "qemu-system-x86_64",
+        "-enable-kvm",
+        "-boot", "d",
+        "-m", str(config.get("memory", 2048)),
+        "-smp", str(threads),
+        "-drive", f"file={image},if=virtio,index=0",
+        "-cdrom", iso,
+        "-net", "nic,model=virtio",
+        "-vga", "virtio",
+        "-cpu", "kvm64",
+        "-chardev", "file,id=charserial0,path=gentoo.log",
+        "-device", "isa-serial,chardev=charserial0,id=serial0",
+        "-chardev", "pty,id=charserial1",
+        "-device", "isa-serial,chardev=charserial1,id=serial1"
+    ]
+    # "-net", "user",
+    # -net user: network backend 'user' is not compiled into this binary"
+
+    cmd += qmounts
+    LOG.info(cmd)
+    proc = Popen(cmd, stderr=PIPE, stdout=PIPE)
+    stdout, stderr = proc.communicate()
+    if stderr:
+        LOG.error(str(stderr))
+
+

src/ansible_gentooimgr/gentooimgr/run.py

@@ -0,0 +1,37 @@
+import os
+
+from gentooimgr import LOG
+import gentooimgr.config
+import gentooimgr.qemu
+import gentooimgr.common
+
+def run(args, config: dict) -> None:
+    LOG.info(": run")
+    mounts = args.mounts
+    # Specified image or look for gentoo.{img,qcow2}
+    image = config.get("imagename") or args.image or gentooimgr.qemu.create_image()
+    # We need to package up our gentooimgr package into an iso and mount it to the running image
+    # Why? basic gentoo livecd has no git and no pip installer. We want install to be simple
+    # and use the same common codebase.
+
+    # This will require a couple mount commands to function though.
+    main_iso = gentooimgr.common.make_iso_from_dir(os.path.join(
+        os.path.abspath(os.path.dirname(__file__)),
+        ".."
+    ))
+
+    assert os.path.isfile(main_iso), f"iso not found {main_iso}"
+    LOG.info(args)
+    LOG.info(f'iso={args.iso}')
+    if args.iso != config['iso']:
+        LOG.warn(f'iso={args.iso}')
+        config['iso'] = args.iso
+    else:
+        LOG.info(f'iso={args.iso}')
+    gentooimgr.qemu.run_image(
+        args,
+        config,
+        # Add our generated mount and livecd (assumed)
+        mounts=[main_iso]
+    )
+    LOG.info("done")

src/ansible_gentooimgr/gentooimgr/shrink.py

@@ -0,0 +1,15 @@
+import os
+import datetime
+from subprocess import PIPE, Popen
+
+def shrink(args, config,  stamp=None):
+    if stamp is None:
+        dt = datetime.datetime.utcnow()
+        # 0 padded month and day timestamp
+        stamp = f"{dt.year}-{dt.month:02d}-{dt.day:02d}"
+    name, ext = os.path.splitext(config.get("imagename") or args.img)
+    # ext includes the .
+    filename = f"{name}-{stamp}{ext}"
+    proc = Popen(["virt-sparsify", "--compress", args.img, filename])
+    proc.communicate()
+    return filename

src/ansible_gentooimgr/gentooimgr/status.py

@@ -0,0 +1,54 @@
+"""Step 1: Disk Partitioning
+Step 2: Mounting {gentooimgr.config.GENTOO_MOUNT}
+Step 3: Stage3 Tarball
+Step 4: Binding Filesystems
+Step 5: Portage
+Step 6: Licenses
+Step 7: Repo Configuration
+Step 8: Resolv
+Step 9: sync
+Step 10: emerge pkgs
+Step 11: kernel
+Step 12: kernel
+Step 13: Serial
+Step 14: Services
+Step 15: Eth Naming
+Step 16: Sysconfig
+Step 17: fstab
+"""
+import os
+import sys
+import json
+
+# from gentooimgr import LOG
+import gentooimgr.config
+import gentooimgr.configs
+from gentooimgr import install
+
+def print_template(args, configjson, prefix='/tmp'):
+    print(__doc__)
+    sys.stderr.write(f"the last step to succeed is {install.getlaststep(prefix)}\n")
+    print(f"the last step to succeed is {install.getlaststep(prefix)}\n")
+    print(f"""------------------------ STATUS ------------------------
+
+CPU_THREADS = {args.threads or 1}
+TEMPORARY_DIRECTORY = {args.temporary_dir}
+PROFILE = {args.profile}
+""")
+    print(f"CONFIG {args.config}")
+    print(json.dumps(configjson, sort_keys=True, indent=4))
+
+    # inherit = configjson.get("inherit")
+    # if inherit:
+    #     print(f"CONFIG {inherit}")
+    #     j = gentooimgr.config.load_default_config(inherit)
+    #     if not j:
+    #         j = gentooimgr.config.load_config(inherit)
+    #
+    #     print(json.dumps(j, sort_keys=True, indent=4))
+
+    # print(f"""------------------------ PACKAGES ------------------------""")
+    # for k, v in configjson.get("packages").items():
+    #     print(k.upper())
+    #     print("\t" + '\n\t'.join(v))
+    #     print()

src/ansible_gentooimgr/library/ansible-keepassxc

@@ -0,0 +1,189 @@
+#!/usr/bin/python
+
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import traceback
+from ansible.module_utils.basic import AnsibleModule, missing_required_lib
+
+IMPORT_ERR = None
+try:
+#    import _argon2_xffi_bindings
+    import pykeepass as keepass
+except ImportError:
+    IMPORT_ERR = traceback.format_exc()
+
+DOCUMENTATION = r'''
+---
+module: ansible-keepassxc
+
+short_description: Module to read credentials from KeePassXC
+
+version_added: "0.0.1"
+
+description: Module to read credentials from KeePassXC
+
+options:
+    database:
+        description: Path to database file
+        required: true
+        type: str
+    password:
+        description: Database Password
+        required: true
+        type: str
+    keyfile:
+        description: Path to key file
+        required: false
+        type: str
+    entry:
+        description: Entry name for the attribute to fetch
+        required: true
+        type: str
+    group:
+        decription: Group name that the Entry belongs to
+        required: false
+        type: str
+
+author:
+    - Jeremy Lumley (@jlumley)
+'''
+
+EXAMPLES = r'''
+# Fetch the credentials for the server_1 entry in any group
+- name: Fetch server_1 credentials
+  jlumley.jlumley.ansible-keepassxc:
+    database: "/secrets/db.kdbx"
+    password: "s3cure_p4550rd"
+    entry: "server_1"
+
+# Fetch the reddit entry in the social group
+- name: Fetching reddit credentials
+  jlumley.jlumley.ansible-keepassxc:
+    database: "/secrets/db.kdbx"
+    password: "sup3r_s3cure_p4550rd"
+    entry: "reddit"
+    group: "social"
+
+# Fetch a custom strig attribute from the github entry
+- name: Fetch Github API Token
+  jlumley.jlumley.ansible-keepassxc:
+    database: "/secrets/db.kdbx"
+    password: "d0pe_s3cure_p4550rd"
+    keyfile: "/secrets/top_secret_key"
+    entry: "github"
+    group: "development"
+
+'''
+
+RETURN = r'''
+# Return values
+username:
+    description: Username of entry if present
+    type: str
+    returned: always
+    sample: 's3cr3t_us3r'
+password:
+    description: Password of entry if present
+    type: str
+    returned: always
+    sample: 's3cr3t_p455word'
+url:
+    description: Url of entry if present
+    type: str
+    returned: always
+    sample: 'http://reddit.com'
+custom_fields:
+    description: dictionary containing all custom fields
+    type: dict
+    returned: always
+    sample: False
+no_log:
+    description: suppress logging of password
+    type: bool
+    returned: never
+    sample: False
+'''
+
+def run_module():
+    # define available arguments/parameters a user can pass to the module
+    module_args = dict(
+        database    = dict(type='str', required=True),
+        password    = dict(type='str', required=False,
+                           default=os.environ.get('ANSIBLE_KEEPASSXC_PASSWORD')),
+        keyfile     = dict(type='str', required=False, default=None),
+        entry       = dict(type='str', required=True),
+        group       = dict(type='str', required=False),
+        no_log     = dict(type='bool', required=False, default=False),
+    )
+
+    # seed the result dict in the object
+    result = dict(
+        changed=False,
+        username='',
+        password='',
+        url='',
+        custom_fields={}
+    )
+
+    # Currently no support for a check_mode this maybe added later if
+    # functionality to modify the database is added later
+    module = AnsibleModule(
+        argument_spec=module_args,
+        supports_check_mode=False,
+    )
+
+    if IMPORT_ERR:
+        module.fail_json(
+            msg=missing_required_lib("pykeepass"),
+            exception=IMPORT_ERR
+        )
+
+    # unlock local keepass database
+    try:
+        kp = keepass.PyKeePass(
+            module.params['database'],
+            password=module.params['password'],
+            keyfile=module.params['keyfile'])
+    except keepass.exceptions.CredentialsError:
+        module.fail_json(msg='Invalid Credentials')
+
+    # find entry
+    entry = kp.find_entries(
+        title=module.params['entry'],
+        group=module.params['group']
+    )
+
+    # fail is entry is not present
+    if not entry:
+        module.fail_json(msg=f"Unable to find entry: {module.params['entry']}")
+
+    else:
+        entry = entry[0]
+        custom_field_keys = entry._get_string_field_keys(exclude_reserved=True)
+        custom_fields = dict()
+        for key in custom_field_keys:
+            custom_fields[key] = entry.get_custom_property(key)
+        result = dict (
+            changed=False,
+            username=entry.username,
+            password=entry.password,
+            url=entry.url,
+            custom_fields=custom_fields
+        )
+
+    # in the event of a successful module execution, you will want to
+    # simple AnsibleModule.exit_json(), passing the key/value results
+    module.exit_json(**result)
+
+
+def main():
+    run_module()
+
+
+if __name__ == '__main__':
+    main()
+
+

src/ansible_gentooimgr/library/ansible-keepassxc.py

@@ -0,0 +1,189 @@
+#!/usr/bin/python
+
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import traceback
+from ansible.module_utils.basic import AnsibleModule, missing_required_lib
+
+IMPORT_ERR = None
+try:
+#    import _argon2_xffi_bindings
+    import pykeepass as keepass
+except ImportError:
+    IMPORT_ERR = traceback.format_exc()
+
+DOCUMENTATION = r'''
+---
+module: ansible-keepassxc
+
+short_description: Module to read credentials from KeePassXC
+
+version_added: "0.0.1"
+
+description: Module to read credentials from KeePassXC
+
+options:
+    database:
+        description: Path to database file
+        required: true
+        type: str
+    password:
+        description: Database Password
+        required: true
+        type: str
+    keyfile:
+        description: Path to key file
+        required: false
+        type: str
+    entry:
+        description: Entry name for the attribute to fetch
+        required: true
+        type: str
+    group:
+        decription: Group name that the Entry belongs to
+        required: false
+        type: str
+
+author:
+    - Jeremy Lumley (@jlumley)
+'''
+
+EXAMPLES = r'''
+# Fetch the credentials for the server_1 entry in any group
+- name: Fetch server_1 credentials
+  jlumley.jlumley.ansible-keepassxc:
+    database: "/secrets/db.kdbx"
+    password: "s3cure_p4550rd"
+    entry: "server_1"
+
+# Fetch the reddit entry in the social group
+- name: Fetching reddit credentials
+  jlumley.jlumley.ansible-keepassxc:
+    database: "/secrets/db.kdbx"
+    password: "sup3r_s3cure_p4550rd"
+    entry: "reddit"
+    group: "social"
+
+# Fetch a custom strig attribute from the github entry
+- name: Fetch Github API Token
+  jlumley.jlumley.ansible-keepassxc:
+    database: "/secrets/db.kdbx"
+    password: "d0pe_s3cure_p4550rd"
+    keyfile: "/secrets/top_secret_key"
+    entry: "github"
+    group: "development"
+
+'''
+
+RETURN = r'''
+# Return values
+username:
+    description: Username of entry if present
+    type: str
+    returned: always
+    sample: 's3cr3t_us3r'
+password:
+    description: Password of entry if present
+    type: str
+    returned: always
+    sample: 's3cr3t_p455word'
+url:
+    description: Url of entry if present
+    type: str
+    returned: always
+    sample: 'http://reddit.com'
+custom_fields:
+    description: dictionary containing all custom fields
+    type: dict
+    returned: always
+    sample: False
+no_log:
+    description: suppress logging of password
+    type: bool
+    returned: never
+    sample: False
+'''
+
+def run_module():
+    # define available arguments/parameters a user can pass to the module
+    module_args = dict(
+        database    = dict(type='str', required=True),
+        password    = dict(type='str', required=False,
+                           default=os.environ.get('ANSIBLE_KEEPASSXC_PASSWORD')),
+        keyfile     = dict(type='str', required=False, default=None),
+        entry       = dict(type='str', required=True),
+        group       = dict(type='str', required=False),
+        no_log     = dict(type='bool', required=False, default=False),
+    )
+
+    # seed the result dict in the object
+    result = dict(
+        changed=False,
+        username='',
+        password='',
+        url='',
+        custom_fields={}
+    )
+
+    # Currently no support for a check_mode this maybe added later if
+    # functionality to modify the database is added later
+    module = AnsibleModule(
+        argument_spec=module_args,
+        supports_check_mode=False,
+    )
+
+    if IMPORT_ERR:
+        module.fail_json(
+            msg=missing_required_lib("pykeepass"),
+            exception=IMPORT_ERR
+        )
+
+    # unlock local keepass database
+    try:
+        kp = keepass.PyKeePass(
+            module.params['database'],
+            password=module.params['password'],
+            keyfile=module.params['keyfile'])
+    except keepass.exceptions.CredentialsError:
+        module.fail_json(msg='Invalid Credentials')
+
+    # find entry
+    entry = kp.find_entries(
+        title=module.params['entry'],
+        group=module.params['group']
+    )
+
+    # fail is entry is not present
+    if not entry:
+        module.fail_json(msg=f"Unable to find entry: {module.params['entry']}")
+
+    else:
+        entry = entry[0]
+        custom_field_keys = entry._get_string_field_keys(exclude_reserved=True)
+        custom_fields = dict()
+        for key in custom_field_keys:
+            custom_fields[key] = entry.get_custom_property(key)
+        result = dict (
+            changed=False,
+            username=entry.username,
+            password=entry.password,
+            url=entry.url,
+            custom_fields=custom_fields
+        )
+
+    # in the event of a successful module execution, you will want to
+    # simple AnsibleModule.exit_json(), passing the key/value results
+    module.exit_json(**result)
+
+
+def main():
+    run_module()
+
+
+if __name__ == '__main__':
+    main()
+
+

src/ansible_gentooimgr/library/ansible_gentooimgr.py

@@ -0,0 +1,233 @@
+#!/usr/bin/python3
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import sys
+import logging
+from argparse import Namespace
+import pathlib
+import traceback
+
+# in the library
+if os.environ.get('PLAY_ANSIBLE_SRC',''):
+    # running from source
+    mod_path = os.environ.get('PLAY_ANSIBLE_SRC','')
+    mod_path = os.path.join(mod_path, 'src', 'ansible_gentooimgr')
+    assert os.path.isdir(mod_path), f"parent {mod_path}"
+    assert os.path.isfile(os.path.join(mod_path, '__init__.py')),f"index {mod_path}"
+    assert os.path.isdir(os.path.join(mod_path, 'gentooimgr')), f"sub {mod_path}"
+    sys.path.append(mod_path)
+else:
+    # in the library
+    mod_path = os.path.dirname(os.path.realpath('__file__'))
+    mod_path = os.path.join(mod_path, 'src', 'ansible_gentooimgr')
+    assert os.path.isdir(mod_path), f"parent {mod_path}"
+    assert os.path.isfile(os.path.join(mod_path, '__init__.py')),f"index {mod_path}"
+    assert os.path.isdir(os.path.join(mod_path, 'gentooimgr')), f"sub {mod_path}"
+    sys.path.append(mod_path)
+try:
+    import gentooimgr
+except Exception as e:
+    sys.stderr.write(f"{mod_path} {sys.path} {traceback.print_exc()}")
+    raise
+import ansible
+
+DOCUMENTATION = rf'''
+---
+module: gentooimgr
+
+short_description: Gentoo Image Builder for Cloud and Turnkey ISO installers
+
+
+version_added: "1.0.0"
+
+description:
+* This project enables easy access to building ``systemd`` or ``openrc`` -based images.
+* Performs automatic download AND verification of the linux iso, stage3 tarball and portage.
+* Caches the iso and stage3 .txt files for at most a day before redownloading and rechecking for new files
+* Sane and readable cli commands to build, run and test.
+* Step system to enable user to continue off at the same place if a step fails
+* No heavy packages like rust included ** TODO
+
+options:
+    action:
+        description: The action to be run by the image builder
+        choices:
+            - build
+            - run
+            - status
+            - install
+            - chroot
+            - unchroot
+            - command
+            - shrink
+            - kernel
+        required: true
+        # clean test
+    config:
+        default: cloud.json
+        description: init configuration file or or base.json or cloud.json
+        required: false
+    loglevel:
+        default: {logging.INFO}
+        description: python logging level <= 50, INFO=20
+        required: false
+    threads:
+        default: 1
+        description: Number of threads to use
+        required: false
+    profile:
+        default: openrc
+        description: The init system
+        choices:
+            - openrc
+            - systemd
+        required: false
+    kernel_dir:
+        default: /usr/src/linux
+        description: Where kernel is specified. By default uses the active linux kernel
+        required: false
+    portage:
+        description: Extract the specified portage tarball onto the filesystem
+        required: false
+    stage3:
+        description: Extract the specified stage3 package onto the filesystema
+        required: false
+    action_args:
+        default: []
+        description: Arguments for some of the actions - UNUSED!
+        required: false
+    temporary_dir:
+        description: Path to temporary directory for downloading files (20G)
+        required: false
+    qcow:
+        description: Path to file to serve as the base image
+        required: false
+
+# Specify this value according to your collection
+# in format of namespace.collection.doc_fragment_name
+# extends_documentation_fragment:
+#     - my_namespace.my_collection.my_doc_fragment_name
+
+author:
+    - Your Name (@yourGitHubHandle)
+'''
+
+#[-y DAYS]
+#                  [-d DOWNLOAD_DIR]
+# [-f]
+#                  [--format FORMAT]
+
+EXAMPLES = r'''
+# Pass in a message
+- name: Test with a message
+  my_namespace.my_collection.my_test:
+    name: hello world
+
+# pass in a message and have changed true
+- name: Test with a message and changed output
+  my_namespace.my_collection.my_test:
+    name: hello world
+    new: true
+
+# fail the module
+- name: Test failure of the module
+  my_namespace.my_collection.my_test:
+    name: fail me
+'''
+
+RETURN = r'''
+# These are examples of possible return values, and in general should use other names for return values.
+message:
+    description: The output message that the test module generates.
+    type: str
+    returned: always
+    sample: 'goodbye'
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def run_module():
+    # define available arguments/parameters a user can pass to the module
+    module_args = dict(
+        action=dict(type='str', required=True),
+        loglevel=dict(type='int', required=False, default=logging.INFO),
+        threads=dict(type='int', required=False, default=1),
+        config=dict(type='str', default='cloud.json', required=False),
+        profile=dict(type='str', required=False),
+        kernel_dir=dict(type='path', required=False),
+        portage=dict(type='path', required=False),
+        stage3=dict(type='path', required=False),
+        temporary_dir=dict(type='path', required=False, default=pathlib.Path(os.getcwd())),
+        download_dir=dict(type='path', required=False, default=pathlib.Path(os.getcwd())),
+        qcow=dict(type='path', required=False),
+    )
+
+    # seed the result dict in the object
+    # we primarily care about changed and state
+    # changed is if this module effectively modified the target
+    # state will include any data that you want your module to pass back
+    # for consumption, for example, in a subsequent task
+    result = dict(
+        changed=False,
+        original_message='',
+        message=''
+    )
+
+    # the AnsibleModule object will be our abstraction working with Ansible
+    # this includes instantiation, a couple of common attr would be the
+    # args/params passed to the execution, as well as if the module
+    # supports check mode
+    module = AnsibleModule(
+        argument_spec=module_args,
+        supports_check_mode=True
+    )
+
+    # if the user is working with this module in only check mode we do not
+    # want to make any changes to the environment, just return the current
+    # state with no modifications
+    if module.check_mode:
+        module.exit_json(**result)
+
+    # manipulate or modify the state as needed (this is going to be the
+    # part where your module will do what it needs to do)
+    # if module.params.get('thirsty'):
+
+    oargs = Namespace(**module.params)
+    # during the execution of the module, if there is an exception or a
+    # conditional state that effectively causes a failure, run
+    # AnsibleModule.fail_json() to pass in the message and the result
+    result['original_message'] = ""
+    try:
+        from gentooimgr.__main__ import main
+        retval = main(oargs)
+    except Exception as e:
+        result['message'] = str(e)
+        e = traceback.print_exc()
+        if e: result['original_message'] += f"{e}"
+        module.fail_json(msg='Exception', **result)
+    else:
+        result['message'] = str(retval)
+
+    # use whatever logic you need to determine whether or not this module
+    # made any modifications to your target
+    if dArgs['action'] in ['status']:
+        result['changed'] = False
+    else:
+        result['changed'] = True
+
+    # in the event of a successful module execution, you will want to
+    # simple AnsibleModule.exit_json(), passing the key/value results
+    module.exit_json(**result)
+
+
+def main():
+    run_module()
+
+
+if __name__ == '__main__':
+    main()
+

src/ansible_gentooimgr/requirements.txt

@@ -0,0 +1,5 @@
+pydantic
+typing
+urllib
+progressbar
+requests