update

.gitignore

@@ -1,166 +1,34 @@
-# ---> Python
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-*.diff
-.pylint.*
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-
 .pylint.err
-.pylint.log
+.pylint.out
+*.pyc
+*.pyo
+
+libs/
+*.egg-info
+*.log
+*.out
+*.bak
+
+.idea
+*~
+#*
+*.iml
+*.junk
+
+*.so
+*.log
+toxygen/build
+toxygen/dist
+*.spec
+dist
+toxygen/avatars
+toxygen/__pycache__
+/*.egg-info
+/*.egg
+html
+Toxygen.egg-info
+*.tox
+.cache
+*.db
+*~
+Makefile

.rsync.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+find * -name \*.py | xargs grep -l '[ 	]*$' | xargs sed -i -e 's/[	 ]*$//'
+rsync "$@" -vax  --include \*.py --exclude \*.log --exclude \*.out \
+	--exclude \*.egg-info --exclude libs --exclude dist  --exclude build \
+	--exclude \*.pyc --exclude .pyl\* --exclude \*~ --exclude \*.so \
+	./ ../tox_profile.git/|grep -v /$

README.md

@@ -3,10 +3,13 @@
 Read and manipulate tox profile files. It started as a simple script from
 <https://stackoverflow.com/questions/30901873/what-format-are-tox-files-stored-in>
 
-```tox_savefile.py``` reads a Tox profile and prints to stderr various
+```tox_profile.py``` reads a Tox profile and prints to stderr various
 things that it finds.  Then can write what it found in JSON/YAML/REPR/PPRINT
 to a file. It can also test the nodes in a profile using ```nmap```.
 
+( There are sometimes problems with the json info dump of bytes keys:
+```TypeError: Object of type bytes is not JSON serializable```)
+
 It can also download, select, or test nodes in a ```DHTnode.json``` file.
 
 It can also decrypt a profile, saving the output to a file.
@@ -30,11 +33,11 @@ to stdout
 to a file.
 
 ```
-usage: tox_savefile.py [-h]
+usage: tox_profile.py [-h]
-                               [--command info|decrypt|nodes|edit]
+       [--command info|decrypt|nodes|edit|onions]
-                               [--info info|repr|yaml|json|pprint|nmap_udp|nmap_tcp]
+       [--info info|repr|yaml|json|pprint|nmap_dht|nmap_relay]
        [--indent INDENT]
-                               [--nodes select_tcp|select_udp|select_version|nmap_tcp|nmap_udp,download]
+       [--nodes select_tcp|select_udp|select_version|nmap_tcp|nmap_udp|download|check|clean]
        [--download_nodes_url DOWNLOAD_NODES_URL]
        [--edit help|section,num,key,val]
        [--output OUTPUT]
@@ -50,7 +53,7 @@ Optional arguments:
   --command {info,decrypt,nodes,edit}
                         Action command - default: info
   --output OUTPUT       Destination for info/decrypt/nodes - can be the same as input
-  --info info|repr|yaml|json|pprint|nmap_udp|nmap_tcp (may require nmap)
+  --info info|repr|yaml|json|pprint|nmap_dht|nmap_relay (may require nmap)
                         Format for info command
   --indent INDENT       Indent for yaml/json/pprint
   --nodes select_tcp|select_udp|select_version|nmap_tcp|nmap_udp|download
@@ -66,10 +69,23 @@ Optional arguments:
 Choose one of ```{info,repr,yaml,json,pprint,save}```
 for the format for info command.
 
-Choose one of ```{nmap_udp,nmap_tcp}```
+Choose one of ```{nmap_dht,nmap_relay,nmap_path}```
 to run tests using ```nmap``` for the ```DHT``` and ```TCP_RELAY```
 sections of the profile. Reguires ```nmap``` and uses ```sudo```.
 
+```
+  --info default='info',
+         choices=[info, save, repr, yaml,json, pprint]
+         with --info=info prints info about the profile to stderr
+         yaml,json, pprint, repr - output format
+         nmap_dht        - test DHT nodes with nmap
+         nmap_relay        - test TCP_RELAY nodes with nmap
+         nmap_path      - test PATH_NODE nodes with nmap
+  --indent for pprint/yaml/json default=2
+
+
+```
+
 #### Saving a copy
 
 The code now can generate a saved copy of the profile as it parses the profile.
@@ -83,6 +99,7 @@ decryption).
 
 ### --command nodes
 
+
 Takes a DHTnodes.json file as an argument.
 Choose one of ```{select_tcp,select_udp,select_version}```
 for ```--nodes``` to select TCP nodes, UDP nodes,
@@ -94,6 +111,29 @@ Reguires ```nmap``` and uses ```sudo```.
 
 Choose ```download``` to download the nodes from ```--download_nodes_url```
 
+Choose ```check``` to check the  downloaded nodes, and the error return
+is the number of nodes with errors. 
+
+Choose ```clean``` to clean the  downloaded nodes, and give
+```--output``` for the file the  nodes ckeaned of  errors.
+
+Check and clean will also try to ping the nodes on the relevant ports,
+and clean will update the ```status_tcp``, ```status_udp```, and
+```last_ping``` fields of the nodes.
+
+  --nodes
+       choices=[select_tcp, select_udp, nmap_tcp, select_version, nmap_udp, check, download]
+       select_udp      - select udp nodes
+       select_tcp      - select tcp nodes
+       nmap_udp        - test UDP nodes with nmap
+       nmap_tcp        - test TCP nodes with nmap
+       select_version  - select nodes that are the latest version
+       download        - download nodes from --download_nodes_url
+       check           - check nodes from --download_nodes_url
+       clean           - check nodes and save them as --output
+  --download_nodes_url https://nodes.tox.chat/json
+```
+
 ### --command decrypt
 
 Decrypt a profile, with ```--output``` to a filename.
@@ -124,6 +164,12 @@ The ```num``` field is to accomodate sections that have lists:
 The ```--output``` can be the same as input as the input file is read
 and closed before processing starts.
 
+```
+  --edit
+       help               - print a summary of what fields can be edited
+      section,num,key,val - edit the field section,num,key with val
+```
+
 You can use the ```---edit``` command to synchronize profiles, by
 keeping the keypair and synchronize profiles between different clients:
 e.g. your could keep your profile from toxic as master, and copy it over
@@ -132,7 +178,8 @@ your qtox/toxygen/TriFa profile while preserving their keypair and NOSPAM:
 1. Use ```--command info --info info``` on the target profile to get the
    ```Nospam```, ```Public_key``` and ```Private_key``` of the target.
 2. Backup the target and copy the source profile to the target.
-3. Edit the target with the values from 1) with:```
+3. Edit the target with the values from 1) with:
+```
 --command edit --edit NOSPAMKEYS,.,Nospam,hexstr --output target target
 --command edit --edit NOSPAMKEYS,.,Public_key,hexstr --output target target
 --command edit --edit NOSPAMKEYS,.,Private_key,hexstr --output target target
@@ -165,7 +212,7 @@ required. It's available in most distros, or <https://nmap.org/>
 
 ## Issues
 
-https://git.macaw.me/emdee/tox_profile/issues
+https://git.plastiras.org/emdee/tox_profile/issues
 
 ## Future Directions
 
@@ -175,7 +222,33 @@ Because it's written in Python it is easy to extend to, for example,
 supporting multidevices:
 <https://git.plastiras.org/emdee/tox_profile/wiki/MultiDevice-Announcements-POC>
 
+There are a couple of bash scripts to show usage:
+* tox_profile_examples.bash - simple example usage
+* tox_profile_test.bash - a real test runner that still needs documenting.
+
 ## Specification
 
 There is a copy of the Tox [spec](https://toktok.ltd/spec.html)
 in the repo - it is missing any description of the groups section.
+
+## Updates
+
+Although Tox works over Tor, we do not recommend its usage for
+anonymity as it leaks DNS requests due to a 6-year old known security
+issue: https://github.com/TokTok/c-toxcore/issues/469 unless your Tox
+client does hostname lookups before calling Tox (like
+[toxygen](https://git.plastiras.org/emdee/toxygen) does).
+Otherwise, do not use it for anonymous communication unless you have a
+TCP and UDP firewall in place.
+
+The Tox project does not follow semantic versioning so the project may
+break the underlying ctypes wrapper at any time; it's not possible to
+use Tox version numbers to tell what the API will be. The last git version
+this code was tested with is ``1623e3ee5c3a5837a92f959f289fcef18bfa9c959```
+of Feb 12 10:06:37 2024. In which case you'll have to go into the tox.py
+file in https://git.plastiras.org/emdee/toxygen_wrapper to fix it yourself.
+
+The uptodate version of this code is on https://git.plastiras.org/emdee/tox_profile
+
+Work on this project is suspended until the
+[MultiDevice](https://git.plastiras.org/emdee/tox_profile/wiki/MultiDevice-Announcements-POC) problem is solved. Fork me!

pyproject.toml

@@ -0,0 +1,48 @@
+[project]
+name = 'tox_profile'
+requires-python = ">= 3.7"
+description = "Read and manipulate tox profile files"
+keywords = ["tox", "tox_profile"]
+classifiers = [
+  # How mature is this project? Common values are
+  #   3 - Alpha
+  #   4 - Beta
+  #   5 - Production/Stable
+  "Development Status :: 4 - Beta",
+
+  # Indicate who your project is intended for
+  "Intended Audience :: Developers",
+
+  # Specify the Python versions you support here.
+  "Programming Language :: Python :: 3",
+  "License :: OSI Approved",
+  "Operating System :: POSIX :: BSD :: FreeBSD",
+  "Operating System :: POSIX :: Linux",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.7",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: Implementation :: CPython",
+]
+dynamic = ["version", "readme", "dependencies"] # cannot be dynamic ['license']
+
+[project.scripts]
+tox_profile = "tox_profile:iMain"
+
+[project.urls]
+repository = "https://git.plastiras.org/emdee/tox_profile"
+homepage = "https://git.plastiras.org/emdee/tox_profile"
+
+[build-system]
+requires = ["setuptools >= 61.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.dynamic]
+version = {attr = "tox_profile.__version__"} 
+readme = {file = ["README.md"]}
+dependencies = {file = ["requirements.txt"]}
+
+#[tool.setuptools.packages.find]
+#where = "src"

requirements.txt

@@ -0,0 +1,9 @@
+# the versions are the current ones tested - may work with earlier versions
+ruamel.yaml >= 0.18.5
+xmsgpack >= 1.0.7
+coloredlogs >= 15.0.1
+# optional
+# nmap
+# this is not on pypi yet - get it from
+# https://git.plastiras.org/emdee/toxygen_wrapper
+# toxygen_wrapper >= 1.0.0

setup.cfg

@@ -0,0 +1,55 @@
+[metadata]
+classifiers =
+    License :: OSI Approved
+    Intended Audience :: Web Developers
+    Operating System :: POSIX :: BSD :: FreeBSD
+    Operating System :: POSIX :: Linux
+    Programming Language :: Python :: 3 :: Only
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
+    Programming Language :: Python :: Implementation :: CPython
+description='Read and manipulate tox profile files'
+long_description='Read and manipulate tox profile files'
+url='https://git.plastiras.org/emdee/tox_profile/'
+keywords='ctypes Tox'
+ 
+[options]
+zip_safe = false
+python_requires = >=3.7
+include_package_data =
+    "*" = ["*.txt", "*.bash"]    
+
+[options.entry_points]
+console_scripts =
+    tox_profile = tox_profile.tox_profile:iMain
+
+[easy_install]
+zip_ok = false
+
+[flake8]
+jobs = 1
+max-line-length = 88
+ignore =
+    E111
+    E114
+    E128
+    E225
+    E261
+    E302
+    E305
+    E402
+    E501
+    E502
+    E541
+    E701
+    E702
+    E704
+    E722
+    E741
+    F508
+    F541
+    W503
+    W601

src/tox_profile/__init__.py

@@ -0,0 +1,2 @@
+
+__version__ = "1.0.0"

src/tox_profile/__main__.py

@@ -0,0 +1,5 @@
+import sys
+from tox_profile.tox_profile import iMain
+
+if __name__ == '__main__':
+    sys.exit(iMain(sys.argv[1:]))

tox_profile_examples.bash

@@ -0,0 +1,24 @@
+#!/bin/sh -e
+# -*- mode: sh; fill-column: 75; tab-width: 8; coding: utf-8-unix -*-
+
+# some examples of tox-profile usage
+
+export PYTHONPATH=/mnt/o/var/local/src/toxygen_wrapper.git
+TOX_HOME=$HOME/.config/tox
+NMAP_CMD='sudo -u debian-tor nmap'
+
+echo INFO: check the download json file
+python3 tox_profile.py --command nodes --nodes check \
+	$TOX_HOME/DHTnodes.json.new \
+	2>&1 | tee /tmp/DHTnodes.json.log
+
+echo INFO: get the tcp nodes/ports from the downloaded json file
+python3 tox_profile.py --command nodes --nodes select_tcp \
+	--output /tmp/DHTnodes.json.tcp \
+	$TOX_HOME/DHTnodes.json.new
+
+echo INFO: run ping/nmap on the tcp nodes/ports from the downloaded json file
+python3 tox_profile.py --command nodes --nodes nmap_tcp \
+	--nmap_cmd $NMAP_CMD \
+	--output /tmp/DHTnodes.json.tcp.out \
+	/tmp/DHTnodes.json.tcp

tox_profile_test.bash

@@ -1,26 +1,55 @@
 #!/bin/sh
 # -*- mode: sh; fill-column: 75; tab-width: 8; coding: utf-8-unix -*-
 
-# tox_savefile.py has a lot of features so it needs test coverage
+# tox_profile.py has a lot of features so it needs test coverage
 
-PREFIX=/o/var/local/src
+PREFIX=/mnt/o/var/local
-EXE=python3.sh
+ROLE=text
-WRAPPER=$PREFIX/toxygen_wrapper
+DEBUG=1
+EXE=/var/local/bin/python3.bash
+WRAPPER=$PREFIX/src/toxygen_wrapper.git
+tox=$HOME/.config/tox/toxic_profile.tox
+[ -s $tox ] || exit 2
+target=$PREFIX/src/tox_profile/tox_profile.py
+
+OUT=/tmp/toxic_profile
+
+ps ax | grep -q tor && netstat -n4le | grep -q :9050
+[ $? -eq 0 ] && HAVE_TOR=1 || HAVE_TOR=0
 
 [ -f /usr/local/bin/usr_local_tput.bash ] && \
     . /usr/local/bin/usr_local_tput.bash || {
-	DEBUG() { echo DEBUG $* ; }
+	DBUG() { echo DEBUG $* ; }
 	INFO() { echo INFO $* ; }
 	WARN() { echo WARN $* ; }
 	ERROR() { echo ERROR $* ; }
     }
 
-# set -- -e
+if [ -z "$TOXCORE_LIBS" ] &&  [ ! -d libs ] ; then
-target=$PREFIX/tox_profile/tox_savefile.py
+    mkdir libs
-[ -s $target ] || exit 1
+    cd libs
+    # /lib/x86_64-linux-gnu/libtoxcore.so.2
+    for pro in qtox toxic ; do
+	if which $pro 2> /dev/null ; then
+	    DBUG linking to $pro libtoxcore
+ 	    lib=$( ldd `which $pro` | grep libtoxcore|sed -e 's/.* => //' -e 's/ .*//')
+	    [ -n "$lib" -a -f "$lib" ] || { WARN $Lib ; continue ; }
+	    INFO linking to $lib
+	    for elt in libtoxcore.so libtoxav.so libtoxencryptsave.so ; do
+		ln -s "$lib" "$elt"
+	    done
+	    export TOXCORE_LIBS=$PWD
+	    break
+	fi
+    done
+    cd ..
+elif [ -z "$TOXCORE_LIBS" ] &&  [ -d libs ] ; then
+    export TOXCORE_LIBS=$PWD/libs
+fi
 
-tox=$HOME/.config/tox/toxic_profile.tox
+
-[ -s $tox ] || exit 2
+# set -- -e
+[ -s $target ] || exit 1
 
 [ -d $WRAPPER ] || {
     ERROR wrapper is required https://git.plastiras.org/emdee/toxygen_wrapper
@@ -34,7 +63,7 @@ json=$HOME/.config/tox/DHTnodes.json
 which jq > /dev/null && HAVE_JQ=1 || HAVE_JQ=0
 which nmap > /dev/null && HAVE_NMAP=1 || HAVE_NMAP=0
 
-sudo rm -f /tmp/toxic_profile.* /tmp/toxic_nodes.*
+sudo rm -f $OUT.* /tmp/toxic_nodes.*
 
 test_jq () {
     [ $# -eq 3 ] || {
@@ -66,42 +95,66 @@ test_jq () {
 
 i=0
 [ "$HAVE_JQ" = 0 ] || \
-    test_jq $json /tmp/toxic_nodes.json /tmp/toxic_nodes.err || exit ${i}$?
+    test_jq $json /tmp/toxic_nodes.json /tmp/toxic_nodes.err || {
+	ERROR test_jq failed on $json
+	exit ${i}$?
+	}
 [ -f /tmp/toxic_nodes.json ] || cp -p $json /tmp/toxic_nodes.json
 json=/tmp/toxic_nodes.json
 
 i=1
 # required password
-INFO $i decrypt /tmp/toxic_profile.bin
+INFO $i decrypt $OUT.bin
-$EXE $target --command decrypt --output /tmp/toxic_profile.bin $tox || exit ${i}1
+$EXE $target --command decrypt --output $OUT.bin $tox || exit ${i}1
-[ -s /tmp/toxic_profile.bin ] || exit ${i}2
+[ -s $OUT.bin ] || exit ${i}2
 
-tox=/tmp/toxic_profile.bin
+tox=$OUT.bin
 INFO $i info $tox
-$EXE $target --command info --info info $tox 2>/tmp/toxic_profile.info || {
+$EXE $target --command info --info info $tox 2>$OUT.info || {
     ERROR $i $EXE $target --command info --info info $tox
     exit ${i}3
 }
-[ -s /tmp/toxic_profile.info ] || exit ${i}4
+[ -s $OUT.info ] || exit ${i}4
 
-INFO $i /tmp/toxic_profile.save
+INFO $i $EXE $target --command info --info save --output $OUT.save $tox
-$EXE $target --command info --info save --output /tmp/toxic_profile.save $tox 2>/dev/null || exit ${i}5
+$EXE $target --command info --info save --output $OUT.save $tox 2>/dev/null ||  {
-[ -s /tmp/toxic_profile.save ] || exit ${i}6
+    ERROR $?
+    exit ${i}5
+}
+
+[ -s $OUT.save ] || exit ${i}6
 
 i=2
-for the_tox in $tox /tmp/toxic_profile.save ; do
+[ $# -ne 0 -a $1 -ne $i ] || \
+! INFO $i Info and editing || \
+for the_tox in $tox $OUT.save ; do
     DBUG $i $the_tox
     the_base=`echo $the_tox | sed -e 's/.save$//' -e 's/.tox$//'`
     for elt in json yaml pprint repr ; do
+	if [ $elt = yaml -o $elt = json ] ; then
+	    # ModuleNotFoundError
+	    python3 -c "import $elt" 2>/dev/null || continue
+	fi
 	INFO $i $the_base.$elt
 	DBUG $EXE $target \
 	     --command info --info $elt \
 	     --output $the_base.$elt $the_tox '2>'$the_base.$elt.err
 	$EXE $target --command info --info $elt \
-		    --output $the_base.$elt $the_tox 2>$the_base.$nmap.err || exit ${i}0
+	     --output $the_base.$elt $the_tox 2>$the_base.$elt.err || {
-       [ -s $the_base.$elt ] || exit ${i}1
+	    tail $the_base.$elt.err
+	    if [ $elt != yaml -a $elt != json ] ; then
+		exit ${i}0
+	    else
+		WARN $elt
+	    fi
+	}
+	[ -s $the_base.$elt ] || {
+	    WARN no output  $the_base.$elt
+#	    exit ${i}1
+	}
     done
 
+    DBUG     $EXE $target --command edit --edit help $the_tox
     $EXE $target --command edit --edit help $the_tox 2>/dev/null  || exit ${i}2
 
     # edit the status message
@@ -128,11 +181,13 @@ for the_tox in $tox /tmp/toxic_profile.save ; do
 done
 
 i=3
+[ "$#" -ne 0 -a "$1" != "$i" ] || \
 [ "$HAVE_JQ" = 0 ] || \
+! INFO $i Nodes || \
 for the_json in $json ; do
     DBUG $i $the_json
     the_base=`echo $the_json | sed -e 's/.json$//' -e 's/.tox$//'`
-    for nmap in select_tcp select_udp select_version ; do
+    for nmap in clean check select_tcp select_udp select_version; do
 	$EXE $target --command nodes --nodes $nmap \
 	             --output $the_base.$nmap.json $the_json || {
             WARN $i $the_json $nmap ${i}1
@@ -160,43 +215,89 @@ for the_json in $json ; do
     done
 done
 
-ls -l /tmp/toxic_profile.* /tmp/toxic_nodes.*
+i=4
+## 
+[ $# -ne 0 -a "$1" -ne $i ] || \
+[ "$HAVE_TOR" = 0 ] || \
+[ ! -f /etc/tor/torrc ] || \
+! INFO $i Onions || \
+for the_tox in /etc/tor/torrc ; do
+    DBUG $i $the_tox
+    the_base=`echo $OUT.save | sed -e 's/.save$//' -e 's/.tox$//'`
+    #  exits
+    for slot in config test; do
+        if [ $slot = exits ] && ! netstat -nle4 | grep -q :9050 ; then
+           WARN Tor not running
+           continue
+        fi
+        INFO $target --command onions --onions $slot \
+             --output $the_base.$slot.out $the_tox
+        DBUG=1 $EXE $target --command onions --onions $slot \
+             --log_level 10 \
+             --output $the_base.$slot.out $the_tox|| {
+             WARN $i $?
+             continue
+        }
+	[ true -o -s $the_base.$slot.out ] || {
+	    WARN $i empty $the_base.$slot.out
+	    continue
+	}
+      done
+  done
+
+# ls -l $OUT.* /tmp/toxic_nodes.*
 
 # DEBUG=0 /usr/local/bin/proxy_ping_test.bash tor || exit 0
 ip route | grep ^def || exit 0
 
-i=4
+i=5
+##
 the_tox=$tox
+[ $# -ne 0 -a "$1" != "$i" ] || \
 [ "$HAVE_JQ" = 0 ] || \
 [ "$HAVE_NMAP" = 0 ] || \
-for the_tox in $tox /tmp/toxic_profile.save ; do
+! INFO $i Making dogfood || \
+for the_tox in $tox $OUT.save ; do
     DBUG $i $the_tox
     the_base=`echo $the_tox | sed -e 's/.save$//' -e 's/.tox$//'`
-    for nmap in nmap_tcp nmap_udp nmap_onion ; do
+    for nmap in nmap_relay nmap_dht nmap_path ; do
 #	[ $nmap = select_tcp ] && continue
-#	[ $nmap = select_udp ] && continue
+	if [ $nmap = nmap_dht ] && [ $HAVE_TOR = 1 ] ; then
+	    INFO skipping $nmap because HAVE_TOR
+	    continue
+	fi
         INFO $i $the_base.$nmap
+	DBUG $target --command info --info $nmap \
+	     --output $the_base.$nmap.out $the_tox
         $EXE $target --command info --info $nmap \
 	     --output $the_base.$nmap.out $the_tox 2>$the_base.$nmap.err || {
 	    # select_tcp may be empty and jq errors
 	    # exit ${i}1
-	    WARN $i $the_base.$nmap.err
+	    WARN $i $? $the_base.$nmap.err
+	    tail $the_base.$nmap.err
 	    continue
 	}
 	[ -s $the_base.$nmap.out ] || {
-	    ERROR $i $the_base.$nmap.out
+	    WARN $i empty $the_base.$nmap.out
 	    continue
 	}
     done
 done
 
-i=5
+i=6
+##
+[ $# -ne 0 -a "$1" != "$i" ] || \
 [ "$HAVE_JQ" = 0 ] || \
+! INFO $i Eating dogfood || \
 for the_json in $json ; do
     DBUG $i $the_json
     the_base=`echo $the_json | sed -e 's/.save$//' -e 's/.json$//'`
     for nmap in nmap_tcp nmap_udp ; do
-        INFO $i $the_base.$nmap 
+	if [ $nmap = nmap_udp ] && [ $HAVE_TOR = 1 ] ; then
+	    INFO skipping $nmap because HAVE_TOR
+	    continue
+	fi
+        INFO $i $target --command nodes --nodes $nmap --output $the_base.$nmap
         $EXE $target --command nodes --nodes $nmap \
 	     --output $the_base.$nmap $the_json 2>$the_base.$nmap.err || {
             WARN $i $the_json $nmap ${i}1
@@ -209,7 +310,7 @@ for the_json in $json ; do
     done
 done
 
-i=6
+i=7
 DBUG $i
 $EXE $target --command nodes --nodes download \
      --output /tmp/toxic_nodes.new $json || {
@@ -217,16 +318,23 @@ $EXE $target --command nodes --nodes download \
 	exit ${i}1
 }
 [ -s /tmp/toxic_nodes.new ] || exit ${i}4
+INFO $i  downloaded  /tmp/toxic_nodes.new
 json=/tmp/toxic_nodes.new
+[ $# -ne 0 -a "$1" != "$i" ] || \
 	[ "$HAVE_JQ" = 0 ] || \
-jq . < $json >/tmp/toxic_nodes.new.json 2>>/tmp/toxic_nodes.new.err || {
+	jq . < $json >/tmp/toxic_nodes.new.json 2>>/tmp/toxic_nodes.new.json.err || {
 	    ERROR $i jq $json
 	    exit ${i}2
 	}
+INFO $i  jq from /tmp/toxic_nodes.new.json
+
+[ $# -ne 0 -a "$1" != "$i" ] || \
 	[ "$HAVE_JQ" = 0 ] || \
-grep error: /tmp/toxic_nodes.new.err && {
+	grep error: /tmp/toxic_nodes.new.json.err && {
 	    ERROR $i jq $json
 	    exit ${i}3
 	}
+INFO $i  no errors in  /tmp/toxic_nodes.new.err
+
 
 exit 0