update

2024-02-21 08:03:10 +00:00 · 2024-02-05 14:26:19 +00:00 · 2024-02-04 03:29:52 +00:00 · 2024-01-17 20:35:20 +00:00 · 2024-01-17 16:30:48 +00:00 · 2024-01-16 18:10:43 +00:00
22 changed files with 1005 additions and 271 deletions
--- a/.gitignore
+++ b/.gitignore
@ -4,6 +4,10 @@ __pycache__/
 *.py[cod]
 *$py.class

+#*
+*~
+*.dst
+
 # C extensions
 *.so

--- a/54
+++ b/54
@ -1,12 +1,60 @@
-LOCAL_DOCTEST=/usr/local/bin/toxcore_run_doctest3.bash
+# to run the tests, run make PASS=controllerpassword test
+
+PREFIX=/usr/local
+PYTHON_EXE_MSYS=${PREFIX}/bin/python3.sh
+PIP_EXE_MSYS=${PREFIX}/bin/pip3.sh
+LOCAL_DOCTEST=${PREFIX}/bin/toxcore_run_doctest3.bash
 DOCTEST=${LOCAL_DOCTEST}
+MOD=stem_examples
+
+check::
+	sh python3.sh -c "import ${MOD}"

 lint::
 	sh .pylint.sh

+install::
+	${PIP_EXE_MSYS} --python ${PYTHON_EXE_MSYS} install \
+		--no-deps \
+		--target ${PREFIX}/lib/python${PYTHON_MINOR}/site-packages/ \
+		--upgrade .
+	sed -i -e "1s@/usr/bin/python${PYTHON_MINOR}@${PYTHON_EXE_MSYS}@" \
+		${PREFIX}/lib/python${PYTHON_MINOR}/site-packages/bin/*
+
 rsync::
 	bash .rsync.sh

+# execute these tests as: make test PASS=password
+test::
+	echo src/${MOD}/check_digests.py
+	TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/check_digests.py
+	echo src/${MOD}/interpreter.py
+	TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/interpreter.py
+	echo src/${MOD}/connection_resolution.py
+	sudo env TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/connection_resolution.py
+# broken because this site fails: http://128.31.0.39:9131/tor/status-vote
+#	${PYTHON_EXE_MSYS} src/${MOD}/compare_flags.py
+# cant use from make: waits for the cmdline to to terminate
+#	TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/exit_used.py 10
+	echo src/${MOD}/introduction_points.py
+	TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/introduction_points.py
+	echo src/${MOD}/list_circuits.py
+	TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/list_circuits.py 
+	echo src/${MOD}/mappaddress.py
+	TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/mappaddress.py
+	echo src/${MOD}/outdated_relays.py NOT WORKING?
+	TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/outdated_relays.py
+	echo src/${MOD}/relay_connections.py
+	sudo env TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/relay_connections.py
+	echo src/${MOD}/tor_bootstrap_check.py
+	TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/tor_bootstrap_check.py
+
 doctest:
-	export PYTHONPATH=${PWD}
-	${DOCTEST} stem_examples.txt
+	sudo -u tor env PYTHONPATH=${PWD}/src \
+	${DOCTEST} ${MOD}.txt
+
+veryclean:: clean
+	rm -rf build dist __pycache__ .pylint.err .pylint.out
+
+clean::
+	find . -name \*~ -delete
--- a/README.md
+++ b/README.md
@ -1,6 +1,114 @@
-# stem_examples
+## stem_examples
+
+Examples of using the Python stem library to query the state of a running tor.
+
+You can set TOR_CONTROLLER_PASSWORD in the environment if your tor control port
+requires a password.
+
+* check_digests
+* compare_flags
+* exit_used
+* introduction_points
+* list_circuits
+* mappaddress
+* outdated_relays
+* relay_connections
+* tor_bootstrap_check
+
+### check_digests
+
+Checking Descriptor Digests
+
+Tor relay information is provided by multiple documents. Signed
+descriptors transitively validate others by inclusion of their
+digest. For example, our consensus references server descriptor
+digest, and server descriptors in turn cite extrainfo digests.
+
+Stem can calculate digests from server, extrainfo, microdescriptor, and consensus documents. For instance, to validate an extrainfo descriptor...
+
+https://stem.torproject.org/tutorials/examples/check_digests.html
+
+### compare_flags Comparing Directory Authority Flags
+
+Compares the votes of two directory authorities, in this case moria1
+and maatuska, with a special interest in the 'Running' flag.
+
+https://stem.torproject.org/tutorials/examples/compare_flags.html
+
+### connection_resolution Connection Resolution
+
+Connection information is a useful tool for learning more about network
+applications like Tor. Our stem.util.connection.get_connections() function
+provides an easy method for accessing this information.
+
+
+### exit_used Exit Used
+
+Determine The Exit You're Using

-stem examples
 https://stem.torproject.org/tutorials/examples/exit_used.html
-http://vt5hknv6sblkgf22.onion/tutorials/examples/list_circuits.html
-http://vt5hknv6sblkgf22.onion/tutorials/examples/relay_connections.html
+
+### introduction_points Introduction Points
+
+This script tests if you can reach a hidden service, passed as an onion address
+as an argument. If no argument is given, 3 common onion sites are tested:
+Facebook, DuckDuckGo, and .
+
+https://stem.torproject.org/tutorials/over_the_river.html
+
+### list_circuits List Circuits
+
+Tor creates new circuits and tears down old ones on your behalf, so
+how can you get information about circuits Tor currently has available?
+https://stem.torproject.org/tutorials/examples/list_circuits.html
+
+### mappaddress
+
+Mappaddress queries the socks proxy with an onion address and returns the
+IP address that it will use for it. the address will be in the block specified
+by the VirtualAddrNetworkIPv4  setting of the torrc, e.g.
+
+VirtualAddrNetworkIPv4 172.16.0.0/12
+
+### outdated_relays List Outdated Relays
+
+Time marches on. Tor makes new releases, and at some point needs to
+drop support for old ones. Below is the script we used on ticket 9476
+to reach out to relay operators that needed to upgrade.
+
+https://stem.torproject.org/tutorials/examples/outdated_relays.html
+
+### relay_connections Connection Summary
+
+The following provides a summary of your relay's inbound and outbound connections.
+
+To use this you must set DisableDebuggerAttachment 0 in your
+torrc. Otherwise connection information will be unavailable.
+
+https://stem.torproject.org/tutorials/examples/relay_connections.html
+
+### Download Tor Descriptors
+
+Tor relays provide a mirror for the tor relay descriptors it has
+cached. These are available from its ORPort using Tor's wire protocol,
+and optionally with http as well from a DirPort.
+
+https://stem.torproject.org/tutorials/examples/download_descriptor.html
+
+### Votes by Bandwidth Authorities
+
+Tor takes into account a relay's throughput when picking a route through the Tor network for its circuits. That is to say large, fast relays receive more traffic than small ones since they can better service the load.
+
+To determine a relay's throughput special authorities, called bandwidth authorities,
+take periodic measurements using them. The lifecycle of new Tor relays
+is a bit more complicated than that, butthat's the general idea.
+
+Bandwidth authorities include their measurements in their votes. The following
+gets their current votes then prints how many relays it had a measurement for.
+
+https://stem.torproject.org/tutorials/examples/votes_by_bandwidth_authorities.html
+
+## tor_bootstrap_check
+
+A script by adrelanos@riseup.net to check what percentage of boostrapping
+tor is at.
--- a/pyproject.toml
+++ b/pyproject.toml
@ -1,12 +1,8 @@
 [project]
 name = "stem_examples"
-version = "2023.12"
 description = "examples of using stem"
-authors = [{ name = "emdee", email = "Ingvar@gitgub.com" } ]
+authors = [{ name = "emdee", email = "emdee@spm.plastiras.org" } ]
 requires-python = ">=3.6"
-dependencies = [
-    'stem',
-]
 keywords = ["stem", "python3", "tor"]
 classifiers = [
    "License :: OSI Approved",
@ -22,7 +18,18 @@ classifiers = [
    "Programming Language :: Python :: Implementation :: CPython",
    ]
 dynamic = ["version", "readme", ] # cannot be dynamic ['license']
-scripts = { exclude_badExits = "stem_examples.exclude_badExits:iMain" }
+
+[project.scripts]
+check_digests = "stem_examples.check_digests:iMain"
+compare_flags = "stem_examples.compare_flags:iMain"
+exit_used = "stem_examples.exit_used:iMain"
+introduction_points = "stem_examples.introduction_points:iMain"
+list_circuits = "stem_examples.list_circuits:iMain"
+mappaddress = "stem_examples.mappaddress:iMain"
+outdated_relays = "stem_examples.outdated_relays:iMain"
+relay_connections = "stem_examples.relay_connections:iMain"
+tor_bootstrap_check = "stem_examples.tor_bootstrap_check:iMain"
+torcontactinfo = "stem_examples.torcontactinfo:iMain"

 #[project.license]
 #file = "LICENSE.md"
@ -36,10 +43,5 @@ build-backend = "setuptools.build_meta"

 [tool.setuptools.dynamic]
 version = {attr = "stem_examples.__version__"} 
-readme = {file = ["README.md", "exclude_badExits.md"]}
+readme = {file = ["README.md", "stem_examples.txt"]}

-[tool.setuptools]
-packages = ["stem_examples"]
-
-#[tool.setuptools.packages.find]
-#where = "src"
--- a/setup.cfg
+++ b/setup.cfg
@ -19,11 +19,9 @@ classifiers =
 [options]
 zip_safe = false
 python_requires = ~=3.6
-include_package_data = false
+include_package_data =
+    "*" = ["*.txt", "*.bash" ]
 install_requires =
-    qasync
-    cryptography
-    rsa
    stem
    ruamel.yaml
 package_dir=
@ -34,9 +32,8 @@ packages=find:
 where=src

 [options.entry_points]
-console_scripts =
-    phantompy = phantompy.__main__:iMain
-    exclude_badExits = exclude_badExits:iMain
+#console_scripts =
+#    exclude_badExits = exclude_badExits:iMain

 [easy_install]
 zip_ok = false
--- a/src/stem_examples/Makefile
+++ b/src/stem_examples/Makefile
@ -0,0 +1,24 @@
+PYTHON_EXE_MSYS=/usr/local/bin/python3.sh
+PYLINT_EXE_MSYS=/usr/local/bin/toxcore_pylint3.bash
+
+lint.phantompy::
+	${PYLINT_EXE_MSYS} lookupdns.py qasync_phantompy.py phantompy.py support_phantompy.py 
+
+lint.badexits::
+	${PYLINT_EXE_MSYS} exclude_badExits.py  \
+		support_onions.py trustor_poc.py 
+	isort -c -diff exclude_badExits.py \
+		support_onions.py trustor_poc.py 
+
+lint::
+	sh .pylint.sh
+
+refresh:: ../../exclude_badExits.md
+
+../../exclude_badExits.md::
+	${PYTHON_EXE_MSYS} -c \
+		'import exclude_badExits; print(exclude_badExits.__doc__)' > $@
+	echo "\n## Usage \n\`\`\`\n" >> $@
+	${PYTHON_EXE_MSYS} exclude_badExits.py --help \
+		| sed -e '/^[^uo ]/d' >> $@
+	echo "\n\`\`\`\n" >> $@
--- a/src/stem_examples/init.py
+++ b/src/stem_examples/init.py
@ -0,0 +1,2 @@
+
+__version__ = "1.0.0"
--- a/src/stem_examples/check_digests.py
+++ b/src/stem_examples/check_digests.py
@ -1,11 +1,41 @@
+#!/usr/local/bin/python3.sh
 # -*-mode: python; py-indent-offset: 2; indent-tabs-mode: nil; coding: utf-8-unix -*-

-# http://vt5hknv6sblkgf22.onion/tutorials/examples/check_digests.html
-import sys
+__doc__ = """
+Checking Descriptor Digests

-import stem.descriptor.remote
-import stem.util.tor_tools
-from tor_controller import set_socks_proxy
+Tor relay information is provided by multiple documents. Signed descriptors transitively validate others by inclusion of their digest. For example, our consensus references server descriptor digest, and server descriptors in turn cite extrainfo digests.
+
+Stem can calculate digests from server, extrainfo, microdescriptor, and
+consensus documents. For instance, to validate an extrainfo descriptor...
+
+https://stem.torproject.org/tutorials/examples/check_digests.html
+"""
+
+import os
+import sys
+import contextlib
+import logging
+
+import stem
+from stem_examples.tor_controller import set_socks_proxy
+
+LOG = logging.getLogger()
+sKNOWN_ONION = 'facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd' # facebook
+iTIMEOUT = 120
+
+@contextlib.contextmanager
+def ignoreStdout() -> None:
+    devnull = os.open(os.devnull, os.O_WRONLY)
+    old_stdout = os.dup(1)
+    sys.stdout.flush()
+    os.dup2(devnull, 1)
+    os.close(devnull)
+    try:
+        yield
+    finally:
+        os.dup2(old_stdout, 1)
+        os.close(old_stdout)

 def download_descriptors(fingerprint):
  """
@ -30,27 +60,67 @@ def download_descriptors(fingerprint):
    extrainfo_query.run()[0],
  )

+def iMain(lArgs=None):
+  global LOG
+#  set_socks_proxy()
+  iRetval = 0
+  if lArgs is None:
+      fingerprint = input("What relay fingerprint would you like to validate?\n")
+      print('')  # blank line
+      lArgs = [fingerprint]
+
+  for fingerprint in  lArgs:
+    LOG.info(f"checking digests of fp={fp}")
+    if not stem.util.tor_tools.is_valid_fingerprint(fingerprint):
+      LOG.error("'%s' is not a valid relay fingerprint" % fingerprint)
+      iRetval += 1
+      continue
+
+    try:
+      router_status_entry, server_desc, extrainfo_desc = download_descriptors(fingerprint)
+    except Exception as exc:
+      LOG.exception(f"Exception in download_descriptors {exc}")
+      iRetval += 1
+      continue
+
+    if router_status_entry.digest == server_desc.digest():
+      LOG.info("Server descriptor digest is correct")
+    else:
+      LOG.warn("Server descriptor digest invalid, expected %s but is %s" % (
+        router_status_entry.digest, server_desc.digest()))
+
+    if server_desc.extra_info_digest == extrainfo_desc.digest():
+      LOG.info("Extrainfo descriptor digest is correct")
+    else:
+      LOG.warn("Extrainfo descriptor digest invalid, expected %s but is %s" % (
+        server_desc.extra_info_digest, extrainfo_desc.digest()))
+  return iRetval
+
 if __name__ == '__main__':
-  set_socks_proxy()
-  fingerprint = input("What relay fingerprint would you like to validate?\n")
-  print('')  # blank line
-
-  if not stem.util.tor_tools.is_valid_fingerprint(fingerprint):
-    print("'%s' is not a valid relay fingerprint" % fingerprint)
-    sys.exit(1)
-
-  try:
-    router_status_entry, server_desc, extrainfo_desc = download_descriptors(fingerprint)
-  except Exception as exc:
-    print(exc)
-    sys.exit(1)
-
-  if router_status_entry.digest == server_desc.digest():
-    print("Server descriptor digest is correct")
-  else:
-    print("Server descriptor digest invalid, expected %s but is %s" % (router_status_entry.digest, server_desc.digest()))
-
-  if server_desc.extra_info_digest == extrainfo_desc.digest():
-    print("Extrainfo descriptor digest is correct")
-  else:
-    print("Extrainfo descriptor digest invalid, expected %s but is %s" % (server_desc.extra_info_digest, extrainfo_desc.digest()))
+    from stem_examples.stem_utils import vsetup_logging
+    if os.environ.get('DEBUG', ''):
+        log_level = 10
+        logging.getLogger('stem').setLevel(20)
+    else:
+        log_level = 20
+        logging.getLogger('stem').setLevel(30)
+    vsetup_logging(LOG, log_level)
+    try:
+        if len(sys.argv) > 1:
+            lArgs = sys.argv[1:]
+            LOG.info(f"Getting some {len(lArgs)}")
+        else:
+            LOG.info(f"Getting some FPs from the IPs to a sKNOWN_ONION TIMEOUT={iTIMEOUT}")
+            from stem_examples.introduction_points import lMain as lIPMain
+            with ignoreStdout():
+                lArgs = lIPMain([sKNOWN_ONION], timeout=iTIMEOUT)
+            LOG.info(f"Got {len(lArgs)} FPs from a sKNOWN_ONION")
+            if not lArgs:
+                sys.exit(1)
+        i = iMain(lArgs)
+    except KeyboardInterrupt as e:
+        i = 0
+    except Exception as e:
+        LOG.exception(f"Exception in iMain {e}")
+        i = 1
+    sys.exit(i)
--- a/src/stem_examples/compare_flags.py
+++ b/src/stem_examples/compare_flags.py
@ -1,5 +1,15 @@
+#!/usr/local/bin/python3.sh
 # -*-mode: python; py-indent-offset: 2; indent-tabs-mode: nil; coding: utf-8-unix -*-
 #
+__doc__ = """
+Compares the votes of two directory authorities, in this case moria1
+and maatuska, with a special interest in the 'Running' flag.
+
+broken because this site fails:
+http://128.31.0.39:9131/tor/status-vote
+
+https://stem.torproject.org/tutorials/examples/compare_flags.html
+"""

 import sys
 import collections
@ -8,7 +18,7 @@ import stem.descriptor
 import stem.descriptor.remote
 import stem.directory

-from tor_controller import set_socks_proxy
+from tor_controller import set_socks_proxy, unset_socks_proxy

 def iMain():
    # Query all authority votes asynchronously.
@ -60,4 +70,10 @@ def iMain():

 if __name__ == '__main__':
    set_socks_proxy()
-    sys.exit(iMain())
+    try:
+        i = iMain()
+    except KeyboardInterrupt as e:
+        i = 0
+    finally:
+        unset_socks_proxy()
+    sys.exit(i)
--- a/src/stem_examples/connection_resolution.py
+++ b/src/stem_examples/connection_resolution.py
@ -0,0 +1,77 @@
+__doc__ = """Connection Resolution
+
+Connection information is a useful tool for learning more about network
+applications like Tor. Our stem.util.connection.get_connections() function
+provides an easy method for accessing this information, with a few caveats...
+
+Connection resolvers are platform specific. We support several platforms but
+not all.
+
+By default Tor runs with a feature called DisableDebuggerAttachment. This
+prevents debugging applications like gdb from analyzing Tor unless it is run as
+root. Unfortunately this also alters the permissions of the Tor process /proc
+contents breaking numerous system tools (including our resolvers). To use this
+function you need to either run as root (discouraged) or add
+DisableDebuggerAttachment 0 to your torrc.
+
+Please note that if you operate an exit relay it is highly discouraged for you
+to look at or record this information. Not only is doing so eavesdropping, but
+likely also a violation of wiretap laws.
+
+With that out of the way, how do you look up this information? Below is a
+simple script that dumps Tor's present connections.
+
+https://stem.torproject.org/tutorials/east_of_the_sun.html
+"""
+
+import os
+import sys
+import logging
+
+from stem.util.connection import get_connections, system_resolvers
+from stem.util.system import pid_by_name
+
+LOG = logging.getLogger()
+
+def iMain (lArgs=None):
+    resolvers = system_resolvers()
+
+    if not resolvers:
+        LOG.error("Stem doesn't support any connection resolvers on our platform.")
+        return 1
+
+    picked_resolver = resolvers[0]  # lets just opt for the first
+    LOG.info("Our platform supports connection resolution via: %s (picked %s)" % (', '.join(resolvers), picked_resolver))
+
+    tor_pids = pid_by_name('tor', multiple = True)
+
+    if not tor_pids:
+        LOG.warn("Unable to get tor's pid. Is it running?")
+        return 1
+
+    if len(tor_pids) > 1:
+        LOG.info("You're running %i instances of tor, picking the one with pid %i" % (len(tor_pids), tor_pids[0]))
+    else:
+        LOG.info("Tor is running with pid %i" % tor_pids[0])
+
+    LOG.info("Connections:\n")
+
+    for conn in get_connections(picked_resolver, process_pid = tor_pids[0], process_name = 'tor'):
+        LOG.info("  %s:%s => %s:%s" % (conn.local_address, conn.local_port, conn.remote_address, conn.remote_port))
+    return 0
+
+if __name__ == '__main__':
+    from stem_examples.stem_utils import vsetup_logging
+    if os.environ.get('DEBUG', ''):
+        log_level = 10
+    else:
+        log_level = 20
+    vsetup_logging(LOG, log_level)
+    try:
+        i = iMain(sys.argv[1:])
+    except KeyboardInterrupt as e:
+        i = 0
+    except Exception as e:
+        LOG.exception(f"Exception {e}")
+        i = 1
+    sys.exit(i)
--- a/src/stem_examples/exit_used.py
+++ b/src/stem_examples/exit_used.py
@ -1,3 +1,4 @@
+#!/usr/local/bin/python3.sh
 # -*-mode: python; py-indent-offset: 2; indent-tabs-mode: nil; coding: utf-8-unix -*-
 # https://stem.torproject.org/tutorials/examples/exit_used.html
 __doc__ = """Determine The Exit You're Using
@ -11,43 +12,74 @@ How can you figure out what exit you're using?
 import functools
 import sys
 import os
+import logging

 from stem import StreamStatus
 from stem.control import EventType, Controller
-from tor_controller import get_controller
+from stem_examples.tor_controller import get_controller
+
+global COUNT, IMAX
+COUNT=0
+global IMAX
+IMAX = 10
+LOG = logging.getLogger()

 def stream_event(controller, event):
+    global COUNT, IMAX
+    COUNT += 1
+    if IMAX and COUNT >= IMAX:
+        LOG.info(f"exiting COUNT={COUNT}")
+        sys.exit(0)
    if event.status == StreamStatus.SUCCEEDED and event.circ_id:
      circ = controller.get_circuit(event.circ_id)

      exit_fingerprint = circ.path[-1][0]
      exit_relay = controller.get_network_status(exit_fingerprint)

-      print("Exit relay for our connection to %s" % (event.target))
-      print("  address: %s:%i" % (exit_relay.address, exit_relay.or_port))
-      print("  fingerprint: %s" % exit_relay.fingerprint)
-      print("  nickname: %s" % exit_relay.nickname)
-      print("  locale: %s" % controller.get_info("ip-to-country/%s" % exit_relay.address, 'unknown'))
-      print("")
+      LOG.info("Exit relay for our connection to %s" % (event.target))
+      LOG.info("  address: %s:%i" % (exit_relay.address, exit_relay.or_port))
+      LOG.info("  fingerprint: %s" % exit_relay.fingerprint)
+      LOG.info("  nickname: %s" % exit_relay.nickname)
+      LOG.info("  locale: %s" % controller.get_info("ip-to-country/%s" % exit_relay.address, 'unknown'))
+      LOG.info("")

-from tor_controller import get_controller
-
-def iMain():
-    print("Please wait for requests for tor exits. Press 'enter' to end.")
-    print("")
-    if os.path.exists('/run/tor/control'):
-      controller = get_controller(unix='/run/tor/control')
-    else:
-      controller = get_controller(port=9051)

+def iMain(lArgs=None):
    password = os.environ.get('TOR_CONTROLLER_PASSWORD')
-    controller.authenticate(password)

+    if os.path.exists('/run/tor/control'):
+      controller = get_controller(password=password, unix='/run/tor/control')
+    else:
+      controller = get_controller(password=password, port=9051)
+
+    if IMAX <= 0:
+        LOG.info("Please wait for requests for tor exits. Press 'enter' to end.")
+        print("")
    stream_listener = functools.partial(stream_event, controller)
    controller.add_event_listener(stream_listener, EventType.STREAM)

 if __name__ == '__main__':
-    iMain()
-    print('Press Enter')
-    input()  # wait for user to press enter
+    from stem_examples.stem_utils import vsetup_logging
+    if len(sys.argv) > 1:
+        IMAX = int(sys.argv[1])
+    else:
+        IMAX = 0
+    if os.environ.get('DEBUG', ''):
+        log_level = 10
+    else:
+        log_level = 20
+    vsetup_logging(LOG, log_level)
+    if len(sys.argv) > 1:
+        IMAX = int(sys.argv[1])
+        del sys.argv[1]
+    try:
+        iMain()
+        input()
+        i = 0
+    except KeyboardInterrupt as e:
+        i = 0
+    except Exception as e:
+        LOG.exception(f"Exception {e}", exc_info=True)
+        i = 1
+    sys.exit(i)

--- a/src/stem_examples/interpreter.py
+++ b/src/stem_examples/interpreter.py
@ -0,0 +1,57 @@
+#!/usr/local/bin/python3.sh
+# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*-
+__doc__ = """List Outdated Relays
+
+Time marches on. Tor makes new releases, and at some point needs to drop
+support for old ones. Below is the script we used on ticket 9476 to reach out
+to relay operators that needed to upgrade.
+
+https://stem.torproject.org/tutorials/examples/outdated_relays.html
+"""
+
+import os
+import sys
+import logging
+
+import stem.interpreter.commands
+from stem.descriptor.remote import DescriptorDownloader
+from stem_examples.tor_controller import get_controller
+from stem.version import Version
+
+from tor_controller import set_socks_proxy, unset_socks_proxy
+
+LOG = logging.getLogger()
+
+def iMain(lArgs=None):
+    if not lArgs:
+        lArgs = ['GETINFO' 'version']
+    password = os.environ.get('TOR_CONTROLLER_PASSWORD', '')
+    if os.path.exists('/run/tor/control'):
+        controller = get_controller(password=password, unix='/run/tor/control')
+    else:
+        controller = get_controller(password=password, port=9051)
+    interpreter = stem.interpreter.commands.ControlInterpreter(controller)
+    run_cmd = ' '.join(lArgs)
+    interpreter.run_command(run_cmd, print_response = True)
+    return 0
+
+if __name__ == '__main__':
+#    set_socks_proxy()
+    from stem_examples.stem_utils import vsetup_logging
+    if os.environ.get('DEBUG', ''):
+        log_level = 10
+    else:
+        log_level = 20
+    vsetup_logging(LOG, log_level)
+    try:
+        l = iMain(sys.argv[1:])
+        if l: print(l)
+        i = 0
+    except KeyboardInterrupt as e:
+        i = 0
+    except Exception as e:
+        LOG.exception(f"Exception {e}")
+        i = 1
+    finally:
+        unset_socks_proxy()
+    sys.exit(i)
--- a/src/stem_examples/introduction_points.py
+++ b/src/stem_examples/introduction_points.py
@ -13,60 +13,122 @@ Tor2web provides a quick and easy way of seeing if your hidden service is workin

 This script tests if you can reach a hidden service, passed as an onion address
 as an argument. If no argument is given, 3 common onion sites are tested:
-Facebook, DuckDuckGo, and .
+Facebook, DuckDuckGo.
+
+
 """

 import sys
 import os
 import getpass
+import logging
+import binascii

+import stem
 from stem.control import Controller
+from stem import Timeout
+from stem.client.datatype import LinkByFingerprint
+from stem.descriptor.hidden_service import HiddenServiceDescriptorV3

-from tor_controller import get_controller
+from stem_examples.tor_controller import get_controller
+from stem.descriptor.hidden_service import HiddenServiceDescriptorV3

+LOG = logging.getLogger()
+TIMEOUT = 60
 lKNOWN_ONIONS = [
  'facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd', # facebook
  'duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad', # ddg
-  'zkaan2xfbuxia2wpf7ofnkbz6r5zdbbvxbunvp5g2iebopbfc4iqmbad', # hks
+  'zkaan2xfbuxia2wpf7ofnkbz6r5zdbbvxbunvp5g2iebopbfc4iqmbad', # keys.openpgp.org
+  'libera75jm6of4wxpxt4aynol3xjmbtxgfyjpu34ss4d7r7q2v5zrpyd',
+  'oftcnet6xg6roj6d7id4y4cu6dchysacqj2ldgea73qzdagufflqxrid',
 ]

+def bin_to_hex(raw_id:int, length: int|None = None) -> str:
+    if length is None: length = len(raw_id)
+    res = ''.join('{:02x}'.format(raw_id[i]) for i in range(length))
+    return res.upper()

 def iMain(lArgs=None):
-    if lArgs is None:
-        lArgs = sys.argv[1:]
-    try:
-        if os.path.exists('/run/tor/control'):
-            controller = get_controller(unix='/run/tor/control')
-        else:
-            controller = get_controller(port=9051)
-
-        password = os.environ.get('TOR_CONTROLLER_PASSWORD')
-        controller.authenticate(password)
-
-        for elt in lArgs:
-            desc = controller.get_hidden_service_descriptor(elt, await_result=True, timeout=None)
-            print(f"{desc} get_hidden_service_descriptor\n")
-            l = desc.introduction_points()
-            if l:
-                print(f"{elt} NO introduction points\n")
-                continue
-            print(f"{elt} introduction points are...\n")
-
-            for introduction_point in l:
-                print('  %s:%s => %s' % (introduction_point.address,
-                                        introduction_point.port,
-                                        introduction_point.identifier))
-
-    except Exception as e:
-        print(e)
-    finally:
-        del controller
+    lRetval = lMain(lArgs)
+    if lRetval is None:
+        return -1
    return 0

-if __name__ == '__main__':
-    if len(sys.argv) <= 1:
+def lMain(lArgs=None, timeout=TIMEOUT) -> list:
+    lRetval = []
+    if not lArgs:
        lArgs = lKNOWN_ONIONS
-    else:
-        lArgs = sys.argv[1:]
-    sys.exit(iMain())
+    try:
+        password = os.environ.get('TOR_CONTROLLER_PASSWORD', '')
+        if os.path.exists('/run/tor/control'):
+            controller = get_controller(password=password, unix='/run/tor/control')
+        else:
+            controller = get_controller(password=password, port=9051)

+        for elt in lArgs:
+            LOG.info(f"onion: {elt}")
+            try:
+                desc = controller.get_hidden_service_descriptor(elt,
+                                                                await_result=True,
+                                                                timeout=timeout)
+            except Exception as e:
+                LOG.warn (f"{elt} EXCEPTION {e}")
+                continue
+            if desc.descriptor_id is None or desc.version is None:
+                # reparse as HSv3
+                inner_layer = HiddenServiceDescriptorV3.from_str(str(desc)).decrypt(elt)
+                if hasattr(inner_layer, 'introduction_points'):
+                    LOG.info (f"{elt} reparsed desc.decrypt len={len(desc.introduction_points())}")
+
+                    l = inner_layer.introduction_points
+                else:
+                    LOG.warn (f"{elt} reparsed desc.decrypt={dir(inner_layer)}")
+                    sys.exit(1)
+
+                #LOG.info(f"version: {desc.version}\n")
+                #LOG.info(f"lifetime: {desc.lifetime}\n")
+            else:
+                LOG.info(f"published: {desc.published}\n")
+                l = desc.introduction_points()
+            if not l:
+                LOG.warn(f"{elt} NO introduction points {l}\n")
+                continue
+
+            lp = []
+            for introduction_point in l:
+                for linkspecifier in introduction_point.link_specifiers:
+                    # if isinstance(linkspecifier, LinkByFingerprint):
+                        # LOG.log(40, f"Getting fingerprint for {linkspecifier}")
+                    if hasattr(linkspecifier, 'fingerprint') and \
+                          len(linkspecifier.value) == 20:
+                            lp += [bin_to_hex(linkspecifier.value)]
+                    elif hasattr(introduction_point, 'address'):
+                        LOG.info('IP:  %s:%s => %s' % (introduction_point.address,
+                                                introduction_point.port,
+                                                introduction_point.identifier))
+                    else:
+                        pass # LOG.warn(f"{elt} introduction_point type={type(linkspecifier)}")
+            LOG.info(f"{elt} {len(lp)} introduction points {lp}")
+    except Exception as e:
+        LOG.exception(f"Exception: {e}")
+    finally:
+        del controller
+    return lRetval
+
+if __name__ == '__main__':
+    from stem_examples.stem_utils import vsetup_logging
+    if os.environ.get('DEBUG', ''):
+        log_level = 10
+    else:
+        log_level = 20
+    vsetup_logging(LOG, log_level)
+    try:
+        l = lMain(sys.argv[1:])
+        if l: print('IPs: ', l)
+        i = 0
+    except KeyboardInterrupt as e:
+        LOG.exception(f"Exception {e}")
+        i = 0
+    except Exception as e:
+        i = 1
+    sys.exit(i)
--- a/src/stem_examples/list_circuits.py
+++ b/src/stem_examples/list_circuits.py
@ -1,3 +1,4 @@
+#!/usr/local/bin/python3.sh
 # -*-mode: python; py-indent-offset: 2; indent-tabs-mode: nil; coding: utf-8-unix -*-

 # http://vt5hknv6sblkgf22.onion/tutorials/examples/list_circuits.html
@ -14,7 +15,7 @@ def iMain():
    else:
        controller = get_controller(port=9051)

-    password = os.environ.get('TOR_CONTROLLER_PASSWORD')
+    password = os.environ.get('TOR_CONTROLLER_PASSWORD', '')
    try:
        controller.authenticate(password)

--- a/src/stem_examples/mappaddress.py
+++ b/src/stem_examples/mappaddress.py
@ -1,5 +1,15 @@
+#!/usr/local/bin/python3.sh
 # -*-mode: python; py-indent-offset: 2; indent-tabs-mode: nil; coding: utf-8-unix -*-
-# https://stem.torproject.org/tutorials/examples/exit_used.html
+#
+__doc__ = """
+Mappaddress queries the socks proxy with an onion address and returns the
+IP address that it will use for it. the address will be in the block specified
+by the VirtualAddrNetworkIPv4  setting of the torrc, e.g.
+
+VirtualAddrNetworkIPv4 172.16.0.0/12
+
+The script takes one argument, an onion address, without the .onion
+"""

 import functools
 import sys
@ -8,22 +18,22 @@ import os

 from stem import StreamStatus
 from stem.control import EventType, Controller
-from tor_controller import set_socks_proxy
-from tor_controller import get_controller
+
+# from tor_controller import set_socks_proxy, unset_socks_proxy
+
+from stem_examples.tor_controller import get_controller

 global LOG
 import logging
-LOG = logging.getLogger('map_address')
+LOG = logging.getLogger()

 def sMapaddressResolv(target, iPort=9051):
    try:
+        password = os.environ.get('TOR_CONTROLLER_PASSWORD', '')
        if os.path.exists('/run/tor/control'):
-            controller = get_controller(unix='/run/tor/control')
+            controller = get_controller(password=password, unix='/run/tor/control')
        else:
-            controller = get_controller(port=9051)
-
-        password = os.environ.get('TOR_CONTROLLER_PASSWORD')
-        controller.authenticate(password)
+            controller = get_controller(password=password, port=iPort)

        map_dict = {"0.0.0.0": target}
        map_ret = controller.map_address(map_dict)
@ -33,9 +43,16 @@ def sMapaddressResolv(target, iPort=9051):
        LOG.exception(e)

 if __name__ == '__main__':
+    from stem_examples.stem_utils import vsetup_logging
+    if os.environ.get('DEBUG', ''):
+        log_level = 10
+    else:
+        log_level = 20
+    LOG = logging.getLogger()
+    vsetup_logging(LOG, log_level)
    if len(sys.argv) < 2:
        target = "l2ct3xnuaiwwtoybtn46qp2av4ndxcguwupzyv6xrsmnwi647vvmwtqd"
    else:
        target = sys.argv[1]
-    print(sMapaddressResolv(target))
+    LOG.info(sMapaddressResolv(target))

--- a/src/stem_examples/outdated_relays.py
+++ b/src/stem_examples/outdated_relays.py
@ -1,28 +1,60 @@
+#!/usr/local/bin/python3.sh
+# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*-
+__doc__ = """List Outdated Relays
+
+Time marches on. Tor makes new releases, and at some point needs to drop
+support for old ones. Below is the script we used on ticket 9476 to reach out
+to relay operators that needed to upgrade.
+
+https://stem.torproject.org/tutorials/examples/outdated_relays.html
+"""
+
+import os
+import sys
+import logging

 from stem.descriptor.remote import DescriptorDownloader
 from stem.version import Version
-from tor_controller import set_socks_proxy

-def iMain():
-    set_socks_proxy()
-    downloader = DescriptorDownloader()
+from tor_controller import set_socks_proxy, unset_socks_proxy
+
+LOG = logging.getLogger()
+
+def iMain(lArgs=None):
+    downloader = DescriptorDownloader(use_mirrors=True)
    count, with_contact = 0, 0
    elts = downloader.get_server_descriptors()
-    print(f"Checking for outdated relays len server_descriptors={len(list(elts))}...")
-    print("")
+    LOG.info(f"Checking for outdated relays len server_descriptors={len(list(elts))}...")

    for desc in elts:
      if desc.tor_version < Version('0.2.3.0'):
        count += 1

        if desc.contact:
-          print('  %-15s %s' % (desc.tor_version, desc.contact.decode("utf-8", "replace")))
+          LOG.info('  %-15s %s' % (desc.tor_version, desc.contact.decode("utf-8", "replace")))
          with_contact += 1

-    print("")
-    print("%i outdated relays found, %i had contact information" % (count, with_contact))
-    # http://vt5hknv6sblkgf22.onion/tutorials/examples/outdated_relays.htmlhttp://vt5hknv6sblkgf22.onion/tutorials/examples/outdated_relays.html
+    LOG.info("%i outdated relays found, %i had contact information" % (count, with_contact))
+    # http://vt5hknv6sblkgf22.onion/tutorials/examples/outdated_relays.html
    return 0

 if __name__ == '__main__':
-    sys.exit(    iMain())
+    set_socks_proxy()
+    from stem_examples.stem_utils import vsetup_logging
+    if os.environ.get('DEBUG', ''):
+        log_level = 10
+    else:
+        log_level = 20
+    vsetup_logging(LOG, log_level)
+    try:
+        l = iMain([])
+        if l: print(l)
+        i = 0
+    except KeyboardInterrupt as e:
+        i = 0
+    except Exception as e:
+        LOG.exception(f"Exception {e}")
+        i = 1
+    finally:
+        unset_socks_proxy()
+    sys.exit(i)
--- a/src/stem_examples/relay_connections.py
+++ b/src/stem_examples/relay_connections.py
@ -1,3 +1,4 @@
+#!/usr/local/bin/python3.sh
 # -*-mode: python; py-indent-offset: 2; indent-tabs-mode: nil; coding: utf-8-unix -*-

 # https://stem.torproject.org/tutorials/examples/relay_connections.html
@ -20,15 +21,14 @@ import stem.connection
 import stem.util.system
 import stem.util.str_tools

-from stem.control import Listener
-from stem.control import Controller
+from stem.control import Listener, Controller
 from stem.util.connection import get_connections, port_usage, is_valid_ipv4_address

-from tor_controller import get_controller
+from stem_examples.tor_controller import get_controller

-global LOG
 import logging
-LOG = logging.getLogger('relay_cons')
+global LOG
+LOG = logging.getLogger()

 HEADER_LINE = " {version}   uptime: {uptime}   flags: {flags}\n"

@ -44,112 +44,86 @@ OUTBOUND_EXIT = 'Outbound exit traffic'
 OUTBOUND_UNKNOWN = 'Outbound uncategorized'

 def iMain(lArgs=None):
-  if lArgs is None:
-      lArgs = sys.argv[1:]
+    if lArgs is None:
+        lArgs = sys.argv[1:]

-  parser = argparse.ArgumentParser()
-  parser.add_argument("--ctrlport", default=9051, type=int, help="default: 9051")
-  parser.add_argument("--resolver", help="default: autodetected")
-  args = parser.parse_args(lArgs)
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--ctrlport", default=9051, type=int, help="default: 9051")
+    parser.add_argument("--resolver", help="default: autodetected")
+    args = parser.parse_args(lArgs)

-  control_port = int(args.ctrlport) if args.ctrlport else 'default'
-  if os.path.exists('/run/tor/control'):
-      controller = get_controller(unix='/run/tor/control')
-  else:
-      controller = get_controller(port=control_port)
-#  controller = stem.connection.connect(control_port = ('127.0.0.1', control_port))
-
-  password = os.environ.get('TOR_CONTROLLER_PASSWORD')
-  controller.authenticate(password)
-
-  if not controller:
-    return 1
-
-  desc = controller.get_network_status(default=None)
-  pid = controller.get_pid()
-  version = str(controller.get_version()).split()[0],
-  uptime = stem.util.str_tools.short_time_label(time.time() - stem.util.system.start_time(pid))
-
-  print(HEADER_LINE.format(
-    version=version,
-    uptime=uptime,
-    flags = ', '.join(desc.flags if desc else ['none']),
-  ))
-
-  policy = controller.get_exit_policy()
-  relays = {}  # address => [orports...]
-
-  for desc in controller.get_network_statuses():
-    relays.setdefault(desc.address, []).append(desc.or_port)
-
-  # categorize our connections
-
-  categories = collections.OrderedDict((
-    (INBOUND_ORPORT, []),
-    (INBOUND_DIRPORT, []),
-    (INBOUND_CONTROLPORT, []),
-    (OUTBOUND_ORPORT, []),
-    (OUTBOUND_EXIT, []),
-    (OUTBOUND_UNKNOWN, []),
-  ))
-
-  exit_connections = {}  # port => [connections]
-
-  for conn in get_connections(resolver = args.resolver, process_pid = pid):
-    if conn.protocol == 'udp':
-        continue
-
-    if conn.local_port in controller.get_ports(Listener.OR, []):
-      categories[INBOUND_ORPORT].append(conn)
-    elif conn.local_port in controller.get_ports(Listener.DIR, []):
-      categories[INBOUND_DIRPORT].append(conn)
-    elif conn.local_port in controller.get_ports(Listener.CONTROL, []):
-      categories[INBOUND_CONTROLPORT].append(conn)
-    elif conn.remote_port in relays.get(conn.remote_address, []):
-      categories[OUTBOUND_ORPORT].append(conn)
-    elif policy.can_exit_to(conn.remote_address, conn.remote_port):
-      categories[OUTBOUND_EXIT].append(conn)
-      exit_connections.setdefault(conn.remote_port, []).append(conn)
+    password = os.environ.get('TOR_CONTROLLER_PASSWORD', '')
+    control_port = int(args.ctrlport) if args.ctrlport else 'default'
+    if False and os.path.exists('/run/tor/control'):
+        controller = get_controller(password=password, unix='/run/tor/control')
    else:
-      categories[OUTBOUND_UNKNOWN].append(conn)
+        controller = get_controller(password=password, port=control_port)

-  print(DIV)
-  print(COLUMN % ('Type', 'IPv4', 'IPv6'))
-  print(DIV)
+    if not controller:
+      return 1

-  total_ipv4, total_ipv6 = 0, 0
+    desc = controller.get_network_status(default=None)
+    pid = controller.get_pid()
+    version = str(controller.get_version()).split()[0],
+    uptime = stem.util.str_tools.short_time_label(time.time() - stem.util.system.start_time(pid))

-  for label, connections in categories.items():
-    if len(connections) == 0:
-      continue
+    LOG.info(HEADER_LINE.format(
+      version=version,
+      uptime=uptime,
+      flags = ', '.join(desc.flags if desc else ['none']),
+    ))

-    ipv4_count = len([conn for conn in connections if is_valid_ipv4_address(conn.remote_address)])
-    ipv6_count = len(connections) - ipv4_count
+    policy = controller.get_exit_policy()
+    relays = {}  # address => [orports...]

-    total_ipv4, total_ipv6 = total_ipv4 + ipv4_count, total_ipv6 + ipv6_count
-    print(COLUMN % (label, ipv4_count, ipv6_count))
+    for desc in controller.get_network_statuses():
+      relays.setdefault(desc.address, []).append(desc.or_port)

-  print(DIV)
-  print(COLUMN % ('Total', total_ipv4, total_ipv6))
-  print(DIV)
-  print('')
+    # categorize our connections
+
+    categories = collections.OrderedDict((
+      (INBOUND_ORPORT, []),
+      (INBOUND_DIRPORT, []),
+      (INBOUND_CONTROLPORT, []),
+      (OUTBOUND_ORPORT, []),
+      (OUTBOUND_EXIT, []),
+      (OUTBOUND_UNKNOWN, []),
+    ))
+
+    exit_connections = {}  # port => [connections]
+
+    for conn in get_connections(resolver = args.resolver, process_pid = pid):
+      if conn.protocol == 'udp':
+          continue
+
+      if conn.local_port in controller.get_ports(Listener.OR, []):
+        categories[INBOUND_ORPORT].append(conn)
+      elif conn.local_port in controller.get_ports(Listener.DIR, []):
+        categories[INBOUND_DIRPORT].append(conn)
+      elif conn.local_port in controller.get_ports(Listener.CONTROL, []):
+        categories[INBOUND_CONTROLPORT].append(conn)
+      elif conn.remote_port in relays.get(conn.remote_address, []):
+        categories[OUTBOUND_ORPORT].append(conn)
+      elif policy.can_exit_to(conn.remote_address, conn.remote_port):
+        categories[OUTBOUND_EXIT].append(conn)
+        exit_connections.setdefault(conn.remote_port, []).append(conn)
+      else:
+        categories[OUTBOUND_UNKNOWN].append(conn)

-  if exit_connections:
    print(DIV)
-    print(COLUMN % ('Exit Port', 'IPv4', 'IPv6'))
+    print(COLUMN % ('Type', 'IPv4', 'IPv6'))
    print(DIV)

    total_ipv4, total_ipv6 = 0, 0

-    for port in sorted(exit_connections):
-      connections = exit_connections[port]
+    for label, connections in categories.items():
+      if len(connections) == 0:
+        continue
+
      ipv4_count = len([conn for conn in connections if is_valid_ipv4_address(conn.remote_address)])
      ipv6_count = len(connections) - ipv4_count
+
      total_ipv4, total_ipv6 = total_ipv4 + ipv4_count, total_ipv6 + ipv6_count
-
-      usage = port_usage(port)
-      label = '%s (%s)' % (port, usage) if usage else port
-
      print(COLUMN % (label, ipv4_count, ipv6_count))

    print(DIV)
@ -157,7 +131,46 @@ def iMain(lArgs=None):
    print(DIV)
    print('')

+    if exit_connections:
+      print(DIV)
+      print(COLUMN % ('Exit Port', 'IPv4', 'IPv6'))
+      print(DIV)
+
+      total_ipv4, total_ipv6 = 0, 0
+
+      for port in sorted(exit_connections):
+        connections = exit_connections[port]
+        ipv4_count = len([conn for conn in connections if is_valid_ipv4_address(conn.remote_address)])
+        ipv6_count = len(connections) - ipv4_count
+        total_ipv4, total_ipv6 = total_ipv4 + ipv4_count, total_ipv6 + ipv6_count
+
+        usage = port_usage(port)
+        label = '%s (%s)' % (port, usage) if usage else port
+
+        print(COLUMN % (label, ipv4_count, ipv6_count))
+
+      print(DIV)
+      print(COLUMN % ('Total', total_ipv4, total_ipv6))
+      print(DIV)
+      print('')

 if __name__ == '__main__':
-    iMain()
+    from stem_examples.stem_utils import vsetup_logging
+    LOG = logging.getLogger()
+    if os.environ.get('DEBUG', ''):
+        log_level = 10
+    else:
+        log_level = 20
+    vsetup_logging(LOG, log_level)
+    LOG.setLevel(logging.DEBUG)
+    try:
+        l = iMain([])
+        if l: print(l)
+        i = 0
+    except KeyboardInterrupt as e:
+        i = 0
+    except Exception as e:
+        LOG.exception(f"Exception {e}")
+        i = 1
+    sys.exit(i)

--- a/src/stem_examples/stem_utils.py
+++ b/src/stem_examples/stem_utils.py
@ -0,0 +1,58 @@
+# -*-mode: python; py-indent-offset: 2; indent-tabs-mode: nil; coding: utf-8-unix -*-
+
+import sys
+import logging
+
+try:
+#    if 'COLOREDLOGS_LEVEL_STYLES' not in os.environ:
+#        os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red'
+    # https://pypi.org/project/coloredlogs/
+    import coloredlogs
+except ImportError:
+    coloredlogs = False
+
+def vsetup_logging(theLOG, log_level, logfile='', stream=sys.stdout) -> None:
+    global LOG
+    LOG = theLOG
+    add = True
+
+    logging._defaultFormatter = logging.Formatter(datefmt='%m-%d %H:%M:%S')
+    logging._defaultFormatter.default_time_format = '%m-%d %H:%M:%S'
+    logging._defaultFormatter.default_msec_format = ''
+
+    if logfile:
+        add = logfile.startswith('+')
+        sub = logfile.startswith('-')
+        if add or sub:
+            logfile = logfile[1:]
+        kwargs['filename'] = logfile
+
+    if coloredlogs:
+        coloredlogs.DEFAULT_LEVEL_STYLES['info']=dict(color='white',bold=True)
+        coloredlogs.DEFAULT_LEVEL_STYLES['debug']=dict(color='cyan')
+        coloredlogs.DEFAULT_LEVEL_STYLES['warn']=dict(color='yellow',bold=True)
+        coloredlogs.DEFAULT_LEVEL_STYLES['error']=dict(color='red',bold=True)
+        coloredlogs.DEFAULT_FIELD_STYLES['levelname=']=dict(color='green', bold=True),
+        # https://pypi.org/project/coloredlogs/
+        aKw = dict(level=log_level,
+                   logger=LOG,
+                   stream=stream,
+                   fmt='%(levelname)s %(message)s',
+                   isatty=True,
+                   milliseconds=False,
+                   )
+        coloredlogs.install(**aKw)
+        if logfile:
+            oHandler = logging.FileHandler(logfile)
+            LOG.addHandler(oHandler)
+        LOG.debug(f"Setting coloured log_level to {log_level}")
+    else:
+        kwargs = dict(level=log_level,
+                    force=True,
+                    format='%(levelname)s %(message)s')
+        logging.basicConfig(**kwargs)
+        if add and logfile:
+            oHandler = logging.StreamHandler(stream)
+            LOG.addHandler(oHandler)
+        LOG.debug(f"SSetting log_level to {log_level}")
+
--- a/src/stem_examples/tor_bootstrap_check.py
+++ b/src/stem_examples/tor_bootstrap_check.py
@ -12,21 +12,21 @@ tor is at.
 import sys
 import os
 import re
+import logging
+import socket

 from stem.connection import connect
-from tor_controller import get_controller
+from stem_examples.tor_controller import get_controller
+
+LOG = logging.getLogger()

 def iMain(lArgs=None):
-    password = os.environ.get('TOR_CONTROLLER_PASSWORD')
+    password = os.environ.get('TOR_CONTROLLER_PASSWORD', '')
    if os.path.exists('/run/tor/control'):
        controller = get_controller(password=password, unix='/run/tor/control')
    else:
        controller = get_controller(password=password, port=9051)

-#    controller.connect()
-
-    bootstrap_status = controller.get_info("status/bootstrap-phase")
-
    ## Possible answer, if network cable has been removed:
    ## 250-status/bootstrap-phase=WARN BOOTSTRAP PROGRESS=80 TAG=conn_or SUMMARY="Connecting to the Tor network" WARNING="No route to host" REASON=NOROUTE COUNT=26 RECOMMENDATION=warn

@ -38,15 +38,33 @@ def iMain(lArgs=None):

    ## TODO: parse the messages above.

-    print(format(bootstrap_status))
+    try:
+        bootstrap_status = controller.get_info("status/bootstrap-phase")
+        LOG.info(format(bootstrap_status))

-    progress_percent = re.match('.* PROGRESS=([0-9]+).*', bootstrap_status)
-    exit_code = int(progress_percent.group(1))
+        progress_percent = re.match('.* PROGRESS=([0-9]+).*', bootstrap_status)
+        exit_code = int(progress_percent.group(1))

-    controller.close()
-
-    return exit_code
+        controller.close()
+    except socket.error as e:
+        # Error while receiving a control message (SocketClosed): received exception "read of closed file"
+        return 0
+    except Exception as e:
+        raise
+    return 0

 if __name__ == '__main__':
-    sys.exit(    iMain())
-
+    from stem_examples.stem_utils import vsetup_logging
+    if os.environ.get('DEBUG', ''):
+        log_level = 10
+    else:
+        log_level = 20
+    vsetup_logging(LOG, log_level)
+    try:
+        i = iMain([])
+    except KeyboardInterrupt as e:
+        i = 0
+    except Exception as e:
+        LOG.exception(f"Exception {e}")
+        i = 1
+    sys.exit(i)
--- a/src/stem_examples/tor_controller.py
+++ b/src/stem_examples/tor_controller.py
@ -12,7 +12,7 @@ def set_socks_proxy(SOCKS5_PROXY_HOST='127.0.0.1', SOCKS5_PROXY_PORT=9050):
    try:
        import socks # you need to install pysocks (see above)
        # Remove this if you don't plan to "deactivate" the proxy later
-        default_socket = socket.socket
+        socks._socket = socket.socket

        # Set up a proxy
        socks.set_default_proxy(socks.SOCKS5, SOCKS5_PROXY_HOST, SOCKS5_PROXY_PORT)
@ -21,6 +21,14 @@ def set_socks_proxy(SOCKS5_PROXY_HOST='127.0.0.1', SOCKS5_PROXY_PORT=9050):
        return False
    return True

+def unset_socks_proxy():
+    import socks # you need to install pysocks (see above)
+    # Remove this if you don't plan to "deactivate" the proxy later
+    socks.socket = socket._socket
+
+    # Set up a proxy
+    socks.set_default_proxy(socks.SOCKS5, None, None)
+
 def get_controller(password=None, address='127.0.0.1', port=9051, unix='/run/tor/control'):
    if unix and os.path.exists(unix):
        #  print(unix)
@ -31,7 +39,7 @@ def get_controller(password=None, address='127.0.0.1', port=9051, unix='/run/tor

    if password is None:
 #        print("DBUG: trying TOR_CONTROLLER_PASSWORD")
-        password = os.environ.get('TOR_CONTROLLER_PASSWORD')
+        password = os.environ.get('TOR_CONTROLLER_PASSWORD', '')
    else:
 #       print(f"DBUG: using a password {len(password)}")
        pass
--- a/src/stem_examples/torcontactinfo.py
+++ b/src/stem_examples/torcontactinfo.py
--- a/stem_examples.txt
+++ b/stem_examples.txt
@ -3,28 +3,75 @@
 == stem_examples tor testing ==

 This is a Python doctest file that is executable documentation.
+stem_examples is a set of small scripts that tell you about your
+running tor instance.

-Pass the controller password if needed as an environment variable:
+Onionoo is a web-based protocol to learn about currently running Tor
+relays and bridges. Onionoo itself was not designed as a service for
+human beings---at least not directly. Onionoo provides the data for
+other applications and websites which in turn present Tor network
+status information to humans: https://metrics.torproject.org/onionoo.html
+
+You can see the status of tor relays at https://torstatus.rueckgr.at/
+The code for that site is at https://github.com/paulchen/torstatus
+You can get a list of exit relays that are marked bad with:
+wget --post-data='SR=FBadExit&SO=Asc&FBadExit=1' 'https://torstatus.rueckgr.at/'
+
+It is assumed that you are running a tor that has its torrc configured with:
+
+ControlPort 127.0.0.1:9051
+
+and/or
+
+ControlSocket /run/tor/control
+ControlSocketsGroupWritable 1
+
+We can authenticate with a password. To set a password first get its hash...
+
+% tor --hash-password "my_password"
+16:E600ADC1B52C80BB6022A0E999A7734571A451EB6AE50FED489B72E3DF
+
+and use that for the HashedControlPassword in your torrc.
+
+HashedControlPassword 16:E600ADC1B52C80BB6022A0E999A7734571A451EB6AE50FED489B72E3DF
+
+so that you have some security on the Control connection.
+Pass the controller password to these scripts as an environment variable:

    >>> import os
    >>> assert os.environ['TOR_CONTROLLER_PASSWORD']
    
+If you are using /run/tor/control you will also need to run the scripts as the user
+that has rw access to that socket, usually tor or debian-tor.
+
 Add our code to the PYTHONPATH 
    >>> import sys
    >>> sys.path.append(os.path.join(os.getcwd(), 'src', 'stem_examples'))

-We'll need the settings defined in  {{{/usr/local/etc/testforge/testforge.yml}}}
+We'll used the settings defined in  {{{/usr/local/etc/testforge/testforge.yml}}}
+If you don't have one, make it with the settings from your torrc:

    >>> print("yaml", file=sys.stderr)
    >>> import yaml
-    >>> sFacts = open('/usr/local/etc/testforge/testforge.yml').read()
-    >>> assert sFacts
-    >>> dFacts = yaml.safe_load(sFacts)
+    >>> try:
+    ...     sFacts = open('/usr/local/etc/testforge/testforge.yml').read()
+    ...     assert sFacts, sFacts 
+    ... except:
+    ...     dFacts = dict(
+    ...       HTTPS_PROXYHOST="127.0.0.1",
+    ...       HTTPS_PROXYPORT=9128,
+    ...       HTTPS_PROXYTYPE="http",
+    ...       SOCKS_PROXYHOST="127.0.0.1",
+    ...       SOCKS_PROXYPORT=9050,
+    ...       SOCKS_PROXYTYPE="socks5",
+    ...     )
+    ... else:
+    ...     dFacts = yaml.safe_load(sFacts)
+
 
 FixMe: use the settings for the ports and directories below.

    >>> import os
-    >>> os.environ['http_proxy']  = 'http://'+dFacts['HTTP_PROXYHOST']+':'+str(dFacts['HTTP_PROXYPORT'])
    >>> os.environ['https_proxy'] = 'http://'+dFacts['HTTPS_PROXYHOST']+':'+str(dFacts['HTTPS_PROXYPORT'])
    >>> os.environ['socks_proxy']  = 'socks5://'+dFacts['SOCKS_PROXYHOST']+':'+str(dFacts['SOCKS_PROXYPORT'])

@ -40,16 +87,17 @@ We test 3 known hidden services: Facebook, DuckDuckGo and .
   >>> lKNOWN_ONIONS = [
   ...  'facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd', # facebook
   ...  'duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad', # ddg
-   ...  'zkaan2xfbuxia2wpf7ofnkbz6r5zdbbvxbunvp5g2iebopbfc4iqmbad', # hks
   ... ]

 We wil expect to get back the hidden service version, the descriptor-lifetime
 and then the descriptor-signing-key-cert:

    >>> introduction_points.iMain(lKNOWN_ONIONS) #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+    0
+
    hs-descriptor 3
    descriptor-lifetime ...
-    <BLANKLINE>
+        <BLANKLINE>
    
 ### exit_used Determine The Exit You're Using

@ -59,22 +107,34 @@ How can you figure out what exit you're using?

    >>> print("exit_used", file=sys.stderr)
    >>> import exit_used
+    >>> exit_used.iMain([])

 ## relay_connections Connection Summary

    >>> print("relay_connections", file=sys.stderr)
    >>> import relay_connections
+    >>> relay_connections.iMain([]) #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+    +------------------------------+------+------+
+    ...
+    +------------------------------+------+------+
+    <BLANKLINE>

 The following provides a summary of your relay's inbound and outbound connections.
 You must be root or tor to run this:
 relay_connections.iMain(["--ctrlport", "9051"])

-## outdated_relays
+## connection_resolution  Connection Resolution 

-    >>> print("outdated_relays", file=sys.stderr)
-    >>> import outdated_relays
-    >>> outdated_relays.iMain() #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
-    Checking for outdated relays ...
+Connection information is a useful tool for learning more about network
+applications like Tor. Our stem.util.connection.get_connections() function
+provides an easy method for accessing this information.
+
+    >>> print("connection_resolution", file=sys.stderr)
+    >>> import connection_resolution
+    >>> connection_resolution.iMain([]) #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+    0
+    
+    INFO Our platform supports connection resolution via: ...
    <BLANKLINE>

 ## tor_bootstrap_check
@ -85,18 +145,46 @@ relay_connections.iMain(["--ctrlport", "9051"])
 A script by adrelanos@riseup.net to check what percentage of boostrapping
 tor is at. This fails under doctest but not from the cmdline

->> tor_bootstrap_check.iMain() #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
-NOTICE ...
-<BLANKLINE>
+    >>> tor_bootstrap_check.iMain([]) #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+    0
    
-    control_port = stem.socket.ControlPort(address, port)
-                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.11/site-packages/stem/socket.py", line 503, in __init__
-    self.connect()
-  File "/usr/local/lib/python3.11/site-packages/stem/socket.py", line 172, in connect
-    self._socket = self._make_socket()
-                   ^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.11/site-packages/stem/socket.py", line 538, in _make_socket
-    raise stem.SocketError(exc)
-stem.SocketError: Socket error: 0x01: General SOCKS server failure
+    NOTICE ...
+    <BLANKLINE>
+
+## check_digests
+
+    >>> print("check_digests", file=sys.stderr)
+    >>> from check_digests import iMain
+    >>> sKNOWN_ONION = 'facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd' # facebook
+    >>> from stem_examples.introduction_points import lMain as lIPMain
+    >>> lArgs = []
+    >>> import stem_examples.support_testing as ts; with ts.ignoreStdout():
+    ...    lArgs = lIPMain([sKNOWN_ONION])
+    >>> iMain(lArgs)
+    0
+    
+## interpreter
+
+    >>> print("interpreter", file=sys.stderr)
+    >>> import interpreter
+    >>> lArgs = ['GETINFO', 'version']
+    >>> interpreter.iMain(lArgs) #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+    250-version=0.4.8.10
+    250 OK
+    <BLANKLINE>
+    0
+
+## outdated_relays List Outdated Relays
+
+Time marches on. Tor makes new releases, and at some point needs to drop
+support for old ones. Below is the script we used on ticket 9476 to reach out
+to relay operators that needed to upgrade.
+
+    >>> print("outdated_relays", file=sys.stderr)
+    >>> import outdated_relays
+    >>> outdated_relays.iMain([]) #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+    0
+    
+    Checking for outdated relays ...
+    <BLANKLINE>
Author	SHA1	Message	Date
emdee@spm.plastiras.org	9c6594c301	update	2024-02-21 08:03:10 +00:00
emdee@spm.plastiras.org	abfc9d28e5	update	2024-02-05 14:26:19 +00:00
emdee@spm.plastiras.org	df8ea68e4d	update	2024-02-04 03:29:52 +00:00
emdee@spm.plastiras.org	b9bdb02067	update	2024-01-17 20:35:20 +00:00
emdee@spm.plastiras.org	91465abf2a	update	2024-01-17 16:30:48 +00:00
emdee@spm.plastiras.org	72010d8f8d	tests	2024-01-16 18:10:43 +00:00
emdee@spm.plastiras.org	42cf62f623	improvements	2024-01-16 14:35:29 +00:00
emdee@spm.plastiras.org	dbe62ffbd7	upgrade	2024-01-15 12:37:32 +00:00