update

.gitignore

@@ -4,6 +4,7 @@ __pycache__/
 *.py[cod]
 *$py.class
 
+#*
 *~
 *.dst
 

Makefile

@@ -1,23 +1,57 @@
-LOCAL_DOCTEST=/usr/local/bin/toxcore_run_doctest3.bash
+# to run the tests, run make PASS=controllerpassword test
+
+PREFIX=/usr/local
+PYTHON_EXE_MSYS=${PREFIX}/bin/python3.sh
+PIP_EXE_MSYS=${PREFIX}/bin/pip3.sh
+LOCAL_DOCTEST=${PREFIX}/bin/toxcore_run_doctest3.bash
 DOCTEST=${LOCAL_DOCTEST}
 MOD=stem_examples
 
-
 check::
 	sh python3.sh -c "import ${MOD}"
 
 lint::
 	sh .pylint.sh
 
+install::
+	${PIP_EXE_MSYS} --python ${PYTHON_EXE_MSYS} install \
+		--no-deps \
+		--target ${PREFIX}/lib/python${PYTHON_MINOR}/site-packages/ \
+		--upgrade .
+	sed -i -e "1s@/usr/bin/python${PYTHON_MINOR}@${PYTHON_EXE_MSYS}@" \
+		${PREFIX}/lib/python${PYTHON_MINOR}/site-packages/bin/*
+
 rsync::
 	bash .rsync.sh
 
-pyi::
+# execute these tests as: make test PASS=password
-	echo FixMe
+test::
+	echo src/${MOD}/check_digests.py
+	TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/check_digests.py
+	echo src/${MOD}/interpreter.py
+	TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/interpreter.py
+	echo src/${MOD}/connection_resolution.py
+	sudo env TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/connection_resolution.py
+# broken because this site fails: http://128.31.0.39:9131/tor/status-vote
+#	${PYTHON_EXE_MSYS} src/${MOD}/compare_flags.py
+# cant use from make: waits for the cmdline to to terminate
+#	TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/exit_used.py 10
+	echo src/${MOD}/introduction_points.py
+	TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/introduction_points.py
+	echo src/${MOD}/list_circuits.py
+	TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/list_circuits.py 
+	echo src/${MOD}/mappaddress.py
+	TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/mappaddress.py
+	echo src/${MOD}/outdated_relays.py NOT WORKING?
+	TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/outdated_relays.py
+	echo src/${MOD}/relay_connections.py
+	sudo env TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/relay_connections.py
+	echo src/${MOD}/tor_bootstrap_check.py
+	TOR_CONTROLLER_PASSWORD=${PASS} ${PYTHON_EXE_MSYS} src/${MOD}/tor_bootstrap_check.py
 
 doctest:
-	export PYTHONPATH=${PWD}
+	sudo -u tor env PYTHONPATH=${PWD}/src \
-	${DOCTEST} ${MOD].txt
+	${DOCTEST} ${MOD}.txt
 
 veryclean:: clean
 	rm -rf build dist __pycache__ .pylint.err .pylint.out

README.md

@@ -35,6 +35,13 @@ and maatuska, with a special interest in the 'Running' flag.
 
 https://stem.torproject.org/tutorials/examples/compare_flags.html
 
+### connection_resolution Connection Resolution
+
+Connection information is a useful tool for learning more about network
+applications like Tor. Our stem.util.connection.get_connections() function
+provides an easy method for accessing this information.
+
+
 ### exit_used Exit Used
 
 Determine The Exit You're Using
@@ -57,6 +64,12 @@ https://stem.torproject.org/tutorials/examples/list_circuits.html
 
 ### mappaddress
 
+Mappaddress queries the socks proxy with an onion address and returns the
+IP address that it will use for it. the address will be in the block specified
+by the VirtualAddrNetworkIPv4  setting of the torrc, e.g.
+
+VirtualAddrNetworkIPv4 172.16.0.0/12
+
 ### outdated_relays List Outdated Relays
 
 Time marches on. Tor makes new releases, and at some point needs to

pyproject.toml

@@ -3,9 +3,6 @@ name = "stem_examples"
 description = "examples of using stem"
 authors = [{ name = "emdee", email = "emdee@spm.plastiras.org" } ]
 requires-python = ">=3.6"
-dependencies = [
-    'stem',
-]
 keywords = ["stem", "python3", "tor"]
 classifiers = [
     "License :: OSI Approved",
@@ -20,7 +17,6 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: Implementation :: CPython",
     ]
-#
 dynamic = ["version", "readme", ] # cannot be dynamic ['license']
 
 [project.scripts]
@@ -49,8 +45,3 @@ build-backend = "setuptools.build_meta"
 version = {attr = "stem_examples.__version__"} 
 readme = {file = ["README.md", "stem_examples.txt"]}
 
-[tool.setuptools]
-packages = ["stem_examples"]
-
-#[tool.setuptools.packages.find]
-#where = "src"

setup.cfg

@@ -19,11 +19,9 @@ classifiers =
 [options]
 zip_safe = false
 python_requires = ~=3.6
-include_package_data = false
+include_package_data =
+    "*" = ["*.txt", "*.bash" ]
 install_requires =
-    qasync
-    cryptography
-    rsa
     stem
     ruamel.yaml
 package_dir=
@@ -34,9 +32,8 @@ packages=find:
 where=src
 
 [options.entry_points]
-console_scripts =
+#console_scripts =
-    phantompy = phantompy.__main__:iMain
+#    exclude_badExits = exclude_badExits:iMain
-    exclude_badExits = exclude_badExits:iMain
 
 [easy_install]
 zip_ok = false

src/stem_examples/__init__.py

@@ -1 +1,2 @@
+
 __version__ = "1.0.0"

src/stem_examples/check_digests.py

@@ -1,3 +1,4 @@
+#!/usr/local/bin/python3.sh
 # -*-mode: python; py-indent-offset: 2; indent-tabs-mode: nil; coding: utf-8-unix -*-
 
 __doc__ = """
@@ -8,16 +9,20 @@ Tor relay information is provided by multiple documents. Signed descriptors tran
 Stem can calculate digests from server, extrainfo, microdescriptor, and
 consensus documents. For instance, to validate an extrainfo descriptor...
 
-https://stem.torproject.org/tutorials/examples/check_digests.html """
+https://stem.torproject.org/tutorials/examples/check_digests.html
+"""
 
 import os
 import sys
 import contextlib
 import logging
 
-from tor_controller import set_socks_proxy
+import stem
+from stem_examples.tor_controller import set_socks_proxy
 
 LOG = logging.getLogger()
+sKNOWN_ONION = 'facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd' # facebook
+iTIMEOUT = 120
 
 @contextlib.contextmanager
 def ignoreStdout() -> None:
@@ -56,6 +61,7 @@ def download_descriptors(fingerprint):
   )
 
 def iMain(lArgs=None):
+  global LOG
 #  set_socks_proxy()
   iRetval = 0
   if lArgs is None:
@@ -64,7 +70,7 @@ def iMain(lArgs=None):
       lArgs = [fingerprint]
 
   for fingerprint in  lArgs:
-    log.INFO(f"checking digests of fp={fp}")
+    LOG.info(f"checking digests of fp={fp}")
     if not stem.util.tor_tools.is_valid_fingerprint(fingerprint):
       LOG.error("'%s' is not a valid relay fingerprint" % fingerprint)
       iRetval += 1
@@ -91,31 +97,26 @@ def iMain(lArgs=None):
   return iRetval
 
 if __name__ == '__main__':
-  LOG.setLevel(logging.DEBUG)
+    from stem_examples.stem_utils import vsetup_logging
-
+    if os.environ.get('DEBUG', ''):
-  try:
+        log_level = 10
-    import stem.descriptor.remote
-    import stem.util.tor_tools
         logging.getLogger('stem').setLevel(20)
-    # bizarre uncatchable stem error
+    else:
-    import stem.response.protocolinfo
+        log_level = 20
-    import stem.response.mapaddress
+        logging.getLogger('stem').setLevel(30)
-    import stem.response.getconf
+    vsetup_logging(LOG, log_level)
-    import stem.response.getinfo
+    try:
-    import stem.response.authchallenge
-    # 'tuple' object has no attribute 'endswith'
-
         if len(sys.argv) > 1:
             lArgs = sys.argv[1:]
             LOG.info(f"Getting some {len(lArgs)}")
         else:
-      sKNOWN_ONION = 'facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd', # facebook
+            LOG.info(f"Getting some FPs from the IPs to a sKNOWN_ONION TIMEOUT={iTIMEOUT}")
-      LOG.info("Getting some FPs from a sKNOWN_ONION")
+            from stem_examples.introduction_points import lMain as lIPMain
-      from stem_examples.introduction_points import lMain
+            with ignoreStdout():
-  #    with ignoreStdout():
+                lArgs = lIPMain([sKNOWN_ONION], timeout=iTIMEOUT)
-      lArgs = lMain([sKNOWN_ONION])
+            LOG.info(f"Got {len(lArgs)} FPs from a sKNOWN_ONION")
-      LOG.debug(f"Got {len(lArgs)} FPs from a sKNOWN_ONION")
+            if not lArgs:
-
+                sys.exit(1)
         i = iMain(lArgs)
     except KeyboardInterrupt as e:
         i = 0
@@ -123,5 +124,3 @@ if __name__ == '__main__':
         LOG.exception(f"Exception in iMain {e}")
         i = 1
     sys.exit(i)
-
-

src/stem_examples/compare_flags.py

@@ -1,9 +1,13 @@
+#!/usr/local/bin/python3.sh
 # -*-mode: python; py-indent-offset: 2; indent-tabs-mode: nil; coding: utf-8-unix -*-
 #
 __doc__ = """
 Compares the votes of two directory authorities, in this case moria1
 and maatuska, with a special interest in the 'Running' flag.
 
+broken because this site fails:
+http://128.31.0.39:9131/tor/status-vote
+
 https://stem.torproject.org/tutorials/examples/compare_flags.html
 """
 
@@ -14,7 +18,7 @@ import stem.descriptor
 import stem.descriptor.remote
 import stem.directory
 
-from tor_controller import set_socks_proxy
+from tor_controller import set_socks_proxy, unset_socks_proxy
 
 def iMain():
     # Query all authority votes asynchronously.
@@ -66,4 +70,10 @@ def iMain():
 
 if __name__ == '__main__':
     set_socks_proxy()
-    sys.exit(iMain())
+    try:
+        i = iMain()
+    except KeyboardInterrupt as e:
+        i = 0
+    finally:
+        unset_socks_proxy()
+    sys.exit(i)

src/stem_examples/connection_resolution.py

@@ -0,0 +1,77 @@
+__doc__ = """Connection Resolution
+
+Connection information is a useful tool for learning more about network
+applications like Tor. Our stem.util.connection.get_connections() function
+provides an easy method for accessing this information, with a few caveats...
+
+Connection resolvers are platform specific. We support several platforms but
+not all.
+
+By default Tor runs with a feature called DisableDebuggerAttachment. This
+prevents debugging applications like gdb from analyzing Tor unless it is run as
+root. Unfortunately this also alters the permissions of the Tor process /proc
+contents breaking numerous system tools (including our resolvers). To use this
+function you need to either run as root (discouraged) or add
+DisableDebuggerAttachment 0 to your torrc.
+
+Please note that if you operate an exit relay it is highly discouraged for you
+to look at or record this information. Not only is doing so eavesdropping, but
+likely also a violation of wiretap laws.
+
+With that out of the way, how do you look up this information? Below is a
+simple script that dumps Tor's present connections.
+
+https://stem.torproject.org/tutorials/east_of_the_sun.html
+"""
+
+import os
+import sys
+import logging
+
+from stem.util.connection import get_connections, system_resolvers
+from stem.util.system import pid_by_name
+
+LOG = logging.getLogger()
+
+def iMain (lArgs=None):
+    resolvers = system_resolvers()
+
+    if not resolvers:
+        LOG.error("Stem doesn't support any connection resolvers on our platform.")
+        return 1
+
+    picked_resolver = resolvers[0]  # lets just opt for the first
+    LOG.info("Our platform supports connection resolution via: %s (picked %s)" % (', '.join(resolvers), picked_resolver))
+
+    tor_pids = pid_by_name('tor', multiple = True)
+
+    if not tor_pids:
+        LOG.warn("Unable to get tor's pid. Is it running?")
+        return 1
+
+    if len(tor_pids) > 1:
+        LOG.info("You're running %i instances of tor, picking the one with pid %i" % (len(tor_pids), tor_pids[0]))
+    else:
+        LOG.info("Tor is running with pid %i" % tor_pids[0])
+
+    LOG.info("Connections:\n")
+
+    for conn in get_connections(picked_resolver, process_pid = tor_pids[0], process_name = 'tor'):
+        LOG.info("  %s:%s => %s:%s" % (conn.local_address, conn.local_port, conn.remote_address, conn.remote_port))
+    return 0
+
+if __name__ == '__main__':
+    from stem_examples.stem_utils import vsetup_logging
+    if os.environ.get('DEBUG', ''):
+        log_level = 10
+    else:
+        log_level = 20
+    vsetup_logging(LOG, log_level)
+    try:
+        i = iMain(sys.argv[1:])
+    except KeyboardInterrupt as e:
+        i = 0
+    except Exception as e:
+        LOG.exception(f"Exception {e}")
+        i = 1
+    sys.exit(i)

src/stem_examples/exit_used.py

@@ -1,3 +1,4 @@
+#!/usr/local/bin/python3.sh
 # -*-mode: python; py-indent-offset: 2; indent-tabs-mode: nil; coding: utf-8-unix -*-
 # https://stem.torproject.org/tutorials/examples/exit_used.html
 __doc__ = """Determine The Exit You're Using
@@ -11,42 +12,74 @@ How can you figure out what exit you're using?
 import functools
 import sys
 import os
+import logging
 
 from stem import StreamStatus
 from stem.control import EventType, Controller
 from stem_examples.tor_controller import get_controller
 
+global COUNT, IMAX
+COUNT=0
+global IMAX
+IMAX = 10
+LOG = logging.getLogger()
+
 def stream_event(controller, event):
+    global COUNT, IMAX
+    COUNT += 1
+    if IMAX and COUNT >= IMAX:
+        LOG.info(f"exiting COUNT={COUNT}")
+        sys.exit(0)
     if event.status == StreamStatus.SUCCEEDED and event.circ_id:
       circ = controller.get_circuit(event.circ_id)
 
       exit_fingerprint = circ.path[-1][0]
       exit_relay = controller.get_network_status(exit_fingerprint)
 
-      print("Exit relay for our connection to %s" % (event.target))
+      LOG.info("Exit relay for our connection to %s" % (event.target))
-      print("  address: %s:%i" % (exit_relay.address, exit_relay.or_port))
+      LOG.info("  address: %s:%i" % (exit_relay.address, exit_relay.or_port))
-      print("  fingerprint: %s" % exit_relay.fingerprint)
+      LOG.info("  fingerprint: %s" % exit_relay.fingerprint)
-      print("  nickname: %s" % exit_relay.nickname)
+      LOG.info("  nickname: %s" % exit_relay.nickname)
-      print("  locale: %s" % controller.get_info("ip-to-country/%s" % exit_relay.address, 'unknown'))
+      LOG.info("  locale: %s" % controller.get_info("ip-to-country/%s" % exit_relay.address, 'unknown'))
-      print("")
+      LOG.info("")
 
 
-def iMain():
+def iMain(lArgs=None):
-    print("Please wait for requests for tor exits. Press 'enter' to end.")
-    print("")
-    if os.path.exists('/run/tor/control'):
-      controller = get_controller(unix='/run/tor/control')
-    else:
-      controller = get_controller(port=9051)
-
     password = os.environ.get('TOR_CONTROLLER_PASSWORD')
-    controller.authenticate(password)
 
+    if os.path.exists('/run/tor/control'):
+      controller = get_controller(password=password, unix='/run/tor/control')
+    else:
+      controller = get_controller(password=password, port=9051)
+
+    if IMAX <= 0:
+        LOG.info("Please wait for requests for tor exits. Press 'enter' to end.")
+        print("")
     stream_listener = functools.partial(stream_event, controller)
     controller.add_event_listener(stream_listener, EventType.STREAM)
 
 if __name__ == '__main__':
+    from stem_examples.stem_utils import vsetup_logging
+    if len(sys.argv) > 1:
+        IMAX = int(sys.argv[1])
+    else:
+        IMAX = 0
+    if os.environ.get('DEBUG', ''):
+        log_level = 10
+    else:
+        log_level = 20
+    vsetup_logging(LOG, log_level)
+    if len(sys.argv) > 1:
+        IMAX = int(sys.argv[1])
+        del sys.argv[1]
+    try:
         iMain()
-    print('Press Enter')
+        input()
-    input()  # wait for user to press enter
+        i = 0
+    except KeyboardInterrupt as e:
+        i = 0
+    except Exception as e:
+        LOG.exception(f"Exception {e}", exc_info=True)
+        i = 1
+    sys.exit(i)
 

src/stem_examples/interpreter.py

@@ -0,0 +1,57 @@
+#!/usr/local/bin/python3.sh
+# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*-
+__doc__ = """List Outdated Relays
+
+Time marches on. Tor makes new releases, and at some point needs to drop
+support for old ones. Below is the script we used on ticket 9476 to reach out
+to relay operators that needed to upgrade.
+
+https://stem.torproject.org/tutorials/examples/outdated_relays.html
+"""
+
+import os
+import sys
+import logging
+
+import stem.interpreter.commands
+from stem.descriptor.remote import DescriptorDownloader
+from stem_examples.tor_controller import get_controller
+from stem.version import Version
+
+from tor_controller import set_socks_proxy, unset_socks_proxy
+
+LOG = logging.getLogger()
+
+def iMain(lArgs=None):
+    if not lArgs:
+        lArgs = ['GETINFO' 'version']
+    password = os.environ.get('TOR_CONTROLLER_PASSWORD', '')
+    if os.path.exists('/run/tor/control'):
+        controller = get_controller(password=password, unix='/run/tor/control')
+    else:
+        controller = get_controller(password=password, port=9051)
+    interpreter = stem.interpreter.commands.ControlInterpreter(controller)
+    run_cmd = ' '.join(lArgs)
+    interpreter.run_command(run_cmd, print_response = True)
+    return 0
+
+if __name__ == '__main__':
+#    set_socks_proxy()
+    from stem_examples.stem_utils import vsetup_logging
+    if os.environ.get('DEBUG', ''):
+        log_level = 10
+    else:
+        log_level = 20
+    vsetup_logging(LOG, log_level)
+    try:
+        l = iMain(sys.argv[1:])
+        if l: print(l)
+        i = 0
+    except KeyboardInterrupt as e:
+        i = 0
+    except Exception as e:
+        LOG.exception(f"Exception {e}")
+        i = 1
+    finally:
+        unset_socks_proxy()
+    sys.exit(i)

src/stem_examples/introduction_points.py

@@ -22,65 +22,113 @@ import sys
 import os
 import getpass
 import logging
+import binascii
 
+import stem
 from stem.control import Controller
+from stem import Timeout
+from stem.client.datatype import LinkByFingerprint
+from stem.descriptor.hidden_service import HiddenServiceDescriptorV3
 
 from stem_examples.tor_controller import get_controller
+from stem.descriptor.hidden_service import HiddenServiceDescriptorV3
 
 LOG = logging.getLogger()
-
+TIMEOUT = 60
 lKNOWN_ONIONS = [
   'facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd', # facebook
   'duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad', # ddg
+  'zkaan2xfbuxia2wpf7ofnkbz6r5zdbbvxbunvp5g2iebopbfc4iqmbad', # keys.openpgp.org
+  'libera75jm6of4wxpxt4aynol3xjmbtxgfyjpu34ss4d7r7q2v5zrpyd',
+  'oftcnet6xg6roj6d7id4y4cu6dchysacqj2ldgea73qzdagufflqxrid',
 ]
 
+def bin_to_hex(raw_id:int, length: int|None = None) -> str:
+    if length is None: length = len(raw_id)
+    res = ''.join('{:02x}'.format(raw_id[i]) for i in range(length))
+    return res.upper()
+
 def iMain(lArgs=None):
     lRetval = lMain(lArgs)
     if lRetval is None:
         return -1
     return 0
 
-def lMain(lArgs=None):
+def lMain(lArgs=None, timeout=TIMEOUT) -> list:
     lRetval = []
-    if lArgs is None:
+    if not lArgs:
         lArgs = lKNOWN_ONIONS
     try:
+        password = os.environ.get('TOR_CONTROLLER_PASSWORD', '')
         if os.path.exists('/run/tor/control'):
-            controller = get_controller(unix='/run/tor/control')
+            controller = get_controller(password=password, unix='/run/tor/control')
         else:
-            controller = get_controller(port=9051)
+            controller = get_controller(password=password, port=9051)
-
-        password = os.environ.get('TOR_CONTROLLER_PASSWORD')
-        controller.authenticate(password)
 
         for elt in lArgs:
-            desc = controller.get_hidden_service_descriptor(elt, await_result=True, timeout=None)
+            LOG.info(f"onion: {elt}")
-            print(f"{desc} get_hidden_service_descriptor\n")
+            try:
-            l = desc.introduction_points()
+                desc = controller.get_hidden_service_descriptor(elt,
-            if l:
+                                                                await_result=True,
-                print(f"{elt} NO introduction points\n")
+                                                                timeout=timeout)
+            except Exception as e:
+                LOG.warn (f"{elt} EXCEPTION {e}")
                 continue
-            print(f"{elt} introduction points are...\n")
+            if desc.descriptor_id is None or desc.version is None:
+                # reparse as HSv3
+                inner_layer = HiddenServiceDescriptorV3.from_str(str(desc)).decrypt(elt)
+                if hasattr(inner_layer, 'introduction_points'):
+                    LOG.info (f"{elt} reparsed desc.decrypt len={len(desc.introduction_points())}")
 
+                    l = inner_layer.introduction_points
+                else:
+                    LOG.warn (f"{elt} reparsed desc.decrypt={dir(inner_layer)}")
+                    sys.exit(1)
+
+                #LOG.info(f"version: {desc.version}\n")
+                #LOG.info(f"lifetime: {desc.lifetime}\n")
+            else:
+                LOG.info(f"published: {desc.published}\n")
+                l = desc.introduction_points()
+            if not l:
+                LOG.warn(f"{elt} NO introduction points {l}\n")
+                continue
+
+            lp = []
             for introduction_point in l:
-                lRetval += [introduction_point]
+                for linkspecifier in introduction_point.link_specifiers:
-                print('  %s:%s => %s' % (introduction_point.address,
+                    # if isinstance(linkspecifier, LinkByFingerprint):
+                        # LOG.log(40, f"Getting fingerprint for {linkspecifier}")
+                    if hasattr(linkspecifier, 'fingerprint') and \
+                          len(linkspecifier.value) == 20:
+                            lp += [bin_to_hex(linkspecifier.value)]
+                    elif hasattr(introduction_point, 'address'):
+                        LOG.info('IP:  %s:%s => %s' % (introduction_point.address,
                                                 introduction_point.port,
                                                 introduction_point.identifier))
-
+                    else:
+                        pass # LOG.warn(f"{elt} introduction_point type={type(linkspecifier)}")
+            LOG.info(f"{elt} {len(lp)} introduction points {lp}")
     except Exception as e:
-        print(e)
+        LOG.exception(f"Exception: {e}")
     finally:
         del controller
     return lRetval
 
 if __name__ == '__main__':
-    LOG.setLevel(logging.INFO)
+    from stem_examples.stem_utils import vsetup_logging
+    if os.environ.get('DEBUG', ''):
+        log_level = 10
+    else:
+        log_level = 20
+    vsetup_logging(LOG, log_level)
     try:
-        i = iMain(sys.argv[1:])
+        l = lMain(sys.argv[1:])
+        if l: print('IPs: ', l)
+        i = 0
     except KeyboardInterrupt as e:
+        LOG.exception(f"Exception {e}")
         i = 0
     except Exception as e:
         i = 1
     sys.exit(i)
-

src/stem_examples/list_circuits.py

@@ -1,3 +1,4 @@
+#!/usr/local/bin/python3.sh
 # -*-mode: python; py-indent-offset: 2; indent-tabs-mode: nil; coding: utf-8-unix -*-
 
 # http://vt5hknv6sblkgf22.onion/tutorials/examples/list_circuits.html
@@ -14,7 +15,7 @@ def iMain():
     else:
         controller = get_controller(port=9051)
 
-    password = os.environ.get('TOR_CONTROLLER_PASSWORD')
+    password = os.environ.get('TOR_CONTROLLER_PASSWORD', '')
     try:
         controller.authenticate(password)
 

src/stem_examples/mappaddress.py

@@ -1,5 +1,15 @@
+#!/usr/local/bin/python3.sh
 # -*-mode: python; py-indent-offset: 2; indent-tabs-mode: nil; coding: utf-8-unix -*-
-# https://stem.torproject.org/tutorials/examples/exit_used.html
+#
+__doc__ = """
+Mappaddress queries the socks proxy with an onion address and returns the
+IP address that it will use for it. the address will be in the block specified
+by the VirtualAddrNetworkIPv4  setting of the torrc, e.g.
+
+VirtualAddrNetworkIPv4 172.16.0.0/12
+
+The script takes one argument, an onion address, without the .onion
+"""
 
 import functools
 import sys
@@ -8,22 +18,22 @@ import os
 
 from stem import StreamStatus
 from stem.control import EventType, Controller
-from tor_controller import set_socks_proxy
+
+# from tor_controller import set_socks_proxy, unset_socks_proxy
+
 from stem_examples.tor_controller import get_controller
 
 global LOG
 import logging
-LOG = logging.getLogger('map_address')
+LOG = logging.getLogger()
 
 def sMapaddressResolv(target, iPort=9051):
     try:
+        password = os.environ.get('TOR_CONTROLLER_PASSWORD', '')
         if os.path.exists('/run/tor/control'):
-            controller = get_controller(unix='/run/tor/control')
+            controller = get_controller(password=password, unix='/run/tor/control')
         else:
-            controller = get_controller(port=9051)
+            controller = get_controller(password=password, port=iPort)
-
-        password = os.environ.get('TOR_CONTROLLER_PASSWORD')
-        controller.authenticate(password)
 
         map_dict = {"0.0.0.0": target}
         map_ret = controller.map_address(map_dict)
@@ -33,9 +43,16 @@ def sMapaddressResolv(target, iPort=9051):
         LOG.exception(e)
 
 if __name__ == '__main__':
+    from stem_examples.stem_utils import vsetup_logging
+    if os.environ.get('DEBUG', ''):
+        log_level = 10
+    else:
+        log_level = 20
+    LOG = logging.getLogger()
+    vsetup_logging(LOG, log_level)
     if len(sys.argv) < 2:
         target = "l2ct3xnuaiwwtoybtn46qp2av4ndxcguwupzyv6xrsmnwi647vvmwtqd"
     else:
         target = sys.argv[1]
-    print(sMapaddressResolv(target))
+    LOG.info(sMapaddressResolv(target))
 

src/stem_examples/outdated_relays.py

@@ -9,36 +9,52 @@ to relay operators that needed to upgrade.
 https://stem.torproject.org/tutorials/examples/outdated_relays.html
 """
 
+import os
 import sys
 import logging
 
 from stem.descriptor.remote import DescriptorDownloader
 from stem.version import Version
 
-from tor_controller import set_socks_proxy
+from tor_controller import set_socks_proxy, unset_socks_proxy
 
 LOG = logging.getLogger()
 
-def iMain():
+def iMain(lArgs=None):
-    set_socks_proxy()
     downloader = DescriptorDownloader(use_mirrors=True)
     count, with_contact = 0, 0
     elts = downloader.get_server_descriptors()
-    print(f"Checking for outdated relays len server_descriptors={len(list(elts))}...")
+    LOG.info(f"Checking for outdated relays len server_descriptors={len(list(elts))}...")
-    print("")
 
     for desc in elts:
       if desc.tor_version < Version('0.2.3.0'):
         count += 1
 
         if desc.contact:
-          print('  %-15s %s' % (desc.tor_version, desc.contact.decode("utf-8", "replace")))
+          LOG.info('  %-15s %s' % (desc.tor_version, desc.contact.decode("utf-8", "replace")))
           with_contact += 1
 
-    print("")
+    LOG.info("%i outdated relays found, %i had contact information" % (count, with_contact))
-    print("%i outdated relays found, %i had contact information" % (count, with_contact))
+    # http://vt5hknv6sblkgf22.onion/tutorials/examples/outdated_relays.html
-    # http://vt5hknv6sblkgf22.onion/tutorials/examples/outdated_relays.htmlhttp://vt5hknv6sblkgf22.onion/tutorials/examples/outdated_relays.html
     return 0
 
 if __name__ == '__main__':
-    sys.exit(    iMain())
+    set_socks_proxy()
+    from stem_examples.stem_utils import vsetup_logging
+    if os.environ.get('DEBUG', ''):
+        log_level = 10
+    else:
+        log_level = 20
+    vsetup_logging(LOG, log_level)
+    try:
+        l = iMain([])
+        if l: print(l)
+        i = 0
+    except KeyboardInterrupt as e:
+        i = 0
+    except Exception as e:
+        LOG.exception(f"Exception {e}")
+        i = 1
+    finally:
+        unset_socks_proxy()
+    sys.exit(i)

src/stem_examples/relay_connections.py

@@ -1,3 +1,4 @@
+#!/usr/local/bin/python3.sh
 # -*-mode: python; py-indent-offset: 2; indent-tabs-mode: nil; coding: utf-8-unix -*-
 
 # https://stem.torproject.org/tutorials/examples/relay_connections.html
@@ -20,15 +21,14 @@ import stem.connection
 import stem.util.system
 import stem.util.str_tools
 
-from stem.control import Listener
+from stem.control import Listener, Controller
-from stem.control import Controller
 from stem.util.connection import get_connections, port_usage, is_valid_ipv4_address
 
 from stem_examples.tor_controller import get_controller
 
-global LOG
 import logging
-LOG = logging.getLogger('relay_cons')
+global LOG
+LOG = logging.getLogger()
 
 HEADER_LINE = " {version}   uptime: {uptime}   flags: {flags}\n"
 
@@ -52,15 +52,12 @@ def iMain(lArgs=None):
     parser.add_argument("--resolver", help="default: autodetected")
     args = parser.parse_args(lArgs)
 
+    password = os.environ.get('TOR_CONTROLLER_PASSWORD', '')
     control_port = int(args.ctrlport) if args.ctrlport else 'default'
-  if os.path.exists('/run/tor/control'):
+    if False and os.path.exists('/run/tor/control'):
-      controller = get_controller(unix='/run/tor/control')
+        controller = get_controller(password=password, unix='/run/tor/control')
     else:
-      controller = get_controller(port=control_port)
+        controller = get_controller(password=password, port=control_port)
-#  controller = stem.connection.connect(control_port = ('127.0.0.1', control_port))
-
-  password = os.environ.get('TOR_CONTROLLER_PASSWORD')
-  controller.authenticate(password)
 
     if not controller:
       return 1
@@ -70,7 +67,7 @@ def iMain(lArgs=None):
     version = str(controller.get_version()).split()[0],
     uptime = stem.util.str_tools.short_time_label(time.time() - stem.util.system.start_time(pid))
 
-  print(HEADER_LINE.format(
+    LOG.info(HEADER_LINE.format(
       version=version,
       uptime=uptime,
       flags = ', '.join(desc.flags if desc else ['none']),
@@ -157,7 +154,23 @@ def iMain(lArgs=None):
       print(DIV)
       print('')
 
-
 if __name__ == '__main__':
-    iMain()
+    from stem_examples.stem_utils import vsetup_logging
+    LOG = logging.getLogger()
+    if os.environ.get('DEBUG', ''):
+        log_level = 10
+    else:
+        log_level = 20
+    vsetup_logging(LOG, log_level)
+    LOG.setLevel(logging.DEBUG)
+    try:
+        l = iMain([])
+        if l: print(l)
+        i = 0
+    except KeyboardInterrupt as e:
+        i = 0
+    except Exception as e:
+        LOG.exception(f"Exception {e}")
+        i = 1
+    sys.exit(i)
 

src/stem_examples/stem_utils.py

@@ -0,0 +1,58 @@
+# -*-mode: python; py-indent-offset: 2; indent-tabs-mode: nil; coding: utf-8-unix -*-
+
+import sys
+import logging
+
+try:
+#    if 'COLOREDLOGS_LEVEL_STYLES' not in os.environ:
+#        os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red'
+    # https://pypi.org/project/coloredlogs/
+    import coloredlogs
+except ImportError:
+    coloredlogs = False
+
+def vsetup_logging(theLOG, log_level, logfile='', stream=sys.stdout) -> None:
+    global LOG
+    LOG = theLOG
+    add = True
+
+    logging._defaultFormatter = logging.Formatter(datefmt='%m-%d %H:%M:%S')
+    logging._defaultFormatter.default_time_format = '%m-%d %H:%M:%S'
+    logging._defaultFormatter.default_msec_format = ''
+
+    if logfile:
+        add = logfile.startswith('+')
+        sub = logfile.startswith('-')
+        if add or sub:
+            logfile = logfile[1:]
+        kwargs['filename'] = logfile
+
+    if coloredlogs:
+        coloredlogs.DEFAULT_LEVEL_STYLES['info']=dict(color='white',bold=True)
+        coloredlogs.DEFAULT_LEVEL_STYLES['debug']=dict(color='cyan')
+        coloredlogs.DEFAULT_LEVEL_STYLES['warn']=dict(color='yellow',bold=True)
+        coloredlogs.DEFAULT_LEVEL_STYLES['error']=dict(color='red',bold=True)
+        coloredlogs.DEFAULT_FIELD_STYLES['levelname=']=dict(color='green', bold=True),
+        # https://pypi.org/project/coloredlogs/
+        aKw = dict(level=log_level,
+                   logger=LOG,
+                   stream=stream,
+                   fmt='%(levelname)s %(message)s',
+                   isatty=True,
+                   milliseconds=False,
+                   )
+        coloredlogs.install(**aKw)
+        if logfile:
+            oHandler = logging.FileHandler(logfile)
+            LOG.addHandler(oHandler)
+        LOG.debug(f"Setting coloured log_level to {log_level}")
+    else:
+        kwargs = dict(level=log_level,
+                    force=True,
+                    format='%(levelname)s %(message)s')
+        logging.basicConfig(**kwargs)
+        if add and logfile:
+            oHandler = logging.StreamHandler(stream)
+            LOG.addHandler(oHandler)
+        LOG.debug(f"SSetting log_level to {log_level}")
+

src/stem_examples/tor_bootstrap_check.py

@@ -12,21 +12,21 @@ tor is at.
 import sys
 import os
 import re
+import logging
+import socket
 
 from stem.connection import connect
 from stem_examples.tor_controller import get_controller
 
+LOG = logging.getLogger()
+
 def iMain(lArgs=None):
-    password = os.environ.get('TOR_CONTROLLER_PASSWORD')
+    password = os.environ.get('TOR_CONTROLLER_PASSWORD', '')
     if os.path.exists('/run/tor/control'):
         controller = get_controller(password=password, unix='/run/tor/control')
     else:
         controller = get_controller(password=password, port=9051)
 
-#    controller.connect()
-
-    bootstrap_status = controller.get_info("status/bootstrap-phase")
-
     ## Possible answer, if network cable has been removed:
     ## 250-status/bootstrap-phase=WARN BOOTSTRAP PROGRESS=80 TAG=conn_or SUMMARY="Connecting to the Tor network" WARNING="No route to host" REASON=NOROUTE COUNT=26 RECOMMENDATION=warn
 
@@ -38,15 +38,33 @@ def iMain(lArgs=None):
 
     ## TODO: parse the messages above.
 
-    print(format(bootstrap_status))
+    try:
+        bootstrap_status = controller.get_info("status/bootstrap-phase")
+        LOG.info(format(bootstrap_status))
 
         progress_percent = re.match('.* PROGRESS=([0-9]+).*', bootstrap_status)
         exit_code = int(progress_percent.group(1))
 
         controller.close()
-
+    except socket.error as e:
-    return exit_code
+        # Error while receiving a control message (SocketClosed): received exception "read of closed file"
+        return 0
+    except Exception as e:
+        raise
+    return 0
 
 if __name__ == '__main__':
-    sys.exit(    iMain())
+    from stem_examples.stem_utils import vsetup_logging
-
+    if os.environ.get('DEBUG', ''):
+        log_level = 10
+    else:
+        log_level = 20
+    vsetup_logging(LOG, log_level)
+    try:
+        i = iMain([])
+    except KeyboardInterrupt as e:
+        i = 0
+    except Exception as e:
+        LOG.exception(f"Exception {e}")
+        i = 1
+    sys.exit(i)

src/stem_examples/tor_controller.py

@@ -12,7 +12,7 @@ def set_socks_proxy(SOCKS5_PROXY_HOST='127.0.0.1', SOCKS5_PROXY_PORT=9050):
     try:
         import socks # you need to install pysocks (see above)
         # Remove this if you don't plan to "deactivate" the proxy later
-        default_socket = socket.socket
+        socks._socket = socket.socket
 
         # Set up a proxy
         socks.set_default_proxy(socks.SOCKS5, SOCKS5_PROXY_HOST, SOCKS5_PROXY_PORT)
@@ -21,6 +21,14 @@ def set_socks_proxy(SOCKS5_PROXY_HOST='127.0.0.1', SOCKS5_PROXY_PORT=9050):
         return False
     return True
 
+def unset_socks_proxy():
+    import socks # you need to install pysocks (see above)
+    # Remove this if you don't plan to "deactivate" the proxy later
+    socks.socket = socket._socket
+
+    # Set up a proxy
+    socks.set_default_proxy(socks.SOCKS5, None, None)
+
 def get_controller(password=None, address='127.0.0.1', port=9051, unix='/run/tor/control'):
     if unix and os.path.exists(unix):
         #  print(unix)
@@ -31,7 +39,7 @@ def get_controller(password=None, address='127.0.0.1', port=9051, unix='/run/tor
 
     if password is None:
 #        print("DBUG: trying TOR_CONTROLLER_PASSWORD")
-        password = os.environ.get('TOR_CONTROLLER_PASSWORD')
+        password = os.environ.get('TOR_CONTROLLER_PASSWORD', '')
     else:
 #       print(f"DBUG: using a password {len(password)}")
         pass

stem_examples.txt

@@ -3,28 +3,75 @@
 == stem_examples tor testing ==
 
 This is a Python doctest file that is executable documentation.
+stem_examples is a set of small scripts that tell you about your
+running tor instance.
 
-Pass the controller password if needed as an environment variable:
+Onionoo is a web-based protocol to learn about currently running Tor
+relays and bridges. Onionoo itself was not designed as a service for
+human beings---at least not directly. Onionoo provides the data for
+other applications and websites which in turn present Tor network
+status information to humans: https://metrics.torproject.org/onionoo.html
+
+You can see the status of tor relays at https://torstatus.rueckgr.at/
+The code for that site is at https://github.com/paulchen/torstatus
+You can get a list of exit relays that are marked bad with:
+wget --post-data='SR=FBadExit&SO=Asc&FBadExit=1' 'https://torstatus.rueckgr.at/'
+
+It is assumed that you are running a tor that has its torrc configured with:
+
+ControlPort 127.0.0.1:9051
+
+and/or
+
+ControlSocket /run/tor/control
+ControlSocketsGroupWritable 1
+
+We can authenticate with a password. To set a password first get its hash...
+
+% tor --hash-password "my_password"
+16:E600ADC1B52C80BB6022A0E999A7734571A451EB6AE50FED489B72E3DF
+
+and use that for the HashedControlPassword in your torrc.
+
+HashedControlPassword 16:E600ADC1B52C80BB6022A0E999A7734571A451EB6AE50FED489B72E3DF
+
+so that you have some security on the Control connection.
+Pass the controller password to these scripts as an environment variable:
 
     >>> import os
     >>> assert os.environ['TOR_CONTROLLER_PASSWORD']
     
+If you are using /run/tor/control you will also need to run the scripts as the user
+that has rw access to that socket, usually tor or debian-tor.
+
 Add our code to the PYTHONPATH 
     >>> import sys
     >>> sys.path.append(os.path.join(os.getcwd(), 'src', 'stem_examples'))
 
-We'll need the settings defined in  {{{/usr/local/etc/testforge/testforge.yml}}}
+We'll used the settings defined in  {{{/usr/local/etc/testforge/testforge.yml}}}
+If you don't have one, make it with the settings from your torrc:
 
     >>> print("yaml", file=sys.stderr)
     >>> import yaml
-    >>> sFacts = open('/usr/local/etc/testforge/testforge.yml').read()
+    >>> try:
-    >>> assert sFacts
+    ...     sFacts = open('/usr/local/etc/testforge/testforge.yml').read()
-    >>> dFacts = yaml.safe_load(sFacts)
+    ...     assert sFacts, sFacts 
+    ... except:
+    ...     dFacts = dict(
+    ...       HTTPS_PROXYHOST="127.0.0.1",
+    ...       HTTPS_PROXYPORT=9128,
+    ...       HTTPS_PROXYTYPE="http",
+    ...       SOCKS_PROXYHOST="127.0.0.1",
+    ...       SOCKS_PROXYPORT=9050,
+    ...       SOCKS_PROXYTYPE="socks5",
+    ...     )
+    ... else:
+    ...     dFacts = yaml.safe_load(sFacts)
+
  
 FixMe: use the settings for the ports and directories below.
 
     >>> import os
-    >>> os.environ['http_proxy']  = 'http://'+dFacts['HTTP_PROXYHOST']+':'+str(dFacts['HTTP_PROXYPORT'])
     >>> os.environ['https_proxy'] = 'http://'+dFacts['HTTPS_PROXYHOST']+':'+str(dFacts['HTTPS_PROXYPORT'])
     >>> os.environ['socks_proxy']  = 'socks5://'+dFacts['SOCKS_PROXYHOST']+':'+str(dFacts['SOCKS_PROXYPORT'])
 
@@ -40,13 +87,14 @@ We test 3 known hidden services: Facebook, DuckDuckGo and .
    >>> lKNOWN_ONIONS = [
    ...  'facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd', # facebook
    ...  'duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad', # ddg
-   ...  'zkaan2xfbuxia2wpf7ofnkbz6r5zdbbvxbunvp5g2iebopbfc4iqmbad', # hks
    ... ]
 
 We wil expect to get back the hidden service version, the descriptor-lifetime
 and then the descriptor-signing-key-cert:
 
     >>> introduction_points.iMain(lKNOWN_ONIONS) #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+    0
+
     hs-descriptor 3
     descriptor-lifetime ...
         <BLANKLINE>
@@ -59,22 +107,34 @@ How can you figure out what exit you're using?
 
     >>> print("exit_used", file=sys.stderr)
     >>> import exit_used
+    >>> exit_used.iMain([])
 
 ## relay_connections Connection Summary
 
     >>> print("relay_connections", file=sys.stderr)
     >>> import relay_connections
+    >>> relay_connections.iMain([]) #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+    +------------------------------+------+------+
+    ...
+    +------------------------------+------+------+
+    <BLANKLINE>
 
 The following provides a summary of your relay's inbound and outbound connections.
 You must be root or tor to run this:
 relay_connections.iMain(["--ctrlport", "9051"])
 
-## outdated_relays
+## connection_resolution  Connection Resolution 
 
-    >>> print("outdated_relays", file=sys.stderr)
+Connection information is a useful tool for learning more about network
-    >>> import outdated_relays
+applications like Tor. Our stem.util.connection.get_connections() function
-    >>> outdated_relays.iMain() #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+provides an easy method for accessing this information.
-    Checking for outdated relays ...
+
+    >>> print("connection_resolution", file=sys.stderr)
+    >>> import connection_resolution
+    >>> connection_resolution.iMain([]) #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+    0
+    
+    INFO Our platform supports connection resolution via: ...
     <BLANKLINE>
 
 ## tor_bootstrap_check
@@ -85,18 +145,46 @@ relay_connections.iMain(["--ctrlport", "9051"])
 A script by adrelanos@riseup.net to check what percentage of boostrapping
 tor is at. This fails under doctest but not from the cmdline
 
->> tor_bootstrap_check.iMain() #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+    >>> tor_bootstrap_check.iMain([]) #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
-NOTICE ...
+    0
-<BLANKLINE>
     
-    control_port = stem.socket.ControlPort(address, port)
+    NOTICE ...
-                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    <BLANKLINE>
-  File "/usr/local/lib/python3.11/site-packages/stem/socket.py", line 503, in __init__
+
-    self.connect()
+## check_digests
-  File "/usr/local/lib/python3.11/site-packages/stem/socket.py", line 172, in connect
+
-    self._socket = self._make_socket()
+    >>> print("check_digests", file=sys.stderr)
-                   ^^^^^^^^^^^^^^^^^^^
+    >>> from check_digests import iMain
-  File "/usr/local/lib/python3.11/site-packages/stem/socket.py", line 538, in _make_socket
+    >>> sKNOWN_ONION = 'facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd' # facebook
-    raise stem.SocketError(exc)
+    >>> from stem_examples.introduction_points import lMain as lIPMain
-stem.SocketError: Socket error: 0x01: General SOCKS server failure
+    >>> lArgs = []
+    >>> import stem_examples.support_testing as ts; with ts.ignoreStdout():
+    ...    lArgs = lIPMain([sKNOWN_ONION])
+    >>> iMain(lArgs)
+    0
+    
+## interpreter
+
+    >>> print("interpreter", file=sys.stderr)
+    >>> import interpreter
+    >>> lArgs = ['GETINFO', 'version']
+    >>> interpreter.iMain(lArgs) #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+    250-version=0.4.8.10
+    250 OK
+    <BLANKLINE>
+    0
+
+## outdated_relays List Outdated Relays
+
+Time marches on. Tor makes new releases, and at some point needs to drop
+support for old ones. Below is the script we used on ticket 9476 to reach out
+to relay operators that needed to upgrade.
+
+    >>> print("outdated_relays", file=sys.stderr)
+    >>> import outdated_relays
+    >>> outdated_relays.iMain([]) #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+    0
+    
+    Checking for outdated relays ...
+    <BLANKLINE>