exclude_badExits/exclude_badExits.py

# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*-

# https://github.com/nusenu/noContactInfo_Exit_Excluder
# https://github.com/TheSmashy/TorExitRelayExclude
"""
This extends nusenu's basic idea of using the stem library to
dynamically exclude nodes that are likely to be bad by putting them
on the ExcludeNodes or ExcludeExitNodes setting of a running Tor.
* https://github.com/nusenu/noContactInfo_Exit_Excluder
* https://github.com/TheSmashy/TorExitRelayExclude

The basic cut is to exclude Exit nodes that do not have a contact.
That can be extended to nodes that do not have an email in the contact etc.
"""
"""But there's a problem, and your Tor notice.log will tell you about it:
you could exclude the nodes needed to access hidden services or
directorues. So we need to add to the process the concept of a whitelist.
In addition, we may have our own blacklist of nodes we want to exclude,
or use these lists for other applications like selektor.

So we make two files that are structured in YAML:
```
/etc/tor/yaml/torrc-goodnodes.yaml
GoodNodes:
  Relays:
    IntroductionPoints:
      - NODEFINGERPRINT
  ...
By default all sections of the goodnodes.yaml are used as a whitelist.

/etc/tor/yaml/torrc-badnodes.yaml
BadNodes:
  ExcludeExitNodes:
    BadExit:
      # $0000000000000000000000000000000000000007
```
That part requires [PyYAML](https://pyyaml.org/wiki/PyYAML)
https://github.com/yaml/pyyaml/

Right now only the ExcludeExitNodes section is used by we may add ExcludeNodes
later, and by default all sub-sections of the badnodes.yaml are used as a
ExcludeExitNodes but it can be customized with the lWanted commandline arg.

The original idea has also been extended to add different conditions for
exclusion: the ```--contact``` commandline arg is a comma sep list of conditions:
* Empty - no contact info
* NoEmail - no @ sign in the contact',
More may be added later.

Because you don't want to exclude the introduction points to any onion
you want to connect to, ```--white_onions``` should whitelist the
introduction points to a  comma sep list of onions, but is
currently broken in stem 1.8.0: see:
* https://github.com/torproject/stem/issues/96
* https://gitlab.torproject.org/legacy/trac/-/issues/25417

```--torrc_output``` will write the torrc ExcludeNodes configuration to a file.

Now for the final part: we lookup the Contact info of every server
that is currently in our Tor, and check it for its existence.
If it fails to provide the well-know url, we assume its a bogus
relay and add it to a list of nodes that goes on ExcludeNodes -
not just exclude Exit.

If the Contact info is good we add the list of fingerprints to add
to ExitNodes, a whitelist of relays to use as exits.

```--good_contacts``` will write the contact info as a ciiss dictionary
to a YAML file. If the proof is uri-rsa, the well-known file of fingerprints
is downloaded and the fingerprints are added on a 'fps' field we create
of that fingerprint's entry of the YAML dictionary. This file is read at the
beginning of the program to start with a trust database, and only new
contact info from new relays are added to the dictionary.

You can expect it to take an hour or two the first time this is run:
>700 domains.

For usage, do ```python3 exclude_badExits.py --help`

"""

import argparse
import os
import sys
import time
from io import StringIO

import stem
import urllib3
from stem import InvalidRequest
from stem.connection import IncorrectPassword
from stem.util.tor_tools import is_valid_fingerprint
from urllib3.util.ssl_match_hostname import CertificateError

# list(ipaddress._find_address_range(ipaddress.IPv4Network('172.16.0.0/12'))

try:
    from ruamel.yaml import YAML
    yaml = YAML(typ='rt')
    yaml.indent(mapping=2, sequence=2)
    safe_load = yaml.load
except:
    yaml = None
if yaml is None:
    try:
        import yaml
        safe_load = yaml.safe_load
    except:
        yaml = None

try:
    from unbound import RR_CLASS_IN, RR_TYPE_TXT, ub_ctx
except:
    ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None

global LOG
import logging
import warnings

warnings.filterwarnings('ignore')
LOG = logging.getLogger()

from support_onions import (bAreWeConnected, icheck_torrc, lIntroductionPoints,
                            oGetStemController, vwait_for_controller,
                            yKNOWN_NODNS, zResolveDomain)
from support_phantompy import vsetup_logging
from trustor_poc import TrustorError, idns_validate
from trustor_poc import oDownloadUrlUrllib3 as oDownloadUrl

LOG.info("imported HTTPSAdapter")

ETC_DIR = '/etc/tor/yaml'
aTRUST_DB = {}
aTRUST_DB_INDEX = {}
aFP_EMAIL = {}
sDETAILS_URL = "https://metrics.torproject.org/rs.html#details/"
# You can call this while bootstrapping
sEXCLUDE_EXIT_KEY = 'ExcludeNodes'
sINCLUDE_EXIT_KEY = 'ExitNodes'
sINCLUDE_GUARD_KEY = 'EntryNodes'

oBAD_NODES = {}
oBAD_ROOT = 'BadNodes'
oBAD_NODES[oBAD_ROOT] = {}
oBAD_NODES[oBAD_ROOT]['ExcludeNodes'] = {}

lKNOWN_NODNS = []
lMAYBE_NODNS = []
def lYamlBadNodes(sFile,
                  section=sEXCLUDE_EXIT_KEY,
                  lWanted=['BadExit']):
    global oBAD_NODES
    global lKNOWN_NODNS
    global lMAYBE_NODNS

    if not yaml:
        return []
    if os.path.exists(sFile):
        with open(sFile, 'rt') as oFd:
            oBAD_NODES = safe_load(oFd)

    # BROKEN
#    root = 'ExcludeNodes'
#        for elt in o[oBAD_ROOT][root][section].keys():
#            if lWanted and elt not in lWanted: continue
#            # l += o[oBAD_ROOT][root][section][elt]

    l = oBAD_NODES[oBAD_ROOT]['ExcludeNodes']['BadExit']

    root = 'ExcludeDomains'
    if root not in oBAD_NODES[oBAD_ROOT] or not oBAD_NODES[oBAD_ROOT][root]:
        lMAYBE_NODNS = safe_load(StringIO(yKNOWN_NODNS))
    else:
        lMAYBE_NODNS = oBAD_NODES[oBAD_ROOT][root]
    return l

oGOOD_NODES = {}
oGOOD_ROOT = 'GoodNodes'
def lYamlGoodNodes(sFile='/etc/tor/torrc-goodnodes.yaml'):
    global oGOOD_NODES
    l = []
    if not yaml: return l
    if os.path.exists(sFile):
        with open(sFile, 'rt') as oFd:
            o = safe_load(oFd)
            oGOOD_NODES = o
        if 'GuardNodes' in o[oGOOD_ROOT].keys():
            l = o[oGOOD_ROOT]['GuardNodes']
    # yq '.Nodes.IntroductionPoints|.[]' < /etc/tor/torrc-goodnodes.yaml
    return l

def bdomain_is_bad(domain, fp):
    global lKNOWN_NODNS
    if domain in lKNOWN_NODNS: return True
    if domain in lMAYBE_NODNS:
        ip = zResolveDomain(domain)
        if ip == '':
            LOG.debug(f"{fp} {domain} does not resolve")
            lKNOWN_NODNS.append(domain)
            lMAYBE_NODNS.remove(domain)
            return True

    for elt in '@(){}$!':
        if elt in domain:
            LOG.warn(f"{elt} in domain {domain}")
            return True
    return False

tBAD_URLS = set()
lATS = ['abuse', 'email']
lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime', 'memory']
lBOOLS = ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone',
          'sandbox', 'offlinemasterkey']
def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050):
    global tBAD_URLS
    global lKNOWN_NODNS
    # cleanups
    for elt in lINTS:
        if elt in a:
            a[elt] = int(a[elt])
    for elt in lBOOLS:
        if elt in a:
            if a[elt] in ['y', 'yes', 'true', 'True']:
                a[elt] = True
            else:
                a[elt] = False
    for elt in lATS:
        if elt in a:
            a[elt] = a[elt].replace('[]', '@')

    a.update({'fps': []})
    keys = list(a.keys())
    if 'email' not in keys:
        LOG.warn(f"{fp} 'email' not in {keys}")
        a['email'] = ''
    if 'ciissversion' not in keys:
        aFP_EMAIL[fp] = a['email']
        LOG.warn(f"{fp} 'ciissversion' not in {keys}")
        a['ciissversion'] = 2
    # test the url for fps and add it to the array
    if 'proof' not in keys:
        aFP_EMAIL[fp] = a['email']
        LOG.warn(f"{fp} 'proof' not in {keys}")
        return a

    if aTRUST_DB_INDEX and fp in aTRUST_DB_INDEX.keys():
        aCachedContact = aTRUST_DB_INDEX[fp]
        if aCachedContact['email'] == a['email']:
            LOG.info(f"{fp} in aTRUST_DB_INDEX")
            return aCachedContact

    if 'url' not in keys:
        if 'uri' not in keys:
            a['url'] = ''
            aFP_EMAIL[fp] = a['email']
            LOG.warn(f"{fp} url and uri not in {keys}")
            return a
        a['url'] = a['uri']
        aFP_EMAIL[fp] = a['email']
        LOG.debug(f"{fp} 'uri' but not 'url' in {keys}")
        # drop through

    c = a['url'].lstrip('https://').lstrip('http://').strip('/')
    a['url'] = 'https://' +c

    # domain should be a unique key for contacts
    domain = a['url'][8:]
    if bdomain_is_bad(domain, fp):
        LOG.warn(f"{domain} is bad from {a['url']}")
        LOG.debug(f"{fp} is bad from {a}")
        return a

    ip = zResolveDomain(domain)
    if ip == '':
        aFP_EMAIL[fp] = a['email']
        LOG.debug(f"{fp} {domain} does not resolve")
        lKNOWN_NODNS.append(domain)
        return {}

    if a['proof'] not in ['uri-rsa']:
        # only support uri for now
        if False and ub_ctx:
            fp_domain = fp + '.' + domain
            if idns_validate(fp_domain,
                             libunbound_resolv_file='resolv.conf',
                             dnssec_DS_file='dnssec-root-trust',
                             ) == 0:
                pass
        LOG.warn(f"{fp} proof={a['proof']} not supported yet")
        return a

    LOG.debug(f"{len(keys)} contact fields for {fp}")
    url = f"https://{domain}/.well-known/tor-relay/rsa-fingerprint.txt"
    try:
        LOG.debug(f"Downloading from {domain} for {fp}")
        o = oDownloadUrl(url, https_cafile,
                         timeout=timeout, host=host, port=port)
        # requests response: text "reason", "status_code"
    except AttributeError as e:
        LOG.exception(f"AttributeError downloading from {domain} {e}")
    except CertificateError as e:
        LOG.warn(f"CertificateError downloading from {domain} {e}")
        tBAD_URLS.add(a['url'])
    except TrustorError as e:
        if e.args == "HTTP Errorcode 404":
            aFP_EMAIL[fp] = a['email']
            LOG.warn(f"TrustorError 404 from {domain} {e.args}")
        else:
            LOG.warn(f"TrustorError downloading from {domain} {e.args}")
        tBAD_URLS.add(a['url'])
    except urllib3.exceptions.MaxRetryError as e: # noqa
        # maybe offline - not bad
        LOG.warn(f"MaxRetryError downloading from {domain} {e}")
    except (BaseException) as e:
        LOG.error(f"Exception {type(e)} downloading from {domain} {e}")
    else:
        if hasattr(o, 'status'):
            status_code = o.status
        else:
            status_code = o.status_code
        if status_code >= 300:
            aFP_EMAIL[fp] = a['email']
            LOG.warn(f"Error from {domain} {status_code} {o.reason}")
            # any reason retry?
            tBAD_URLS.add(a['url'])
            return a

        if hasattr(o, 'text'):
            data = o.text
        else:
            data = str(o.data, 'UTF-8')
        l = data.upper().strip().split('\n')
        LOG.debug(f"Downloaded from {domain} {len(l)} lines {len(data)} bytes")

        a['modified'] = int(time.time())
        if not l:
            LOG.warn(f"Downloading from {domain} empty for {fp}")
        else:
            a['fps'] = [elt for elt in l if elt and len(elt) == 40 \
                        and not elt.startswith('#')]
            LOG.info(f"Downloaded from {domain} {len(a['fps'])} FPs")
    return a

def aParseContactYaml(contact, fp):
    """
    See the Tor ContactInfo Information Sharing Specification v2
    https://nusenu.github.io/ContactInfo-Information-Sharing-Specification/
    """
    lelts = contact.split()
    a = {}
    if len(lelts) % 1 != 0:
        LOG.warn(f"bad contact for {fp} odd number of components")
        LOG.debug(f"{fp} {a}")
        return a
    key = ''
    for elt in lelts:
        if key == '':
            key = elt
            continue
        a[key] = elt
        key = ''
    LOG.debug(f"{fp} {len(a.keys())} fields")
    return a

def aParseContact(contact, fp):
    """
    See the Tor ContactInfo Information Sharing Specification v2
    https://nusenu.github.io/ContactInfo-Information-Sharing-Specification/
    """
    l = [line for line in contact.strip().replace('"', '').split(' ')
         if ':' in line]
    LOG.debug(f"{fp} {len(l)} fields")
    s = f'"{fp}":\n'
    s += '\n'.join([f"  {line}\"".replace(':', ': \"', 1)
                    for line in l])
    oFd = StringIO(s)
    a = safe_load(oFd)
    return a

def oMainArgparser(_=None):

    try:
        from OpenSSL import SSL
        lCAfs = SSL._CERTIFICATE_FILE_LOCATIONS
    except:
        lCAfs = []

    CAfs = []
    for elt in lCAfs:
        if os.path.exists(elt):
            CAfs.append(elt)
    if not CAfs:
        CAfs = ['']

    parser = argparse.ArgumentParser(add_help=True,
                                     epilog=__doc__)
    parser.add_argument('--https_cafile', type=str,
                        help="Certificate Authority file (in PEM)",
                        default=CAfs[0])
    parser.add_argument('--proxy_host', '--proxy-host', type=str,
                        default='127.0.0.1',
                        help='proxy host')
    parser.add_argument('--proxy_port', '--proxy-port', default=9050, type=int,
                        help='proxy control port')
    parser.add_argument('--proxy_ctl', '--proxy-ctl',
                        default='/run/tor/control' if os.path.exists('/run/tor/control') else 9051,
                        type=str,
                        help='control socket - or port')

    parser.add_argument('--torrc',
                        default='/etc/tor/torrc-defaults',
                        type=str,
                        help='torrc to check for suggestions')
    parser.add_argument('--timeout', default=60, type=int,
                        help='proxy download connect timeout')

    parser.add_argument('--good_nodes', type=str,
                        default=os.path.join(ETC_DIR, 'goodnodes.yaml'),
                        help="Yaml file of good info that should not be excluded")
    parser.add_argument('--bad_nodes', type=str,
                        default=os.path.join(ETC_DIR, 'badnodes.yaml'),
                        help="Yaml file of bad nodes that should also be excluded")
    parser.add_argument('--contact', type=str, default='Empty,NoEmail',
                        help="comma sep list of conditions - Empty,NoEmail")
    parser.add_argument('--bad_contacts', type=str,
                        default=os.path.join(ETC_DIR, 'badcontacts.yaml'),
                        help="Yaml file of bad contacts that bad FPs are using")

    parser.add_argument('--strict_nodes', type=int, default=0, choices=[0, 1],
                        help="Set StrictNodes: 1 is less anonymous but more secure, although some sites may be unreachable")
    parser.add_argument('--wait_boot', type=int, default=120,
                        help="Seconds to wait for Tor to booststrap")
    parser.add_argument('--points_timeout', type=int, default=0,
                        help="Timeout for getting introduction points - must be long >120sec. 0 means disabled looking for IPs")
    parser.add_argument('--log_level', type=int, default=20,
                        help="10=debug 20=info 30=warn 40=error")
    parser.add_argument('--bad_sections', type=str,
                        default='MyBadExit',
                        help="sections of the badnodes.yaml to use, comma separated, '' BROKEN")
    parser.add_argument('--white_onions', type=str,
                        default='',
                        help="comma sep. list of onions to whitelist their introduction points - BROKEN")
    parser.add_argument('--torrc_output', type=str,
                        default=os.path.join(ETC_DIR, 'torrc.new'),
                        help="Write the torrc configuration to a file")
    parser.add_argument('--good_contacts', type=str, default=os.path.join(ETC_DIR, 'goodcontacts.yaml'),
                        help="Write the proof data of the included nodes to a YAML file")
    return parser

def vwrite_badnodes(oargs, oBAD_NODES, slen):
    if oargs.bad_nodes:
        tmp = oargs.bad_nodes +'.tmp'
        bak = oargs.bad_nodes +'.bak'
        with open(tmp, 'wt') as oFYaml:
            yaml.dump(oBAD_NODES, oFYaml)
            LOG.info(f"Wrote {slen}  to {oargs.bad_nodes}")
            oFYaml.close()
        if os.path.exists(oargs.bad_nodes):
            os.rename(oargs.bad_nodes, bak)
        os.rename(tmp, oargs.bad_nodes)

def vwrite_goodnodes(oargs, oGOOD_NODES, ilen):
    if oargs.good_nodes:
        tmp = oargs.good_nodes +'.tmp'
        bak = oargs.good_nodes +'.bak'
        with open(tmp, 'wt') as oFYaml:
            yaml.dump(oGOOD_NODES, oFYaml)
            LOG.info(f"Wrote {ilen} good relays to {oargs.good_nodes}")
            oFYaml.close()
        if os.path.exists(oargs.good_nodes):
            os.rename(oargs.good_nodes, bak)
        os.rename(tmp, oargs.good_nodes)

def iMain(lArgs):
    global aTRUST_DB
    global aTRUST_DB_INDEX
    global oBAD_NODES
    global oGOOD_NODES
    global lKNOWN_NODNS
    parser = oMainArgparser()
    oargs = parser.parse_args(lArgs)

    vsetup_logging(oargs.log_level)
    if bAreWeConnected() is False:
        raise SystemExit("we are not connected")

    sFile = oargs.torrc
    if sFile and os.path.exists(sFile):
        icheck_torrc(sFile, oargs)

    twhitelist_set = set()
    sFile = oargs.good_contacts
    if sFile and os.path.exists(sFile):
        try:
            with open(sFile, 'rt') as oFd:
                aTRUST_DB = safe_load(oFd)
            LOG.info(f"{len(aTRUST_DB.keys())} trusted contacts from {sFile}")
            # reverse lookup of fps to contacts
            # but...
            for (k, v,) in aTRUST_DB.items():
                if 'modified' not in v.keys():
                    v['modified'] = int(time.time())
                aTRUST_DB_INDEX[k] = v
                if 'fps' in aTRUST_DB[k].keys():
                    for fp in aTRUST_DB[k]['fps']:
                        if fp in aTRUST_DB_INDEX:
                            continue
                        aTRUST_DB_INDEX[fp] = v
            LOG.info(f"{len(aTRUST_DB_INDEX.keys())} good relays from {sFile}")

        except Exception as e:
            LOG.exception(f"Error reading YAML TrustDB {sFile} {e}")

    if os.path.exists(oargs.proxy_ctl):
        controller = oGetStemController(log_level=oargs.log_level, sock_or_pair=oargs.proxy_ctl)
    else:
        port =int(oargs.proxy_ctl)
        controller = oGetStemController(port=port)

    vwait_for_controller(controller, oargs.wait_boot)

    if oargs.good_contacts:
        good_contacts_tmp = oargs.good_contacts + '.tmp'

    elt = controller.get_conf('UseMicrodescriptors')
    if elt != '0':
        LOG.error('"UseMicrodescriptors 0" is required in your /etc/tor/torrc. Exiting.')
        controller.set_conf('UseMicrodescriptors', 0)
        # does it work dynamically?
        return 2

    elt = controller.get_conf(sEXCLUDE_EXIT_KEY)
    if elt and elt != '{??}':
        LOG.warn(f"{sEXCLUDE_EXIT_KEY} is in use already")

    twhitelist_set.update(set(lYamlGoodNodes(oargs.good_nodes)))
    LOG.info(f"lYamlGoodNodes {len(twhitelist_set)} GuardNodes from {oargs.good_nodes}")

    global oGOOD_NODES
    t = set()
    if 'IntroductionPoints' in oGOOD_NODES[oGOOD_ROOT]['Relays'].keys():
        t = set(oGOOD_NODES[oGOOD_ROOT]['Relays']['IntroductionPoints'])
    w = set()
    if 'Services' in oGOOD_NODES[oGOOD_ROOT].keys():
        # 'Onions' can I use the IntroductionPoints for Services too?
        # w = set(oGOOD_NODES[oGOOD_ROOT]['Services'])
        pass
    if 'Onions' in oGOOD_NODES[oGOOD_ROOT].keys():
        # Provides the descriptor for a hidden service. The **address** is the
        # '.onion' address of the hidden service
        w = set(oGOOD_NODES[oGOOD_ROOT]['Onions'])
    if oargs.white_onions:
        w.update(oargs.white_onions.split(','))
    if oargs.points_timeout > 0:
        LOG.info(f"{len(w)} services will be checked from IntroductionPoints")
        t.update(lIntroductionPoints(controller, w, itimeout=oargs.points_timeout))
    if len(t) > 0:
        LOG.info(f"IntroductionPoints {len(t)} relays from {len(w)} services")
        twhitelist_set.update(t)

    texclude_set = set()
    if oargs.bad_nodes and os.path.exists(oargs.bad_nodes):
        if False and oargs.bad_sections:
            # BROKEN
            sections = oargs.bad_sections.split(',')
            texclude_set = set(lYamlBadNodes(oargs.bad_nodes,
                                             lWanted=sections,
                                             section=sEXCLUDE_EXIT_KEY))
            LOG.info(f"Preloaded {len(texclude_set)} bad fps")

    ttrust_db_index = aTRUST_DB_INDEX.keys()
    tdns_urls = set()
    iFakeContact = 0
    iTotalContacts = 0
    aBadContacts = {}

    lConds = oargs.contact.split(',')
    iR = 0

    relays = controller.get_server_descriptors()
    for relay in relays:
        iR += 1
        if not is_valid_fingerprint(relay.fingerprint):
            LOG.warn('Invalid Fingerprint: %s' % relay.fingerprint)
            continue
        relay.fingerprint = relay.fingerprint.upper()

        sofar = f"G:{len(aTRUST_DB.keys())} U:{len(tdns_urls)} F:{iFakeContact} BF:{len(texclude_set)} GF:{len(ttrust_db_index)} TC:{iTotalContacts} #{iR}"
        if not relay.exit_policy.is_exiting_allowed():
            if sEXCLUDE_EXIT_KEY == 'ExcludeNodes':
                pass # LOG.debug(f"{relay.fingerprint} not an exit {sofar}")
            else:
                pass # LOG.warn(f"{relay.fingerprint} not an exit {sofar}")
            # continue

        # great contact had good fps and we are in them
        if relay.fingerprint in aTRUST_DB_INDEX.keys():
            # a cached entry
            continue

        if type(relay.contact) == bytes:
            # dunno
            relay.contact = str(relay.contact, 'UTF-8')

        if ('Empty' in lConds and not relay.contact) or \
           ('NoEmail' in lConds and relay.contact and 'email:' not in relay.contact):
            texclude_set.add(relay.fingerprint)
            continue

        if not relay.contact or 'ciissversion:' not in relay.contact:
            # should be unreached 'Empty' should always be in lConds
            continue
        iTotalContacts += 1

        fp = relay.fingerprint
        if relay.contact and 'url:' not in relay.contact:
            LOG.info(f"{fp} skipping bad contact - no url: {sofar}")
            LOG.debug(f"{fp} {relay.contact} {sofar}")
            texclude_set.add(fp)
            continue

        c = relay.contact.lower()
        # first rough cut
        i = c.find('url:')
        if i >=0:
            c = c[i + 4:]
        i = c.find(' ')
        if i >=0: c = c[:i]
        c = c.lstrip('https://').lstrip('http://').strip('/')
        i = c.find('/')
        if i >=0: c = c[:i]
        domain = c
        if domain and bdomain_is_bad(domain, fp):
            LOG.info(f"{fp} skipping bad {domain} {sofar}")
            LOG.debug(f"{fp} {relay.contact} {sofar}")
            texclude_set.add(fp)
            continue

        if domain:
            ip = zResolveDomain(domain)
            if not ip:
                LOG.warn(f"{fp} {domain} did not resolve {sofar}")
                texclude_set.add(fp)
                lKNOWN_NODNS.append(domain)
                iFakeContact += 1
                continue

        if 'dns-rsa' in relay.contact.lower():
            target = f"{relay.fingerprint}.{domain}"
            LOG.info(f"skipping 'dns-rsa' {target} {sofar}")
            tdns_urls.add(target)

        elif 'proof:uri-rsa' in relay.contact.lower():
            a = aParseContact(relay.contact, relay.fingerprint)
            if not a:
                LOG.warn(f"{relay.fingerprint} did not parse {sofar}")
                texclude_set.add(relay.fingerprint)
                continue
            if 'url' in a and a['url']:
                if a['url'] in tBAD_URLS:
                    # The fp is using a contact with a URL we know is bad
                    LOG.info(f"{relay.fingerprint} skipping in tBAD_URLS {a['url']}  {sofar}")
                    LOG.debug(f"{relay.fingerprint} {a} {sofar}")
                    iFakeContact += 1
                    texclude_set.add(relay.fingerprint)
                    continue
                domain = a['url'].replace('https://', '').replace('http://', '')
                if domain in lKNOWN_NODNS:
                    # The fp is using a contact with a URL we know is bogus
                    LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {a['url']} {sofar}")
                    LOG.debug(f"{relay.fingerprint} {a} {sofar}")
                    iFakeContact += 1
                    texclude_set.add(relay.fingerprint)
                    continue

            b = aVerifyContact(list(a.values())[0],
                               relay.fingerprint,
                               oargs.https_cafile,
                               timeout=oargs.timeout,
                               host=oargs.proxy_host,
                               port=oargs.proxy_port)
            # need to skip urllib3.exceptions.MaxRetryError
            if not b or 'fps' not in b or not b['fps'] or not b['url']:
                LOG.warn(f"{relay.fingerprint} did NOT VERIFY {sofar}")
                LOG.debug(f"{relay.fingerprint} {b} {sofar}")
                # If it's giving contact info that doesnt check out
                # it could be a bad exit with fake contact info
                texclude_set.add(relay.fingerprint)
                aBadContacts[relay.fingerprint] = b
                continue

            if relay.fingerprint not in b['fps']:
                LOG.warn(f"{relay.fingerprint} the FP IS NOT in the list of fps {sofar}")
                # assume a fp is using a bogus contact
                texclude_set.add(relay.fingerprint)
                iFakeContact += 1
                aBadContacts[relay.fingerprint] = b
                continue

            LOG.info(f"{relay.fingerprint} verified {b['url']} {sofar}")
            # add our contact info to the trustdb
            aTRUST_DB[relay.fingerprint] = b
            for elt in b['fps']:
                aTRUST_DB_INDEX[elt] = b
            if oargs.good_contacts and oargs.log_level <= 20:
                # as we go along then clobber
                with open(good_contacts_tmp, 'wt') as oFYaml:
                    yaml.dump(aTRUST_DB, oFYaml)
                    oFYaml.close()

    LOG.info(f"Filtered {len(twhitelist_set)} whitelisted relays")
    texclude_set = texclude_set.difference(twhitelist_set)
    # accept the dns-rsa urls for now until we test them
    texclude_set = texclude_set.difference(tdns_urls)
    LOG.info(f"{len(list(aTRUST_DB.keys()))} good contacts out of {iTotalContacts}")

    if oargs.torrc_output and texclude_set:
        with open(oargs.torrc_output, 'wt') as oFTorrc:
            oFTorrc.write(f"{sEXCLUDE_EXIT_KEY} {','.join(texclude_set)}\n")
            oFTorrc.write(f"{sINCLUDE_EXIT_KEY} {','.join(aTRUST_DB_INDEX.keys())}\n")
            oFTorrc.write(f"{sINCLUDE_GUARD_KEY} {','.join(oGOOD_NODES[oGOOD_ROOT]['GuardNodes'])}\n")
            LOG.info(f"Wrote tor configuration to {oargs.torrc_output}")
            oFTorrc.close()

    if oargs.bad_contacts and aBadContacts:
        # for later analysis
        with open(oargs.bad_contacts, 'wt') as oFYaml:
            yaml.dump(aBadContacts, oFYaml)
            oFYaml.close()

    if oargs.good_contacts != '' and aTRUST_DB:
        with open(good_contacts_tmp, 'wt') as oFYaml:
            yaml.dump(aTRUST_DB, oFYaml)
            oFYaml.close()
        if os.path.exists(oargs.good_contacts):
            bak = oargs.good_contacts +'.bak'
            os.rename(oargs.good_contacts, bak)
        os.rename(good_contacts_tmp, oargs.good_contacts)
        LOG.info(f"Wrote {len(list(aTRUST_DB.keys()))} good contact details to {oargs.good_contacts}")

    oBAD_NODES[oBAD_ROOT]['ExcludeNodes']['BadExit'] = list(texclude_set)
    oBAD_NODES[oBAD_ROOT]['ExcludeDomains'] = lKNOWN_NODNS
    vwrite_badnodes(oargs, oBAD_NODES, str(len(texclude_set)))

    oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = list(aTRUST_DB_INDEX.keys())
    # GuardNodes are readonl
    vwrite_goodnodes(oargs, oGOOD_NODES, len(aTRUST_DB_INDEX.keys()))

    retval = 0
    try:
        logging.getLogger('stem').setLevel(30)
        try:
            if texclude_set:
                LOG.info(f"{sEXCLUDE_EXIT_KEY} {len(texclude_set)} net bad exit relays")
                controller.set_conf(sEXCLUDE_EXIT_KEY, texclude_set)

        except stem.SocketClosed as e: # noqa
            LOG.error(f"Failed setting {sEXCLUDE_EXIT_KEY} bad exit relays in Tor")
            retval += 1

        try:
            if aTRUST_DB_INDEX.keys():
                LOG.info(f"{sINCLUDE_EXIT_KEY} {len(aTRUST_DB_INDEX.keys())} good relays")
                controller.set_conf(sINCLUDE_EXIT_KEY, aTRUST_DB_INDEX.keys())
        except stem.SocketClosed as e: # noqa
            LOG.error(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor")
            retval += 1

        try:
            if 'GuardNodes' in oGOOD_NODES[oGOOD_ROOT].keys():
                LOG.info(f"{sINCLUDE_GUARD_KEY} {len(oGOOD_NODES[oGOOD_ROOT]['GuardNodes'])} guard nodes")
                # FixMe for now override StrictNodes it may be unusable otherwise
                controller.set_conf(sINCLUDE_GUARD_KEY,
                                    oGOOD_NODES[oGOOD_ROOT]['GuardNodes'])
                cur = controller.get_conf('StrictNodes')
                if oargs.strict_nodes and int(cur) != oargs.strict_nodes:
                    LOG.info(f"OVERRIDING StrictNodes to {oargs.strict_nodes}")
                    controller.set_conf('StrictNodes', oargs.strict_nodes)
                else:
                    LOG.info(f"StrictNodes is set to {cur}")
        except stem.SocketClosed as e: # noqa
            LOG.errro(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor")
            retval += 1

    except InvalidRequest as e:
        # Unacceptable option value: Invalid router list.
        LOG.error(str(e))
        retval = 1
        return retval
    except KeyboardInterrupt:
        return 0
    except Exception as e:
        LOG.exception(str(e))
        retval = 2
        return retval
    finally:
        # wierd we are getting stem errors during the final return
        # with a traceback that doesnt correspond to any real flow
        # File "/usr/lib/python3.9/site-packages/stem/control.py", line 2474, in set_conf
        #   self.set_options({param: value}, False)
        logging.getLogger('stem').setLevel(40)
        try:
            for elt in controller._event_listeners:
                controller.remove_event_listener(elt)
            controller.close()
        except Exception as e:
            LOG.warn(str(e))

    sys.stdout.write("dns-rsa domains:\n" +'\n'.join(tdns_urls) +'\n')
    return retval

if __name__ == '__main__':
    try:
        i = iMain(sys.argv[1:])
    except IncorrectPassword as e:
        LOG.error(e)
        i = 1
    except KeyboardInterrupt:
        i = 0
    except Exception as e:
        LOG.exception(e)
        i = 2
    sys.exit(i)