From d6200d6302f5828c95a2301c3d962dd66dd9565e Mon Sep 17 00:00:00 2001 From: emdee Date: Wed, 16 Nov 2022 22:15:00 +0000 Subject: [PATCH] add exclude_badExits.bash --- exclude_badExits.bash | 41 ++++++++++++++++++++++++++++++++++ exclude_badExits.py | 52 ++++++++++++++++++++++--------------------- 2 files changed, 68 insertions(+), 25 deletions(-) create mode 100644 exclude_badExits.bash diff --git a/exclude_badExits.bash b/exclude_badExits.bash new file mode 100644 index 0000000..d5d46aa --- /dev/null +++ b/exclude_badExits.bash @@ -0,0 +1,41 @@ +#!/bin/bash +# -*- mode: sh; fill-column: 75; tab-width: 8; coding: utf-8-unix -*- + +PROG=exclude_badExits.py +SOCKS_PORT=9050 +CAFILE=/etc/ssl/certs/ca-certificates.crt + +# an example of running exclude_badExits with full debugging +# expected to take an hour or so +declare -a LARGS +LARGS=( + --log_level 10 + ) +# you may have a special python for installed packages +EXE=`which python3.bash` +LARGS+=( + --strict_nodes 0 + --points_timeout 120 + --proxy-host 127.0.0.1 + --proxy-port $SOCKS_PORT + --https_cafile $CAFILE +) + +if [ -f '/run/tor/control' ] ; then + LARGS+=(--proxy-ctl '/run/tor/control' ) +else + LARGS+=(--proxy-ctl 9051 ) +fi + +ddg=duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad +# for example, whitelist the introduction points to DuckDuckGo +LARGS+=( --white_onions $ddg ) + +# you may need to be the tor user to read /run/tor/control +grep -q ^debian-tor /etc/group && TORU=debian-tor || { + grep -q ^tor /etc/group && TORU=tor +} +sudo -u $TORU $EXE exclude_badExits.py "${LARGS[@]}" \ + 2>&1|tee exclude_badExits6.log + +# The DEBUG statements contain the detail of why the relay was considered bad. diff --git a/exclude_badExits.py b/exclude_badExits.py index 50b9ecf..406e65c 100644 --- a/exclude_badExits.py +++ b/exclude_badExits.py @@ -65,7 +65,7 @@ not just exclude Exit. If the Contact info is good we add the list of fingerprints to add to ExitNodes, a whitelist of relays to use as exits. -```--proof_output``` will write the contact info as a ciiss dictionary +```--good_contacts``` will write the contact info as a ciiss dictionary to a YAML file. If the proof is uri-rsa, the well-known file of fingerprints is downloaded and the fingerprints are added on a 'fps' field we create of that fingerprint's entry of the YAML dictionary. This file is read at the @@ -99,7 +99,8 @@ from stem.connection import IncorrectPassword from stem.util.tor_tools import is_valid_fingerprint try: from ruamel.yaml import YAML - yaml = YAML() + yaml = YAML(typ='rt') + yaml.indent(mapping=2, sequence=2) safe_load = yaml.load except: yaml = None @@ -451,7 +452,7 @@ def oMainArgparser(_=None): parser.add_argument('--torrc_output', type=str, default=os.path.join(ETC_DIR, 'torrc.new'), help="Write the torrc configuration to a file") - parser.add_argument('--proof_output', type=str, default=os.path.join(ETC_DIR, 'goodcontacts.yaml'), + parser.add_argument('--good_contacts', type=str, default=os.path.join(ETC_DIR, 'goodcontacts.yaml'), help="Write the proof data of the included nodes to a YAML file") return parser @@ -460,7 +461,7 @@ def vwrite_badnodes(oArgs, oBAD_NODES, slen): tmp = oArgs.bad_nodes +'.tmp' bak = oArgs.bad_nodes +'.bak' with open(tmp, 'wt') as oFYaml: - yaml.dump(oBAD_NODES, indent=2, stream=oFYaml) + yaml.dump(oBAD_NODES, oFYaml) LOG.info(f"Wrote {slen} to {oArgs.bad_nodes}") oFYaml.close() if os.path.exists(oArgs.bad_nodes): @@ -472,7 +473,7 @@ def vwrite_goodnodes(oArgs, oGOOD_NODES, ilen): tmp = oArgs.good_nodes +'.tmp' bak = oArgs.good_nodes +'.bak' with open(tmp, 'wt') as oFYaml: - yaml.dump(oGOOD_NODES, indent=2, stream=oFYaml) + yaml.dump(oGOOD_NODES, oFYaml) LOG.info(f"Wrote {ilen} good relays to {oArgs.good_nodes}") oFYaml.close() if os.path.exists(oArgs.good_nodes): @@ -497,7 +498,7 @@ def iMain(lArgs): icheck_torrc(sFile, oArgs) twhitelist_set = set() - sFile = oArgs.proof_output + sFile = oArgs.good_contacts if sFile and os.path.exists(sFile): try: with open(sFile, 'rt') as oFd: @@ -527,8 +528,8 @@ def iMain(lArgs): vwait_for_controller(controller, oArgs.wait_boot) - if oArgs.proof_output: - proof_output_tmp = oArgs.proof_output + '.tmp' + if oArgs.good_contacts: + good_contacts_tmp = oArgs.good_contacts + '.tmp' elt = controller.get_conf('UseMicrodescriptors') if elt != '0' : @@ -711,10 +712,10 @@ def iMain(lArgs): aTRUST_DB[relay.fingerprint] = b for elt in b['fps']: aTRUST_DB_INDEX[elt] = b - if oArgs.proof_output and oArgs.log_level <= 20: + if oArgs.good_contacts and oArgs.log_level <= 20: # as we go along then clobber - with open(proof_output_tmp, 'wt') as oFYaml: - yaml.dump(aTRUST_DB, indent=2, stream=oFYaml) + with open(good_contacts_tmp, 'wt') as oFYaml: + yaml.dump(aTRUST_DB, oFYaml) oFYaml.close() LOG.info(f"Filtered {len(twhitelist_set)} whitelisted relays") @@ -723,16 +724,6 @@ def iMain(lArgs): texclude_set = texclude_set.difference(tdns_urls) LOG.info(f"{len(list(aTRUST_DB.keys()))} good contacts out of {iTotalContacts}") - if oArgs.proof_output and aTRUST_DB: - with open(proof_output_tmp, 'wt') as oFYaml: - yaml.dump(aTRUST_DB, indent=2, stream=oFYaml) - oFYaml.close() - if os.path.exists(oArgs.proof_output): - bak = oArgs.proof_output +'.bak' - os.rename(oArgs.proof_output, bak) - os.rename(proof_output_tmp, oArgs.proof_output) - LOG.info(f"Wrote {len(list(aTRUST_DB.keys()))} good contact details to {oArgs.proof_output}") - if oArgs.torrc_output and texclude_set: with open(oArgs.torrc_output, 'wt') as oFTorrc: oFTorrc.write(f"{sEXCLUDE_EXIT_KEY} {','.join(texclude_set)}\n") @@ -744,9 +735,19 @@ def iMain(lArgs): if oArgs.bad_contacts and aBadContacts: # for later analysis with open(oArgs.bad_contacts, 'wt') as oFYaml: - yaml.dump(aBadContacts, indent=2, stream=oFYaml) + yaml.dump(aBadContacts, oFYaml) oFYaml.close() + if oArgs.good_contacts != '' and aTRUST_DB: + with open(good_contacts_tmp, 'wt') as oFYaml: + yaml.dump(aTRUST_DB, oFYaml) + oFYaml.close() + if os.path.exists(oArgs.good_contacts): + bak = oArgs.good_contacts +'.bak' + os.rename(oArgs.good_contacts, bak) + os.rename(good_contacts_tmp, oArgs.good_contacts) + LOG.info(f"Wrote {len(list(aTRUST_DB.keys()))} good contact details to {oArgs.good_contacts}") + oBAD_NODES[oBAD_ROOT]['ExcludeNodes']['BadExit'] = list(texclude_set) oBAD_NODES[oBAD_ROOT]['ExcludeDomains'] = lKNOWN_NODNS vwrite_badnodes(oArgs, oBAD_NODES, str(len(texclude_set))) @@ -754,6 +755,7 @@ def iMain(lArgs): oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = list(aTRUST_DB_INDEX.keys()) # GuardNodes are readonl vwrite_goodnodes(oArgs, oGOOD_NODES, len(aTRUST_DB_INDEX.keys())) + retval = 0 try: logging.getLogger('stem').setLevel(30) @@ -790,9 +792,6 @@ def iMain(lArgs): LOG.errro(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor") retval += 1 - sys.stdout.write("dns-rsa domains:\n" +'\n'.join(tdns_urls) +'\n') - return retval - except InvalidRequest as e: # Unacceptable option value: Invalid router list. LOG.error(str(e)) @@ -816,6 +815,9 @@ def iMain(lArgs): controller.close() except Exception as e: LOG.warn(str(e)) + + sys.stdout.write("dns-rsa domains:\n" +'\n'.join(tdns_urls) +'\n') + return retval if __name__ == '__main__': try: