885 lines
33 KiB
Python
885 lines
33 KiB
Python
# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*-
|
|
|
|
# https://github.com/nusenu/noContactInfo_Exit_Excluder
|
|
# https://github.com/TheSmashy/TorExitRelayExclude
|
|
"""
|
|
This extends nusenu's basic idea of using the stem library to
|
|
dynamically exclude nodes that are likely to be bad by putting them
|
|
on the ExcludeNodes or ExcludeExitNodes setting of a running Tor.
|
|
* https://github.com/nusenu/noContactInfo_Exit_Excluder
|
|
* https://github.com/TheSmashy/TorExitRelayExclude
|
|
|
|
The basic cut is to exclude Exit nodes that do not have a contact.
|
|
That can be extended to nodes that do not have an email in the contact etc.
|
|
"""
|
|
"""But there's a problem, and your Tor notice.log will tell you about it:
|
|
you could exclude the nodes needed to access hidden services or
|
|
directorues. So we need to add to the process the concept of a whitelist.
|
|
In addition, we may have our own blacklist of nodes we want to exclude,
|
|
or use these lists for other applications like selektor.
|
|
|
|
So we make two files that are structured in YAML:
|
|
```
|
|
/etc/tor/yaml/torrc-goodnodes.yaml
|
|
GoodNodes:
|
|
Relays:
|
|
IntroductionPoints:
|
|
- NODEFINGERPRINT
|
|
...
|
|
By default all sections of the goodnodes.yaml are used as a whitelist.
|
|
|
|
/etc/tor/yaml/torrc-badnodes.yaml
|
|
BadNodes:
|
|
ExcludeExitNodes:
|
|
BadExit:
|
|
# $0000000000000000000000000000000000000007
|
|
```
|
|
That part requires [PyYAML](https://pyyaml.org/wiki/PyYAML)
|
|
https://github.com/yaml/pyyaml/
|
|
|
|
Right now only the ExcludeExitNodes section is used by we may add ExcludeNodes
|
|
later, and by default all sub-sections of the badnodes.yaml are used as a
|
|
ExcludeExitNodes but it can be customized with the lWanted commandline arg.
|
|
|
|
The original idea has also been extended to add different conditions for
|
|
exclusion: the ```--contact``` commandline arg is a comma sep list of conditions:
|
|
* Empty - no contact info
|
|
* NoEmail - no @ sign in the contact',
|
|
More may be added later.
|
|
|
|
Because you don't want to exclude the introduction points to any onion
|
|
you want to connect to, ```--white_services``` should whitelist the
|
|
introduction points to a comma sep list of onions, but is
|
|
currently broken in stem 1.8.0: see:
|
|
* https://github.com/torproject/stem/issues/96
|
|
* https://gitlab.torproject.org/legacy/trac/-/issues/25417
|
|
|
|
```--torrc_output``` will write the torrc ExcludeNodes configuration to a file.
|
|
|
|
Now for the final part: we lookup the Contact info of every server
|
|
that is currently in our Tor, and check it for its existence.
|
|
If it fails to provide the well-know url, we assume its a bogus
|
|
relay and add it to a list of nodes that goes on ExcludeNodes -
|
|
not just exclude Exit.
|
|
|
|
If the Contact info is good we add the list of fingerprints to add
|
|
to ExitNodes, a whitelist of relays to use as exits.
|
|
|
|
```--proof_output``` will write the contact info as a ciiss dictionary
|
|
to a YAML file. If the proof is uri-rsa, the well-known file of fingerprints
|
|
is downloaded and the fingerprints are added on a 'fps' field we create
|
|
of that fingerprint's entry of the YAML dictionary. This file is read at the
|
|
beginning of the program to start with a trust database, and only new
|
|
contact info from new relays are added to the dictionary.
|
|
|
|
You can expect it to take an hour or two the first time this is run:
|
|
>700 domains.
|
|
|
|
For usage, do ```python3 exclude_badExits.py --help`
|
|
|
|
"""
|
|
|
|
import sys
|
|
|
|
import os
|
|
import re
|
|
import socket
|
|
import time
|
|
import argparse
|
|
import string
|
|
from io import StringIO
|
|
import ipaddr
|
|
# list(ipaddress._find_address_range(ipaddress.IPv4Network('172.16.0.0/12'))
|
|
|
|
from urllib3.util.ssl_match_hostname import CertificateError
|
|
import stem
|
|
from stem import InvalidRequest
|
|
from stem.control import Controller
|
|
from stem.connection import IncorrectPassword
|
|
from stem.util.tor_tools import is_valid_fingerprint
|
|
try:
|
|
import yaml
|
|
except:
|
|
yaml = None
|
|
try:
|
|
from unbound import ub_ctx,RR_TYPE_TXT,RR_CLASS_IN
|
|
except:
|
|
ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None
|
|
|
|
try:
|
|
if 'COLOREDLOGS_LEVEL_STYLES' not in os.environ:
|
|
os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red'
|
|
# https://pypi.org/project/coloredlogs/
|
|
import coloredlogs
|
|
except ImportError as e:
|
|
coloredlogs = False
|
|
|
|
global LOG
|
|
import logging
|
|
import warnings
|
|
warnings.filterwarnings('ignore')
|
|
LOG = logging.getLogger()
|
|
|
|
import requests
|
|
from trustor_poc import oDownloadUrlUrllib3 as oDownloadUrl
|
|
from trustor_poc import idns_validate, TrustorError
|
|
from support_onions import icheck_torrc, bAreWeConnected, lIntroductionPoints, zResolveDomain, vwait_for_controller, yKNOWN_NODNS
|
|
LOG.info("imported HTTPSAdapter")
|
|
|
|
ETC_DIR = '/etc/tor/yaml'
|
|
aTRUST_DB = {}
|
|
aTRUST_DB_INDEX = {}
|
|
aFP_EMAIL = {}
|
|
sDETAILS_URL = "https://metrics.torproject.org/rs.html#details/"
|
|
# You can call this while bootstrapping
|
|
sEXCLUDE_EXIT_KEY = 'ExcludeNodes'
|
|
sINCLUDE_EXIT_KEY = 'ExitNodes'
|
|
sINCLUDE_GUARD_KEY = 'EntryNodes'
|
|
|
|
def oMakeController(sSock='', port=9051):
|
|
import getpass
|
|
if sSock and os.path.exists(sSock):
|
|
controller = Controller.from_socket_file(path=sSock)
|
|
else:
|
|
controller = Controller.from_port(port=port)
|
|
sys.stdout.flush()
|
|
p = getpass.unix_getpass(prompt='Controller Password: ', stream=sys.stderr)
|
|
controller.authenticate(p)
|
|
return controller
|
|
|
|
oBAD_NODES = {}
|
|
oBAD_ROOT = 'BadNodes'
|
|
oBAD_NODES[oBAD_ROOT] = {}
|
|
oBAD_NODES[oBAD_ROOT]['ExcludeNodes'] = {}
|
|
|
|
lKNOWN_NODNS = []
|
|
lMAYBE_NODNS = []
|
|
def lYamlBadNodes(sFile,
|
|
section=sEXCLUDE_EXIT_KEY,
|
|
lWanted=['BadExit']):
|
|
global oBAD_NODES
|
|
global lKNOWN_NODNS
|
|
global lMAYBE_NODNS
|
|
|
|
l = []
|
|
if not yaml: return l
|
|
if os.path.exists(sFile):
|
|
with open(sFile, 'rt') as oFd:
|
|
oBAD_NODES = yaml.safe_load(oFd)
|
|
|
|
# BROKEN
|
|
# root = 'ExcludeNodes'
|
|
# for elt in o[oBAD_ROOT][root][section].keys():
|
|
# if lWanted and elt not in lWanted: continue
|
|
# # l += o[oBAD_ROOT][root][section][elt]
|
|
|
|
l = oBAD_NODES[oBAD_ROOT]['ExcludeNodes']['BadExit']
|
|
|
|
root = 'ExcludeDomains'
|
|
if root not in oBAD_NODES[oBAD_ROOT] or not oBAD_NODES[oBAD_ROOT][root]:
|
|
lMAYBE_NODNS = yaml.safe_load(StringIO(yKNOWN_NODNS))
|
|
else:
|
|
lMAYBE_NODNS = oBAD_NODES[oBAD_ROOT][root]
|
|
return l
|
|
|
|
oGOOD_NODES = {}
|
|
oGOOD_ROOT = 'GoodNodes'
|
|
def lYamlGoodNodes(sFile='/etc/tor/torrc-goodnodes.yaml'):
|
|
global oGOOD_NODES
|
|
root = oGOOD_ROOT
|
|
l = []
|
|
if not yaml: return l
|
|
if os.path.exists(sFile):
|
|
with open(sFile, 'rt') as oFd:
|
|
o = yaml.safe_load(oFd)
|
|
oGOOD_NODES = o
|
|
if 'GuardNodes' in o[oGOOD_ROOT].keys():
|
|
l = o[oGOOD_ROOT]['GuardNodes']
|
|
# yq '.Nodes.IntroductionPoints|.[]' < /etc/tor/torrc-goodnodes.yaml
|
|
return l
|
|
|
|
def bdomain_is_bad(domain, fp):
|
|
global lKNOWN_NODNS
|
|
if domain in lKNOWN_NODNS: return True
|
|
if domain in lMAYBE_NODNS:
|
|
ip = zResolveDomain(domain)
|
|
if ip == '':
|
|
LOG.debug(f"{fp} {domain} does not resolve")
|
|
lKNOWN_NODNS.append(domain)
|
|
lMAYBE_NODNS.remove(domain)
|
|
return True
|
|
|
|
for elt in '@(){}$!':
|
|
if elt in domain:
|
|
LOG.warn(f"{elt} in domain {domain}")
|
|
return True
|
|
return False
|
|
|
|
tBAD_URLS = set()
|
|
lATS = ['abuse', 'email']
|
|
lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime', 'memory']
|
|
lBOOLS = ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone',
|
|
'sandbox', 'offlinemasterkey']
|
|
def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050):
|
|
global tBAD_URLS
|
|
global lKNOWN_NODNS
|
|
# cleanups for yaml
|
|
for elt in lINTS:
|
|
if elt in a:
|
|
a[elt] = int(a[elt])
|
|
for elt in lBOOLS:
|
|
if elt in a:
|
|
if a[elt] in ['y','yes', 'true', 'True']:
|
|
a[elt] = True
|
|
else:
|
|
a[elt] = False
|
|
for elt in lATS:
|
|
if elt in a:
|
|
a[elt] = a[elt].replace('[]', '@')
|
|
|
|
a.update({'fps': []})
|
|
keys = list(a.keys())
|
|
if 'email' not in keys:
|
|
LOG.warn(f"{fp} 'email' not in {keys}")
|
|
a['email'] = ''
|
|
if 'ciissversion' not in keys:
|
|
aFP_EMAIL[fp] = a['email']
|
|
LOG.warn(f"{fp} 'ciissversion' not in {keys}")
|
|
a['ciissversion'] = 2
|
|
# test the url for fps and add it to the array
|
|
if 'proof' not in keys:
|
|
aFP_EMAIL[fp] = a['email']
|
|
LOG.warn(f"{fp} 'proof' not in {keys}")
|
|
return a
|
|
|
|
if aTRUST_DB_INDEX and fp in aTRUST_DB_INDEX.keys():
|
|
aCachedContact = aTRUST_DB_INDEX[fp]
|
|
if aCachedContact['email'] == a['email']:
|
|
LOG.info(f"{fp} in aTRUST_DB_INDEX")
|
|
return aCachedContact
|
|
|
|
if 'url' not in keys:
|
|
if 'uri' not in keys:
|
|
a['url'] = ''
|
|
aFP_EMAIL[fp] = a['email']
|
|
LOG.warn(f"{fp} url and uri not in {keys}")
|
|
return a
|
|
a['url'] = a['uri']
|
|
aFP_EMAIL[fp] = a['email']
|
|
LOG.debug(f"{fp} 'uri' but not 'url' in {keys}")
|
|
# drop through
|
|
|
|
c = a['url'].lstrip('https://').lstrip('http://').strip('/')
|
|
a['url'] = 'https://' +c
|
|
|
|
# domain should be a unique key for contacts
|
|
domain = a['url'][8:]
|
|
if bdomain_is_bad(domain, fp):
|
|
LOG.warn(f"{domain} is bad from {a['url']}")
|
|
LOG.debug(f"{fp} is bad from {a}")
|
|
return a
|
|
|
|
ip = zResolveDomain(domain)
|
|
if ip == '':
|
|
aFP_EMAIL[fp] = a['email']
|
|
LOG.debug(f"{fp} {domain} does not resolve")
|
|
lKNOWN_NODNS.append(domain)
|
|
return {}
|
|
|
|
if a['proof'] not in ['uri-rsa']:
|
|
# only support uri for now
|
|
if False and ub_ctx:
|
|
fp_domain = fp +'.'+domain
|
|
if idns_validate(fp_domain,
|
|
libunbound_resolv_file='resolv.conf',
|
|
dnssec_DS_file='dnssec-root-trust',
|
|
) == 0:
|
|
pass
|
|
LOG.warn(f"{fp} proof={a['proof']} not supported yet")
|
|
return a
|
|
|
|
LOG.debug(f"{len(keys)} contact fields for {fp}")
|
|
url="https://"+domain+"/.well-known/tor-relay/rsa-fingerprint.txt"
|
|
try:
|
|
LOG.debug(f"Downloading from {domain} for {fp}")
|
|
o = oDownloadUrl(url, https_cafile,
|
|
timeout=timeout, host=host, port=port)
|
|
# requests response: text "reason", "status_code"
|
|
except AttributeError as e:
|
|
LOG.exception(f"AttributeError downloading from {domain} {e}")
|
|
except CertificateError as e:
|
|
LOG.warn(f"CertificateError downloading from {domain} {e}")
|
|
tBAD_URLS.add(a['url'])
|
|
except TrustorError as e:
|
|
if e.args == "HTTP Errorcode 404":
|
|
aFP_EMAIL[fp] = a['email']
|
|
LOG.warn(f"TrustorError 404 from {domain} {e.args}")
|
|
else:
|
|
LOG.warn(f"TrustorError downloading from {domain} {e.args}")
|
|
tBAD_URLS.add(a['url'])
|
|
except (BaseException ) as e:
|
|
LOG.error(f"Exception {type(e)} downloading from {domain} {e}")
|
|
else:
|
|
if hasattr(o, 'status'):
|
|
status_code = o.status
|
|
else:
|
|
status_code = o.status_code
|
|
if status_code >= 300:
|
|
aFP_EMAIL[fp] = a['email']
|
|
LOG.warn(f"Error from {domain} {status_code} {o.reason}")
|
|
# any reason retry?
|
|
tBAD_URLS.add(a['url'])
|
|
return a
|
|
|
|
if hasattr(o, 'text'):
|
|
data = o.text
|
|
else:
|
|
data = str(o.data, 'UTF-8')
|
|
l = data.upper().strip().split('\n')
|
|
LOG.debug(f"Downloaded from {domain} {len(l)} lines {len(data)} bytes")
|
|
|
|
a['modified'] = int(time.time())
|
|
if not l:
|
|
LOG.warn(f"Downloading from {domain} empty for {fp}")
|
|
else:
|
|
a['fps'] = [elt for elt in l if elt and len(elt) == 40
|
|
and not elt.startswith('#')]
|
|
LOG.info(f"Downloaded from {domain} {len(a['fps'])} FPs")
|
|
return a
|
|
|
|
def aParseContactYaml(contact, fp):
|
|
"""
|
|
See the Tor ContactInfo Information Sharing Specification v2
|
|
https://nusenu.github.io/ContactInfo-Information-Sharing-Specification/
|
|
"""
|
|
lelts = contact.split()
|
|
a = {}
|
|
if len(lelts) % 1 != 0:
|
|
LOG.warn(f"bad contact for {fp} odd number of components")
|
|
LOG.debug(f"{fp} {a}")
|
|
return a
|
|
key = ''
|
|
for elt in lelts:
|
|
if key == '':
|
|
key = elt
|
|
continue
|
|
a[key] = elt
|
|
key = ''
|
|
LOG.debug(f"{fp} {len(a.keys())} fields")
|
|
return a
|
|
|
|
def aParseContact(contact, fp):
|
|
"""
|
|
See the Tor ContactInfo Information Sharing Specification v2
|
|
https://nusenu.github.io/ContactInfo-Information-Sharing-Specification/
|
|
"""
|
|
l = [line for line in contact.strip().replace('"', '').split(' ')
|
|
if ':' in line]
|
|
LOG.debug(f"{fp} {len(l)} fields")
|
|
s = f'"{fp}":\n'
|
|
s += '\n'.join([f" {line}\"".replace(':',': \"', 1)
|
|
for line in l])
|
|
oFd = StringIO(s)
|
|
a = yaml.safe_load(oFd)
|
|
return a
|
|
|
|
def vsetup_logging(log_level, logfile=''):
|
|
global LOG
|
|
add = True
|
|
|
|
# stem fucks up logging
|
|
from stem.util import log
|
|
logging.getLogger('stem').setLevel(30)
|
|
|
|
logging._defaultFormatter = logging.Formatter(datefmt='%m-%d %H:%M:%S')
|
|
logging._defaultFormatter.default_time_format = '%m-%d %H:%M:%S'
|
|
logging._defaultFormatter.default_msec_format = ''
|
|
|
|
kwargs = dict(level=log_level,
|
|
force=True,
|
|
format='%(levelname)s %(message)s')
|
|
|
|
if logfile:
|
|
add = logfile.startswith('+')
|
|
sub = logfile.startswith('-')
|
|
if add or sub:
|
|
logfile = logfile[1:]
|
|
kwargs['filename'] = logfile
|
|
|
|
if coloredlogs:
|
|
# https://pypi.org/project/coloredlogs/
|
|
aKw = dict(level=log_level,
|
|
logger=LOG,
|
|
stream=sys.stdout if add else None,
|
|
fmt='%(levelname)s %(message)s'
|
|
)
|
|
coloredlogs.install(**aKw)
|
|
if logfile:
|
|
oHandler = logging.FileHandler(logfile)
|
|
LOG.addHandler(oHandler)
|
|
LOG.info(f"CSetting log_level to {log_level!s}")
|
|
else:
|
|
logging.basicConfig(**kwargs)
|
|
if add and logfile:
|
|
oHandler = logging.StreamHandler(sys.stdout)
|
|
LOG.addHandler(oHandler)
|
|
LOG.info(f"SSetting log_level to {log_level!s}")
|
|
|
|
logging._levelToName = {
|
|
logging.CRITICAL: 'CRITICAL',
|
|
logging.ERROR: 'ERROR',
|
|
logging.WARNING: 'WARN',
|
|
logging.INFO: 'INFO',
|
|
logging.DEBUG: 'DEBUG',
|
|
logging.NOTSET: 'NOTSET',
|
|
}
|
|
logging._nameToLevel = {
|
|
'CRITICAL': logging.CRITICAL,
|
|
'FATAL': logging.FATAL,
|
|
'ERROR': logging.ERROR,
|
|
'WARN': logging.WARNING,
|
|
'WARNING': logging.WARNING,
|
|
'INFO': logging.INFO,
|
|
'DEBUG': logging.DEBUG,
|
|
'NOTSET': logging.NOTSET,
|
|
}
|
|
|
|
def oMainArgparser(_=None):
|
|
|
|
try:
|
|
from OpenSSL import SSL
|
|
lCAfs = SSL._CERTIFICATE_FILE_LOCATIONS
|
|
except:
|
|
lCAfs = []
|
|
|
|
CAfs = []
|
|
for elt in lCAfs:
|
|
if os.path.exists(elt):
|
|
CAfs.append(elt)
|
|
if not CAfs:
|
|
CAfs = ['']
|
|
|
|
parser = argparse.ArgumentParser(add_help=True,
|
|
epilog=__doc__)
|
|
parser.add_argument('--https_cafile', type=str,
|
|
help="Certificate Authority file (in PEM)",
|
|
default=CAfs[0])
|
|
parser.add_argument('--proxy_host', '--proxy-host', type=str,
|
|
default='127.0.0.1',
|
|
help='proxy host')
|
|
parser.add_argument('--proxy_port', '--proxy-port', default=9050, type=int,
|
|
help='proxy control port')
|
|
parser.add_argument('--proxy_ctl', '--proxy-ctl',
|
|
default='/run/tor/control' if os.path.exists('/run/tor/control') else 9051,
|
|
type=str,
|
|
help='control socket - or port')
|
|
|
|
parser.add_argument('--torrc',
|
|
default='/etc/tor/torrc-defaults',
|
|
type=str,
|
|
help='torrc to check for suggestions')
|
|
parser.add_argument('--timeout', default=60, type=int,
|
|
help='proxy download connect timeout')
|
|
|
|
parser.add_argument('--good_nodes', type=str,
|
|
default=os.path.join(ETC_DIR, 'torrc-goodnodes.yaml'),
|
|
help="Yaml file of good info that should not be excluded")
|
|
parser.add_argument('--bad_nodes', type=str,
|
|
default=os.path.join(ETC_DIR, 'torrc-badnodes.yaml'),
|
|
help="Yaml file of bad nodes that should also be excluded")
|
|
parser.add_argument('--contact', type=str, default='Empty,NoEmail',
|
|
help="comma sep list of conditions - Empty,NoEmail")
|
|
parser.add_argument('--bad_contacts', type=str,
|
|
default=os.path.join(ETC_DIR, 'badcontacts.yaml'),
|
|
help="Yaml file of bad contacts that bad FPs are using")
|
|
|
|
parser.add_argument('--strict_nodes', type=int, default=0,
|
|
choices=[0,1],
|
|
help="Set StrictNodes: 1 is less anonymous but more secure, although some sites may be unreachable")
|
|
parser.add_argument('--wait_boot', type=int, default=120,
|
|
help="Seconds to wait for Tor to booststrap")
|
|
parser.add_argument('--points_timeout', type=int, default=0,
|
|
help="Timeout for getting introduction points - must be long >120sec. 0 means disabled looking for IPs")
|
|
parser.add_argument('--log_level', type=int, default=20,
|
|
help="10=debug 20=info 30=warn 40=error")
|
|
parser.add_argument('--bad_sections', type=str,
|
|
default='MyBadExit',
|
|
help="sections of the badnodes.yaml to use, comma separated, '' BROKEN")
|
|
parser.add_argument('--white_services', type=str,
|
|
default='',
|
|
help="comma sep. list of onions to whitelist their introduction points - BROKEN")
|
|
parser.add_argument('--torrc_output', type=str, default='',
|
|
help="Write the torrc configuration to a file")
|
|
parser.add_argument('--proof_output', type=str, default=os.path.join(ETC_DIR, 'proof.yaml'),
|
|
help="Write the proof data of the included nodes to a YAML file")
|
|
return parser
|
|
|
|
def vwrite_badnodes(oArgs, oBAD_NODES, slen):
|
|
if oArgs.bad_nodes:
|
|
tmp = oArgs.bad_nodes +'.tmp'
|
|
bak = oArgs.bad_nodes +'.bak'
|
|
with open(tmp, 'wt') as oFYaml:
|
|
yaml.dump(oBAD_NODES, indent=2, stream=oFYaml)
|
|
LOG.info(f"Wrote {slen} to {oArgs.bad_nodes}")
|
|
oFYaml.close()
|
|
if os.path.exists(oArgs.bad_nodes):
|
|
os.rename(oArgs.bad_nodes, bak)
|
|
os.rename(tmp, oArgs.bad_nodes)
|
|
|
|
def vwrite_goodnodes(oArgs, oGOOD_NODES, ilen):
|
|
if oArgs.good_nodes:
|
|
tmp = oArgs.good_nodes +'.tmp'
|
|
bak = oArgs.good_nodes +'.bak'
|
|
with open(tmp, 'wt') as oFYaml:
|
|
yaml.dump(oGOOD_NODES, indent=2, stream=oFYaml)
|
|
LOG.info(f"Wrote {ilen} good relays to {oArgs.good_nodes}")
|
|
oFYaml.close()
|
|
if os.path.exists(oArgs.good_nodes):
|
|
os.rename(oArgs.good_nodes, bak)
|
|
os.rename(tmp, oArgs.good_nodes)
|
|
|
|
def iMain(lArgs):
|
|
global aTRUST_DB
|
|
global aTRUST_DB_INDEX
|
|
global oBAD_NODES
|
|
global oGOOD_NODES
|
|
global lKNOWN_NODNS
|
|
parser = oMainArgparser()
|
|
oArgs = parser.parse_args(lArgs)
|
|
|
|
vsetup_logging(oArgs.log_level)
|
|
if bAreWeConnected() is False:
|
|
raise SystemExit("we are not connected")
|
|
|
|
sFile = oArgs.torrc
|
|
if sFile and os.path.exists(sFile):
|
|
icheck_torrc(sFile, oArgs)
|
|
|
|
twhitelist_set = set()
|
|
sFile = oArgs.proof_output
|
|
if sFile and os.path.exists(sFile):
|
|
try:
|
|
with open(sFile, 'rt') as oFd:
|
|
aTRUST_DB = yaml.safe_load(oFd)
|
|
assert type(aTRUST_DB) == dict
|
|
LOG.info(f"{len(aTRUST_DB.keys())} trusted contacts from {sFile}")
|
|
# reverse lookup of fps to contacts
|
|
# but...
|
|
for k,v in aTRUST_DB.items():
|
|
if 'modified' not in v.keys():
|
|
v['modified'] = int(time.time())
|
|
aTRUST_DB_INDEX[k] = v
|
|
if 'fps' in aTRUST_DB[k].keys():
|
|
for fp in aTRUST_DB[k]['fps']:
|
|
if fp in aTRUST_DB_INDEX:
|
|
continue
|
|
aTRUST_DB_INDEX[fp] = v
|
|
LOG.info(f"{len(aTRUST_DB_INDEX.keys())} good relays from {sFile}")
|
|
|
|
except Exception as e:
|
|
LOG.exception(f"Error reading YAML TrustDB {sFile} {e}")
|
|
|
|
if os.path.exists(oArgs.proxy_ctl):
|
|
controller = oMakeController(sSock=oArgs.proxy_ctl)
|
|
else:
|
|
port =int(oArgs.proxy_ctl)
|
|
controller = oMakeController(port=port)
|
|
|
|
vwait_for_controller(controller, oArgs.wait_boot)
|
|
|
|
if oArgs.proof_output:
|
|
proof_output_tmp = oArgs.proof_output + '.tmp'
|
|
|
|
elt = controller.get_conf('UseMicrodescriptors')
|
|
if elt != '0' :
|
|
LOG.error('"UseMicrodescriptors 0" is required in your /etc/tor/torrc. Exiting.')
|
|
controller.set_conf('UseMicrodescriptors', 0)
|
|
# does it work dynamically?
|
|
return 2
|
|
|
|
elt = controller.get_conf(sEXCLUDE_EXIT_KEY)
|
|
if elt and elt != '{??}':
|
|
LOG.warn(f"{sEXCLUDE_EXIT_KEY} is in use already")
|
|
|
|
twhitelist_set.update(set(lYamlGoodNodes(oArgs.good_nodes)))
|
|
LOG.info(f"lYamlGoodNodes {len(twhitelist_set)} GuardNodes from {oArgs.good_nodes}")
|
|
|
|
global oGOOD_NODES
|
|
t = set()
|
|
if 'IntroductionPoints' in oGOOD_NODES[oGOOD_ROOT]['Relays'].keys():
|
|
t = set(oGOOD_NODES[oGOOD_ROOT]['Relays']['IntroductionPoints'])
|
|
# not working = maybe when stem is updated
|
|
w = set(oGOOD_NODES[oGOOD_ROOT]['Services'])
|
|
if oArgs.white_services:
|
|
w.update(oArgs.white_services.split(','))
|
|
if oArgs.points_timeout > 0:
|
|
LOG.info(f"{len(w)} services will be checked from IntroductionPoints")
|
|
t.update(lIntroductionPoints(controller, w, itimeout=oArgs.points_timeout))
|
|
if len(t) > 0:
|
|
LOG.info(f"IntroductionPoints {len(t)} relays from {len(w)} services")
|
|
twhitelist_set.update(t)
|
|
|
|
texclude_set = set()
|
|
if oArgs.bad_nodes and os.path.exists(oArgs.bad_nodes):
|
|
if False and oArgs.bad_sections:
|
|
# BROKEN
|
|
sections = oArgs.bad_sections.split(',')
|
|
texclude_set = set(lYamlBadNodes(oArgs.bad_nodes,
|
|
lWanted=sections,
|
|
section=sEXCLUDE_EXIT_KEY))
|
|
LOG.info(f"Preloaded {len(texclude_set)} bad fps")
|
|
|
|
ttrust_db_index = aTRUST_DB_INDEX.keys()
|
|
tdns_urls = set()
|
|
iFakeContact = 0
|
|
iTotalContacts = 0
|
|
aBadContacts = {}
|
|
|
|
lConds = oArgs.contact.split(',')
|
|
iR = 0
|
|
|
|
relays = controller.get_server_descriptors()
|
|
for relay in relays:
|
|
iR += 1
|
|
if not is_valid_fingerprint(relay.fingerprint):
|
|
LOG.warn('Invalid Fingerprint: %s' % relay.fingerprint)
|
|
continue
|
|
relay.fingerprint = relay.fingerprint.upper()
|
|
|
|
sofar = f"G:{len(aTRUST_DB.keys())} U:{len(tdns_urls)} F:{iFakeContact} BF:{len(texclude_set)} GF:{len(ttrust_db_index)} TC:{iTotalContacts} #{iR}"
|
|
if not relay.exit_policy.is_exiting_allowed():
|
|
if sEXCLUDE_EXIT_KEY == 'ExcludeNodes':
|
|
pass # LOG.debug(f"{relay.fingerprint} not an exit {sofar}")
|
|
else:
|
|
pass # LOG.warn(f"{relay.fingerprint} not an exit {sofar}")
|
|
# continue
|
|
|
|
# great contact had good fps and we are in them
|
|
if relay.fingerprint in aTRUST_DB_INDEX.keys():
|
|
# a cached entry
|
|
continue
|
|
|
|
if type(relay.contact) == bytes:
|
|
# dunno
|
|
relay.contact = str(relay.contact, 'UTF-8')
|
|
|
|
if ('Empty' in lConds and not relay.contact) or \
|
|
('NoEmail' in lConds and relay.contact and not 'email:' in relay.contact):
|
|
texclude_set.add(relay.fingerprint)
|
|
continue
|
|
|
|
if not relay.contact or not 'ciissversion:' in relay.contact:
|
|
# should be unreached 'Empty' should always be in lConds
|
|
continue
|
|
iTotalContacts += 1
|
|
|
|
fp = relay.fingerprint
|
|
if relay.contact and not 'url:' in relay.contact:
|
|
LOG.info(f"{fp} skipping bad contact - no url: {sofar}")
|
|
LOG.debug(f"{fp} {relay.contact} {sofar}")
|
|
texclude_set.add(fp)
|
|
continue
|
|
|
|
c = relay.contact.lower()
|
|
# first rough cut
|
|
i = c.find('url:')
|
|
if i >=0:
|
|
c = c[i+4:]
|
|
i = c.find(' ')
|
|
if i >=0: c = c[:i]
|
|
c = c.lstrip('https://').lstrip('http://').strip('/')
|
|
i = c.find('/')
|
|
if i >=0: c = c[:i]
|
|
domain = c
|
|
if domain and bdomain_is_bad(domain, fp):
|
|
LOG.info(f"{fp} skipping bad {domain} {sofar}")
|
|
LOG.debug(f"{fp} {relay.contact} {sofar}")
|
|
texclude_set.add(fp)
|
|
continue
|
|
|
|
if domain:
|
|
ip = zResolveDomain(domain)
|
|
if not ip:
|
|
LOG.warn(f"{fp} {domain} did not resolve {sofar}")
|
|
texclude_set.add(fp)
|
|
lKNOWN_NODNS.append(domain)
|
|
iFakeContact += 1
|
|
continue
|
|
|
|
if 'dns-rsa' in relay.contact.lower():
|
|
target = f"{relay.fingerprint}.{domain}"
|
|
LOG.info(f"skipping 'dns-rsa' {target} {sofar}")
|
|
tdns_urls.add(target)
|
|
|
|
elif 'proof:uri-rsa' in relay.contact.lower():
|
|
a = aParseContact(relay.contact, relay.fingerprint)
|
|
if not a:
|
|
LOG.warn(f"{relay.fingerprint} did not parse {sofar}")
|
|
texclude_set.add(relay.fingerprint)
|
|
continue
|
|
if 'url' in a and a['url']:
|
|
if a['url'] in tBAD_URLS:
|
|
# The fp is using a contact with a URL we know is bad
|
|
LOG.info(f"{relay.fingerprint} skipping in tBAD_URLS {a['url']} {sofar}")
|
|
LOG.debug(f"{relay.fingerprint} {a} {sofar}")
|
|
iFakeContact += 1
|
|
texclude_set.add(relay.fingerprint)
|
|
continue
|
|
domain = a['url'].replace('https://', '').replace('http://', '')
|
|
if domain in lKNOWN_NODNS:
|
|
# The fp is using a contact with a URL we know is bogus
|
|
LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {a['url']} {sofar}")
|
|
LOG.debug(f"{relay.fingerprint} {a} {sofar}")
|
|
iFakeContact += 1
|
|
texclude_set.add(relay.fingerprint)
|
|
continue
|
|
|
|
|
|
b = aVerifyContact(list(a.values())[0],
|
|
relay.fingerprint,
|
|
oArgs.https_cafile,
|
|
timeout=oArgs.timeout,
|
|
host=oArgs.proxy_host,
|
|
port=oArgs.proxy_port)
|
|
|
|
if not b or not 'fps' in b or not b['fps'] or not b['url']:
|
|
LOG.warn(f"{relay.fingerprint} did NOT VERIFY {sofar}")
|
|
LOG.debug(f"{relay.fingerprint} {b} {sofar}")
|
|
# If it's giving contact info that doesnt check out
|
|
# it could be a bad exit with fake contact info
|
|
texclude_set.add(relay.fingerprint)
|
|
aBadContacts[relay.fingerprint] = b
|
|
continue
|
|
|
|
if relay.fingerprint not in b['fps']:
|
|
LOG.warn(f"{relay.fingerprint} the FP IS NOT in the list of fps {sofar}")
|
|
# assume a fp is using a bogus contact
|
|
texclude_set.add(relay.fingerprint)
|
|
iFakeContact += 1
|
|
aBadContacts[relay.fingerprint] = b
|
|
continue
|
|
|
|
LOG.info(f"{relay.fingerprint} verified {b['url']} {sofar}")
|
|
# add our contact info to the trustdb
|
|
aTRUST_DB[relay.fingerprint] = b
|
|
for elt in b['fps']:
|
|
aTRUST_DB_INDEX[elt] = b
|
|
if oArgs.proof_output and oArgs.log_level <= 20:
|
|
# as we go along then clobber
|
|
with open(proof_output_tmp, 'wt') as oFYaml:
|
|
yaml.dump(aTRUST_DB, indent=2, stream=oFYaml)
|
|
oFYaml.close()
|
|
|
|
LOG.info(f"Filtered {len(twhitelist_set)} whitelisted relays")
|
|
texclude_set = texclude_set.difference(twhitelist_set)
|
|
# accept the dns-rsa urls for now until we test them
|
|
texclude_set = texclude_set.difference(tdns_urls)
|
|
LOG.info(f"{len(list(aTRUST_DB.keys()))} good contacts out of {iTotalContacts}")
|
|
|
|
if oArgs.proof_output and aTRUST_DB:
|
|
with open(proof_output_tmp, 'wt') as oFYaml:
|
|
yaml.dump(aTRUST_DB, indent=2, stream=oFYaml)
|
|
oFYaml.close()
|
|
if os.path.exists(oArgs.proof_output):
|
|
bak = oArgs.proof_output +'.bak'
|
|
os.rename(oArgs.proof_output, bak)
|
|
os.rename(proof_output_tmp, oArgs.proof_output)
|
|
LOG.info(f"Wrote {len(list(aTRUST_DB.keys()))} good contact details to {oArgs.proof_output}")
|
|
|
|
if oArgs.torrc_output and texclude_set:
|
|
with open(oArgs.torrc_output, 'wt') as oFTorrc:
|
|
oFTorrc.write(f"{sEXCLUDE_EXIT_KEY} {','.join(texclude_set)}\n")
|
|
oFTorrc.write(f"{sINCLUDE_EXIT_KEY} {','.join(aTRUST_DB_INDEX.keys())}\n")
|
|
oFTorrc.write(f"{sINCLUDE_GUARD_KEY} {','.join(oGOOD_NODES[oGOOD_ROOT]['GuardNodes'])}\n")
|
|
LOG.info(f"Wrote tor configuration to {oArgs.torrc_output}")
|
|
oFTorrc.close()
|
|
|
|
if oArgs.bad_contacts and aBadContacts:
|
|
# for later analysis
|
|
with open(oArgs.bad_contacts, 'wt') as oFYaml:
|
|
yaml.dump(aBadContacts, indent=2, stream=oFYaml)
|
|
oFYaml.close()
|
|
|
|
oBAD_NODES[oBAD_ROOT]['ExcludeNodes']['BadExit'] = list(texclude_set)
|
|
oBAD_NODES[oBAD_ROOT]['ExcludeDomains'] = lKNOWN_NODNS
|
|
vwrite_badnodes(oArgs, oBAD_NODES, str(len(texclude_set)))
|
|
|
|
oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = list(aTRUST_DB_INDEX.keys())
|
|
# GuardNodes are readonl
|
|
vwrite_goodnodes(oArgs, oGOOD_NODES, len(aTRUST_DB_INDEX.keys()))
|
|
retval = 0
|
|
try:
|
|
logging.getLogger('stem').setLevel(30)
|
|
try:
|
|
if texclude_set:
|
|
LOG.info(f"{sEXCLUDE_EXIT_KEY} {len(texclude_set)} net bad exit relays")
|
|
controller.set_conf(sEXCLUDE_EXIT_KEY, texclude_set)
|
|
|
|
except stem.SocketClosed as e:
|
|
LOG.error(f"Failed setting {sEXCLUDE_EXIT_KEY} bad exit relays in Tor")
|
|
retval += 1
|
|
|
|
try:
|
|
if aTRUST_DB_INDEX.keys():
|
|
LOG.info(f"{sINCLUDE_EXIT_KEY} {len(aTRUST_DB_INDEX.keys())} good relays")
|
|
controller.set_conf(sINCLUDE_EXIT_KEY, aTRUST_DB_INDEX.keys())
|
|
except stem.SocketClosed as e:
|
|
LOG.error(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor")
|
|
retval += 1
|
|
|
|
try:
|
|
if 'GuardNodes' in oGOOD_NODES[oGOOD_ROOT].keys():
|
|
LOG.info(f"{sINCLUDE_GUARD_KEY} {len(oGOOD_NODES[oGOOD_ROOT]['GuardNodes'])} guard nodes")
|
|
# FixMe for now override StrictNodes it may be unusable otherwise
|
|
controller.set_conf(sINCLUDE_GUARD_KEY,
|
|
oGOOD_NODES[oGOOD_ROOT]['GuardNodes'])
|
|
cur = controller.get_conf('StrictNodes')
|
|
if oArgs.strict_nodes and int(cur) != oArgs.strict_nodes:
|
|
LOG.info(f"OVERRIDING StrictNodes to {oArgs.strict_nodes}")
|
|
controller.set_conf('StrictNodes', oArgs.strict_nodes)
|
|
else:
|
|
LOG.info(f"StrictNodes is set to {cur}")
|
|
except stem.SocketClosed as e:
|
|
LOG.errro(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor")
|
|
retval += 1
|
|
|
|
sys.stdout.write("dns-rsa domains:\n" +'\n'.join(tdns_urls) +'\n')
|
|
return retval
|
|
|
|
except InvalidRequest as e:
|
|
# Unacceptable option value: Invalid router list.
|
|
LOG.error(str(e))
|
|
retval = 1
|
|
return retval
|
|
except KeyboardInterrupt:
|
|
return 0
|
|
except Exception as e:
|
|
LOG.exception(str(e))
|
|
retval = 2
|
|
return retval
|
|
finally:
|
|
# wierd we are getting stem errors during the final return
|
|
# with a traceback that doesnt correspond to any real flow
|
|
# File "/usr/lib/python3.9/site-packages/stem/control.py", line 2474, in set_conf
|
|
# self.set_options({param: value}, False)
|
|
logging.getLogger('stem').setLevel(40)
|
|
try:
|
|
for elt in controller._event_listeners:
|
|
controller.remove_event_listener(elt)
|
|
controller.close()
|
|
except Exception as e:
|
|
LOG.warn(str(e))
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
i = iMain(sys.argv[1:])
|
|
except IncorrectPassword as e:
|
|
LOG.error(e)
|
|
i = 1
|
|
except KeyboardInterrupt:
|
|
i = 0
|
|
except Exception as e:
|
|
LOG.exception(e)
|
|
i = 2
|
|
sys.exit(i)
|