396 lines
16 KiB
Python
396 lines
16 KiB
Python
# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -
|
|
|
|
# from https://github.com/nusenu/trustor-poc
|
|
# with minor refactoring to make the code more Pythonic.
|
|
|
|
import os
|
|
import sys
|
|
import datetime
|
|
|
|
import requests
|
|
from stem.control import Controller
|
|
from stem.util.tor_tools import *
|
|
from urllib.parse import urlparse
|
|
from urllib3.util.retry import Retry
|
|
|
|
try:
|
|
# unbound is not on pypi
|
|
from unbound import ub_ctx,RR_TYPE_TXT,RR_CLASS_IN
|
|
except:
|
|
ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None
|
|
|
|
global LOG
|
|
import logging
|
|
import warnings
|
|
warnings.filterwarnings('ignore')
|
|
LOG = logging.getLogger()
|
|
|
|
# download this python library from
|
|
# https://github.com/erans/torcontactinfoparser
|
|
#sys.path.append('/home/....')
|
|
try:
|
|
from torcontactinfo import TorContactInfoParser
|
|
except:
|
|
TorContactInfoParser = None
|
|
|
|
class TrustorError(Exception): pass
|
|
|
|
# https://stackoverflow.com/questions/2532053/validate-a-hostname-string
|
|
# FIXME this check allows non-fqdn names
|
|
def is_valid_hostname(hostname):
|
|
if len(hostname) > 255:
|
|
return False
|
|
if hostname[-1] == ".":
|
|
hostname = hostname[:-1] # strip exactly one dot from the right, if present
|
|
allowed = re.compile("(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)
|
|
return all(allowed.match(x) for x in hostname.split("."))
|
|
|
|
def read_local_trust_config(trust_config):
|
|
'''
|
|
reads a local configuration file containing trusted domains
|
|
and returns them in an array
|
|
'''
|
|
|
|
result = []
|
|
|
|
# for now we support max_depth = 0 only
|
|
# this PoC version has no support for recursion
|
|
# https://github.com/nusenu/tor-relay-operator-ids-trust-information#trust-information-consumers
|
|
supported_max_depths = ['0']
|
|
|
|
if (os.path.isfile(trust_config)):
|
|
f = open(trust_config)
|
|
for line in f:
|
|
line = line.strip()
|
|
if line[0] == '#':
|
|
continue
|
|
try:
|
|
domain, max_depth = line.split(':')
|
|
except:
|
|
LOG.error('invalid trust config line detected: %s aborting!' % line)
|
|
sys.exit(8)
|
|
|
|
if max_depth in supported_max_depths:
|
|
if is_valid_hostname(domain) and domain not in result:
|
|
result.append(domain)
|
|
else:
|
|
LOG.error('invalid duplicate domain in trust config file: %s: %s aborting!' % (trust_config, domain))
|
|
sys.exit(9)
|
|
else:
|
|
LOG.error('unsupported max_depth value (%s) used, aborting!' % line)
|
|
sys.exit(10)
|
|
|
|
return result
|
|
else:
|
|
LOG.error("trust config file %s missing, aborting!" % trust_config)
|
|
sys.exit(11)
|
|
|
|
|
|
def read_local_validation_cache(validation_cache_file, trusted_domains=[]):
|
|
'''
|
|
reads the local validation cache and returns all fingerprints in the cache
|
|
for trusted domains
|
|
|
|
format of each entry in the cache:
|
|
domain:fingerprint:prooftype:date
|
|
'''
|
|
|
|
result = []
|
|
if trusted_domains == []:
|
|
return result
|
|
if os.path.isfile(validation_cache_file):
|
|
with open(validation_cache_file, 'rt') as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line[0] == '#':
|
|
continue
|
|
try:
|
|
domain, fingerprint, prooftype, dt = line.split(':')
|
|
except:
|
|
LOG.error('invalid trust cache entry detected: %s aborting!' % line)
|
|
sys.exit(12)
|
|
|
|
if domain in trusted_domains:
|
|
result.append(fingerprint)
|
|
else:
|
|
LOG.warn('ignoring cached entry for untrusted domain %s' % domain)
|
|
|
|
else:
|
|
LOG.info("Validation cache file not present. It will be created.")
|
|
return result
|
|
|
|
def get_controller(address='127.0.0.1', port=9151, password=''):
|
|
'''
|
|
connects to a local tor client via the tor ControlPort
|
|
and returns a controller that allows us to easily set specific tor
|
|
configuration options or read tor relay ContactInfo strings for validation
|
|
'''
|
|
|
|
try:
|
|
#controller = Controller.from_socket_file(path=torsocketpath)
|
|
controller = Controller.from_port(address=address, port=port)
|
|
controller.authenticate(password=password)
|
|
except Exception as e:
|
|
LOG.error(f"Failed to connect to the tor process, {e}")
|
|
sys.exit(1)
|
|
|
|
if not controller.is_set('UseMicrodescriptors'):
|
|
LOG.error('"UseMicrodescriptors 0" is required in your torrc configuration. Exiting.')
|
|
sys.exit(2)
|
|
|
|
return controller
|
|
|
|
def find_validation_candidates(controller, trusted_domains=[],validation_cache=[],accept_all=False):
|
|
'''
|
|
connect to a tor client via controlport and return a dict of all
|
|
not yet validated fingerprints per trusted operators
|
|
format:
|
|
{ trusted_domain: { prooftype: [fingerprint, fingerprint, ...]} }
|
|
|
|
example content:
|
|
{ 'emeraldonion.org' : { 'uri-rsa': ['044600FD968728A6F220D5347AD897F421B757C0', '09DCA3360179C6C8A5A20DDDE1C54662965EF1BA']}}
|
|
'''
|
|
# https://github.com/nusenu/ContactInfo-Information-Sharing-Specification#proof
|
|
accepted_proof_types = ['uri-rsa','dns-rsa']
|
|
|
|
# https://github.com/nusenu/ContactInfo-Information-Sharing-Specification#ciissversion
|
|
accepted_ciissversions = ['2']
|
|
|
|
result = {}
|
|
|
|
try:
|
|
relays = controller.get_server_descriptors()
|
|
except:
|
|
LOG.error('Failed to get relay descriptors via tor\'s ControlPort. Exiting.')
|
|
sys.exit(3)
|
|
|
|
ci = TorContactInfoParser()
|
|
|
|
for relay in relays:
|
|
if relay.contact:
|
|
fingerprint = relay.fingerprint
|
|
# skip fingerprints we have already successfully validated in the past
|
|
# a future version would check the cache age as well
|
|
if fingerprint in validation_cache:
|
|
continue
|
|
contactstring = relay.contact.decode('utf-8')
|
|
parsed_ci = ci.parse(contactstring)
|
|
if len(parsed_ci) > 0:
|
|
if 'ciissversion' in parsed_ci and 'proof' in parsed_ci and 'url' in parsed_ci:
|
|
prooftype = parsed_ci['proof']
|
|
ciurl = parsed_ci['url']
|
|
if parsed_ci['ciissversion'] in accepted_ciissversions and prooftype in accepted_proof_types:
|
|
if ciurl.startswith('http://') or ciurl.startswith('https://'):
|
|
try:
|
|
domain=urlparse(ciurl).netloc
|
|
except:
|
|
LOG.warning('failed to parse domain %s' % ciurl)
|
|
domain='error'
|
|
continue
|
|
else:
|
|
domain=ciurl
|
|
if not is_valid_hostname(domain):
|
|
domain='error'
|
|
continue
|
|
# we can ignore relays that do not claim to be operated by a trusted operator
|
|
# if we do not accept all
|
|
if domain not in trusted_domains and not accept_all:
|
|
continue
|
|
if domain in result.keys():
|
|
if prooftype in result[domain].keys():
|
|
result[domain][prooftype].append(fingerprint)
|
|
else:
|
|
result[domain] = { prooftype : [fingerprint] }
|
|
# mixed proof types are not allowd as per spec but we are not strict here
|
|
LOG.warning('%s is using mixed prooftypes %s' % (domain, prooftype))
|
|
else:
|
|
result[domain] = {prooftype : [fingerprint]}
|
|
return result
|
|
|
|
def oDownloadUrl(domain, sCAfile, timeout=30, host='127.0.0.1', port=9050):
|
|
uri="https://"+domain+"/.well-known/tor-relay/rsa-fingerprint.txt"
|
|
# socks proxy used for outbound web requests (for validation of proofs)
|
|
proxy = {'https': 'socks5h://' +host +':' +str(port)}
|
|
# we use this UA string when connecting to webservers to fetch rsa-fingerprint.txt proof files
|
|
# https://nusenu.github.io/ContactInfo-Information-Sharing-Specification/#uri-rsa
|
|
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0'}
|
|
|
|
LOG.debug("fetching %s...." % uri)
|
|
try:
|
|
# grr. fix urllib3
|
|
# urllib3.connection WARNING Certificate did not match expected hostname:
|
|
head = requests.head(uri, timeout=timeout, proxies=proxy, headers=headers)
|
|
except Exception as e:
|
|
raise TrustorError(f"HTTP HEAD request failed for {uri} {e}")
|
|
|
|
if head.status_code >= 300:
|
|
raise TrustorError(f"HTTP Errorcode {head.status_code}")
|
|
if not head.headers['Content-Type'].startswith('text/plain'):
|
|
raise TrustorError(f"HTTP Content-Type != text/plain")
|
|
if not os.path.exists(sCAfile):
|
|
raise TrustorError(f"File not found CAfile {sCAfile}")
|
|
try:
|
|
from https_adapter import HTTPSAdapter
|
|
except Exception as e:
|
|
LOG.warn(f"Could not import HTTPSAdapter {e}")
|
|
HTTPSAdapter = None
|
|
raise SystemExit(f"{e}")
|
|
else:
|
|
LOG.info(f"Loaded HTTPSAdapter")
|
|
|
|
try:
|
|
with requests.sessions.Session() as session:
|
|
if HTTPSAdapter:
|
|
retries = Retry(connect=3, read=2, redirect=0)
|
|
# FixMe: upgrade to TLS1.3
|
|
session.mount("https://", HTTPSAdapter(pool_maxsize=1,
|
|
max_retries=retries,))
|
|
oReqResp = session.request(method="get", url=uri,
|
|
proxies=proxy, timeout=timeout,
|
|
headers=headers,
|
|
allow_redirects=False,
|
|
verify=True
|
|
)
|
|
except:
|
|
LOG.warn("HTTP GET request failed for %s" % uri)
|
|
raise
|
|
if oReqResp.status_code != 200:
|
|
raise TrustorError(f"HTTP Errorcode {head.status_code}")
|
|
if not oReqResp.headers['Content-Type'].startswith('text/plain'):
|
|
raise TrustorError(f"HTTP Content-Type != text/plain")
|
|
|
|
#check for redirects (not allowed as per spec)
|
|
if oReqResp.url != uri:
|
|
LOG.error(f'Redirect detected %s vs %s (final)' % (uri, oReqResp.url))
|
|
raise TrustorError(f'Redirect detected %s vs %s (final)' % (uri, oReqResp.url))
|
|
return oReqResp
|
|
|
|
def lDownloadUrlFps(domain, sCAfile, timeout=30, host='127.0.0.1', port=9050):
|
|
o = oDownloadUrl(domain, sCAfile, timeout=timeout, host=host, port=port)
|
|
well_known_content = o.text.upper().strip().split('\n')
|
|
well_known_content = [i for i in well_known_content if i and len(i) == 40]
|
|
return well_known_content
|
|
|
|
def validate_proofs(candidates, validation_cache_file, timeout=20, host='127.0.0.1', port=9050):
|
|
'''
|
|
This function takes the return value of find_validation_candidates()
|
|
and validated them according to their proof type (uri-rsa, dns-rsa)
|
|
and writes properly validated relay fingerprints to the local validation cache
|
|
'''
|
|
dt_utc = datetime.datetime.now(datetime.timezone.utc).date()
|
|
|
|
f = open(validation_cache_file, mode='a')
|
|
count = 0
|
|
|
|
for domain in candidates.keys():
|
|
for prooftype in candidates[domain].keys():
|
|
if prooftype == 'uri-rsa':
|
|
well_known_content = lDownloadUrlFps(domain, timeout=timeout, host=host, port=port)
|
|
for fingerprint in candidates[domain][prooftype]:
|
|
if fingerprint in well_known_content:
|
|
# write cache entry
|
|
count += 1
|
|
f.write('%s:%s:%s:%s\n' % (domain, fingerprint, prooftype, dt_utc))
|
|
else:
|
|
LOG.error('%s:%s:%s' % (fingerprint, domain, prooftype))
|
|
elif prooftype == 'dns-rsa' and ub_ctx:
|
|
for fingerprint in candidates[domain][prooftype]:
|
|
fp_domain = fingerprint+'.'+domain
|
|
if idns_validate(fp_domain,
|
|
libunbound_resolv_file='resolv.conf',
|
|
dnssec_DS_file='dnssec-root-trust',
|
|
) == 0:
|
|
count += 1
|
|
f.write('%s:%s:%s:%s\n' % (domain, fingerprint, prooftype, dt_utc))
|
|
else:
|
|
LOG.error('%s:%s:%s' % (fingerprint, domain, prooftype))
|
|
f.close()
|
|
LOG.info('successfully validated %s new (not yet validated before) relays' % count)
|
|
|
|
def idns_validate(domain,
|
|
libunbound_resolv_file='resolv.conf',
|
|
dnssec_DS_file='dnssec-root-trust',
|
|
):
|
|
'''
|
|
performs DNS TXT lookups and verifies the reply
|
|
- is DNSSEC valid and
|
|
- contains only a single TXT record
|
|
- the DNS record contains a hardcoded string as per specification
|
|
https://nusenu.github.io/ContactInfo-Information-Sharing-Specification/#dns-rsa
|
|
'''
|
|
if not ub_ctx: return -1
|
|
|
|
# this is not the system wide /etc/resolv.conf
|
|
# use dnscrypt-proxy to encrypt your DNS and route it via tor's SOCKSPort
|
|
|
|
|
|
ctx = ub_ctx()
|
|
if (os.path.isfile(libunbound_resolv_file)):
|
|
ctx.resolvconf(libunbound_resolv_file)
|
|
else:
|
|
LOG.error('libunbound resolv config file: "%s" is missing, aborting!' % libunbound_resolv_file)
|
|
return 5
|
|
if (os.path.isfile(dnssec_DS_file)):
|
|
ctx.add_ta_file(dnssec_DS_file)
|
|
else:
|
|
LOG.error('DNSSEC trust anchor file "%s" is missing, aborting!' % dnssec_DS_file)
|
|
return 6
|
|
|
|
status, result = ctx.resolve(domain, RR_TYPE_TXT, RR_CLASS_IN)
|
|
if status == 0 and result.havedata:
|
|
if len(result.rawdata) == 1 and result.secure:
|
|
# ignore the first byte, it is the TXT length
|
|
if result.data.as_raw_data()[0][1:] == b'we-run-this-tor-relay':
|
|
return 0
|
|
return 1
|
|
|
|
def configure_tor(controller, trusted_fingerprints, exitonly=True):
|
|
'''
|
|
takes the list of trusted fingerprints and configures a tor client
|
|
to only use trusted relays in a certain position
|
|
for now we only set exits.
|
|
we refuse to set the configuration if there are less then 40 trusted relays
|
|
'''
|
|
|
|
relay_count = len(trusted_fingerprints)
|
|
|
|
if relay_count < 41:
|
|
LOG.error('Too few trusted relays (%s), aborting!' % relay_count)
|
|
sys.exit(15)
|
|
|
|
try:
|
|
controller.set_conf('ExitNodes', trusted_fingerprints)
|
|
LOG.error('limited exits to %s relays' % relay_count)
|
|
except Exception as e:
|
|
LOG.exception('Failed to set ExitNodes tor config to trusted relays')
|
|
sys.exit(20)
|
|
|
|
if __name__ == '__main__':
|
|
trust_config = 'trust_config'
|
|
assert os.path.exists(trust_config)
|
|
trusted_domains = read_local_trust_config(trust_config)
|
|
|
|
validation_cache_file = 'validation_cache'
|
|
trusted_fingerprints = read_local_validation_cache(validation_cache_file,
|
|
trusted_domains=trusted_domains)
|
|
# tor ControlPort password
|
|
controller_password=''
|
|
# tor ControlPort IP
|
|
controller_address = '127.0.0.1'
|
|
timeout = 20
|
|
port = 9050
|
|
controller = get_controller(address=controller_address,password=controller_password)
|
|
|
|
r = find_validation_candidates(controller,
|
|
validation_cache=trusted_fingerprints,
|
|
trusted_domains=trusted_domains)
|
|
validate_proofs(r, validation_cache_file,
|
|
timeout=timeout,
|
|
host=controller_address,
|
|
port=port)
|
|
|
|
# refresh list with newly validated fingerprints
|
|
trusted_fingerprints = read_local_validation_cache(validation_cache_file,
|
|
trusted_domains=trusted_domains)
|
|
configure_tor(controller, trusted_fingerprints)
|