emdee/exclude_badExits
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improvements

Browse Source
This commit is contained in:
emdee 2022-11-09 09:30:43 +00:00
parent d11d95aafe
commit 6f3207f02e
4 changed files with 241 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
exclude_badExits.py
View File

@ -83,6 +83,7 @@ import sys
import os
import re
import socket
import time
import argparse
from io import StringIO
@ -120,6 +121,7 @@ LOG = logging.getLogger()
ETC_DIR = '/etc/tor/yaml'
aTRUST_DB = {}
aTRUST_DB_INDEX = {}
sDETAILS_URL = "https://metrics.torproject.org/rs.html#details/"
# You can call this while bootstrapping
sEXCLUDE_EXIT_KEY = 'ExcludeNodes'
@ -131,14 +133,30 @@ sINCLUDE_GUARD_KEY = 'EntryNodes'
lKNOWN_NODNS = [
'0x0.is',
'a9.wtf',
'artikel5ev.de',
'arvanode.net',
'dodo.pm',
'dra-family.github.io',
'eraldonion.org',
'galtland.network',
'interfesse.net',
'kryptonit.org',
'lonet.sh',
'moneneis.de',
'nx42.de',
'ormycloud.org',
'plied-privacy.net',
'rification-for-nusenu.net',
'sv.ch',
'thingtohide.nl',
'tikel10.org',
'tor-exit-2.aa78i2efsewr0neeknk.xyz',
'tor-exit-3.aa78i2efsewr0neeknk.xyz',
'torix-relays.org',
'tse.com',
'tuxli.org',
'w.digidow.eu',
'www.quintex.com',
]
def oMakeController(sSock='', port=9051):
import getpass
@ -157,7 +175,6 @@ def lYamlBadNodes(sFile,
section=sEXCLUDE_EXIT_KEY,
lWanted=['BadExit']):
global oBAD_NODES
root = 'ExcludeNodes'
l = []
if not yaml: return l
if os.path.exists(sFile):
@ -166,54 +183,19 @@ def lYamlBadNodes(sFile,
oBAD_NODES = o
# BROKEN
# root = 'ExcludeNodes'
# for elt in o[oBAD_ROOT][root][section].keys():
# if lWanted and elt not in lWanted: continue
# # l += o[oBAD_ROOT][root][section][elt]
global lKNOWN_NODNS
root = 'ExcludeDomains'
if root not in o[oBAD_ROOT] or not o[oBAD_ROOT][root]:
o[oBAD_ROOT][root] = lKNOWN_NODNS
else:
lKNOWN_NODNS = o[oBAD_ROOT][root]
return l
def icheck_torrc(sFile, oArgs):
l = open(sFile, 'rt').readlines()
a = {}
for elt in l:
elt = elt.strip()
if not elt or not ' ' in elt: continue
k,v = elt.split(' ', 1)
a[k] = v
keys = a
if 'HashedControlPassword' not in keys:
LOG.info('Add HashedControlPassword for security')
print('run: tor --hashcontrolpassword <TopSecretWord>')
if 'ExcludeExitNodes' in keys:
elt = 'BadNodes.ExcludeExitNodes.BadExit'
LOG.warn(f"Remove ExcludeNodes and move then to {oArgs.bad_nodes}")
print(f"move to the {elt} section as a list")
if 'GuardNodes' in keys:
elt = 'GoodNodes.GuardNodes'
LOG.warn(f"Remove GuardNodes and move then to {oArgs.good_nodes}")
print(f"move to the {elt} section as a list")
if 'ExcludeNodes' in keys:
elt = 'BadNodes.ExcludeNodes.BadExit'
LOG.warn(f"Remove ExcludeNodes and move then to {oArgs.bad_nodes}")
print(f"move to the {elt} section as a list")
if 'ControlSocket' not in keys and os.path.exists('/run/tor/control'):
LOG.info('Add ControlSocket /run/tor/control for us')
print('ControlSocket /run/tor/control GroupWritable RelaxDirModeCheck')
if 'UseMicrodescriptors' not in keys or keys['UseMicrodescriptors'] != '1':
LOG.info('Add UseMicrodescriptors 0 for us')
print('UseMicrodescriptors 0')
if 'AutomapHostsSuffixes' not in keys:
LOG.info('Add AutomapHostsSuffixes for onions')
print('AutomapHostsSuffixes .exit,.onion')
if 'AutoMapHostsOnResolve' not in keys:
LOG.info('Add AutoMapHostsOnResolve for onions')
print('AutoMapHostsOnResolve 1')
if 'VirtualAddrNetworkIPv4' not in keys:
LOG.info('Add VirtualAddrNetworkIPv4 for onions')
print('VirtualAddrNetworkIPv4 172.16.0.0/12')
return 0
oGOOD_NODES = {}
oGOOD_ROOT = 'GoodNodes'
def lYamlGoodNodes(sFile='/etc/tor/torrc-goodnodes.yaml'):
@ -248,13 +230,14 @@ def lIntroductionPoints(controller, lOnions):
l += [introduction_point.address]
return l
lBAD_URLS = []
tBAD_URLS = set()
lATS = ['abuse', 'email']
lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime', 'memory']
lBOOLS = ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone',
'sandbox', 'offlinemasterkey']
def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050):
global lBAD_URLS
global tBAD_URLS
global lKNOWN_NODNS
# cleanups for yaml
for elt in lINTS:
if elt in a:
@ -271,12 +254,17 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
a.update({'fps': []})
keys = list(a.keys())
# test the url for fps and add it to the array
if 'proof' not in keys:
LOG.warn(f"{fp} 'proof' not in {keys}")
return a
if aTRUST_DB_INDEX and fp in aTRUST_DB_INDEX.keys():
aCachedContact = aTRUST_DB_INDEX[fp]
if aCachedContact['email'] = a['email']:
return aCachedContact
if 'url' not in keys:
if 'uri' not in keys:
a['url'] = ''
@ -292,12 +280,21 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
a['url'] = 'https:' +a['url']
# domain should be a unique key for contacts
domain = a['url'][8:]
domain = a['url'][8:].strip('/')
try:
ip = sTorResolve(domain)
except Exception as e:
lpair = getaddrinfo(domain, 443)
if lpait is None:
try:
lpair = getaddrinfo(domain, 443)
except (socket.gaierror, ) as e:
LOG.debug("{e}")
lpair = None
lKNOWN_NODNS.append(domain)
except Exception as e:
LOG.warn("{e}")
lpair = None
lKNOWN_NODNS.append(domain)
if lpair is None:
LOG.warn(f"TorResolv and getaddrinfo failed for {domain}")
return a
ip = lpair[0]
@ -324,24 +321,24 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
LOG.exception(f"AttributeError downloading from {domain} {e}")
except CertificateError as e:
LOG.warn(f"CertificateError downloading from {domain} {e}")
lBAD_URLS += [a['url']]
tBAD_URLS.add(a['url'])
except TrustorError as e:
LOG.warn(f"TrustorError downloading from {domain} {e.args}")
lBAD_URLS += [a['url']]
tBAD_URLS.add(a['url'])
except (BaseException ) as e:
LOG.error(f"Exception {type(e)} downloading from {domain} {e}")
else:
if o.status_code >= 300:
LOG.warn(f"Error downloading from {domain} {o.status_code} {o.reason}")
# any reason retry?
lBAD_URLS += [a['url']]
tBAD_URLS.add(a['url'])
return a
l = o.text.upper().strip().split('\n')
if not l:
# already squacked in lD
LOG.warn(f"Downloading from {domain} empty for {fp}")
lBAD_URLS += [a['url']]
tBAD_URLS.add(a['url'])
else:
a['fps'] = [elt for elt in l if elt and len(elt) == 40
and not elt.startswith('#')]
@ -521,13 +518,23 @@ def iMain(lArgs):
icheck_torrc(sFile, oArgs)
global aTRUST_DB
global aTRUST_DB_INDEX
sFile = oArgs.proof_output
if sFile and os.path.exists(sFile):
try:
with open(sFile, 'rt') as oFd:
aTRUST_DB = yaml.safe_load(oFd)
except:
aTRUST_DB = {}
# reverse lookup of fps to contacts
# but...
for k,v in aTRUST_DB:
aTRUST_DB_INDEX[k] = v
if 'fps' in aTRUST_DB[k].keys():
for fp in aTRUST_DB[k]['fps']:
aTRUST_DB_INDEX[fp] = v
except Exception as e:
LOG.warn(f"Error reading YAML TrustDB {sFile} {e}")
if os.path.exists(oArgs.proxy_ctl):
controller = oMakeController(sSock=oArgs.proxy_ctl)
@ -615,7 +622,12 @@ def iMain(lArgs):
i = c.find(' ')
if i >=0:
c = c[:i]
LOG.info(f"{relay.fingerprint} skipping 'dns-rsa' {c} {sofar}")
if c in lKNOWN_NODNS:
LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {c} {sofar}")
exit_excludelist.append(relay.fingerprint)
continue
LOG.info(f"skipping 'dns-rsa' {relay.fingerprint}.{c} {sofar}")
iDnsContact += 1
continue
@ -626,17 +638,18 @@ def iMain(lArgs):
LOG.warn(f"{relay.fingerprint} did not parse {sofar}")
exit_excludelist.append(relay.fingerprint)
continue
if 'url' in a and a['url'] and a['url'] in lBAD_URLS:
# The fp is using a contact with a URL we know is bad
LOG.info(f"{relay.fingerprint} skipping in lBAD_URLS {a['url']} {sofar}")
exit_excludelist.append(relay.fingerprint)
continue
domain = a['url'][8:]
if domain in lKNOWN_NODNS:
# The fp is using a contact with a URL we know is nonexistent
LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {a['url']} {sofar}")
exit_excludelist.append(relay.fingerprint)
continue
if 'url' in a and a['url']:
if a['url'] in tBAD_URLS:
# The fp is using a contact with a URL we know is bad
LOG.info(f"{relay.fingerprint} skipping in tBAD_URLS {a['url']} {sofar}")
exit_excludelist.append(relay.fingerprint)
continue
domain = a['url'].replace('https://', '').replace('http://', '')
if domain in lKNOWN_NODNS:
# The fp is using a contact with a URL we know is bogus
LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {a['url']} {sofar}")
exit_excludelist.append(relay.fingerprint)
continue
b = aVerifyContact(list(a.values())[0],
@ -661,9 +674,13 @@ def iMain(lArgs):
iFakeContact += 1
aBadContacts[relay.fingerprint] = b
continue
# great contact had good fps and we are in them
lProofGoodFps += b['fps']
if relay.fingerprint in aProofUri.keys():
# a cached entry
continue
LOG.info(f"{relay.fingerprint} verified {b['url']} {sofar}")
# add our contact info to the trustdb
aProofUri[relay.fingerprint] = b
@ -706,7 +723,10 @@ def iMain(lArgs):
global oBAD_NODES
oBAD_NODES['BadNodes']['ExcludeNodes']['BadExit'] = exit_excludelist
global lKNOWN_NODNS
o[oBAD_ROOT]['ExcludeDomains'] = lKNOWN_NODNS
vwrite_badnodes(oArgs)
global oGOOD_NODES
oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = lProofGoodFps
vwrite_goodnodes(oArgs)
@ -739,7 +759,7 @@ def iMain(lArgs):
except stem.SocketClosed as e:
LOG.errro(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor")
retval += 1
)
return retval
except InvalidRequest as e:

67
https_adapter.py
View File

@ -261,3 +261,70 @@ class HTTPSAdapter(HTTPAdapter):
raise
return self.build_response(request, resp)
from urllib3.util.ssl_match_hostname import match_hostname as _match_hostname
def match_hostname(cert, hostname):
"""Verify that *cert* (in decoded format as returned by
SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125
rules are followed, but IP addresses are not accepted for *hostname*.
CertificateError is raised on failure. On success, the function
returns nothing.
"""
if not cert:
raise ValueError(
"empty or no certificate, match_hostname needs a "
"SSL socket or SSL context with either "
"CERT_OPTIONAL or CERT_REQUIRED"
)
try:
# Divergence from upstream: ipaddress can't handle byte str
host_ip = ipaddress.ip_address(_to_unicode(hostname))
except (UnicodeError, ValueError):
# ValueError: Not an IP address (common case)
# UnicodeError: Divergence from upstream: Have to deal with ipaddress not taking
# byte strings. addresses should be all ascii, so we consider it not
# an ipaddress in this case
host_ip = None
except AttributeError:
# Divergence from upstream: Make ipaddress library optional
if ipaddress is None:
host_ip = None
else: # Defensive
raise
dnsnames = []
san = cert.get("subjectAltName", ())
for key, value in san:
if key == "DNS":
if host_ip is None and _dnsname_match(value, hostname):
return
dnsnames.append(value)
elif key == "IP Address":
if host_ip is not None and _ipaddress_match(value, host_ip):
return
dnsnames.append(value)
if not dnsnames:
# The subject is only checked when there is no dNSName entry
# in subjectAltName
for sub in cert.get("subject", ()):
for key, value in sub:
# XXX according to RFC 2818, the most specific Common Name
# must be used.
if key == "commonName":
if _dnsname_match(value, hostname):
return
dnsnames.append(value)
if len(dnsnames) > 1:
# soften this to allow subdomain matching
raise CertificateError(
"hostname %r "
"doesn't match any of %s" % (hostname, ", ".join(map(repr, dnsnames)))
)
elif len(dnsnames) == 1:
raise CertificateError("hostname %r doesn't match %r" % (hostname, dnsnames[0]))
else:
raise CertificateError(
"no appropriate commonName or subjectAltName fields were found"
)
from urllib3.util.ssl_match_hostname = match_hostname

60
support_onions.py
View File

@ -2,19 +2,18 @@
import os
import sys
import argparse
import re
import traceback
import logging
import shutil
import json
import socket
import select
from ctypes import *
import time, contextlib
import unittest
from random import Random
random = Random()
import time
global LOG
import logging
import warnings
warnings.filterwarnings('ignore')
LOG = logging.getLogger()
bHAVE_TORR = shutil.which('tor-resolve')
@ -77,7 +76,8 @@ def sTorResolve(target,
SOCK_TIMEOUT_TRIES=3,
):
MAX_INFO_RESPONSE_PACKET_LENGTH = 8
target = target.strip('/')
seb = b"\o004\o360\o000\o000\o000\o000\o000\o001\o000"
seb = b"\x04\xf0\x00\x00\x00\x00\x00\x01\x00"
seb += bytes(target, 'US-ASCII') + b"\x00"
@ -158,3 +158,45 @@ def getaddrinfo(sHost, sPort):
return None
return lPair
def icheck_torrc(sFile, oArgs):
l = open(sFile, 'rt').readlines()
a = {}
for elt in l:
elt = elt.strip()
if not elt or not ' ' in elt: continue
k,v = elt.split(' ', 1)
a[k] = v
keys = a
if 'HashedControlPassword' not in keys:
LOG.info('Add HashedControlPassword for security')
print('run: tor --hashcontrolpassword <TopSecretWord>')
if 'ExcludeExitNodes' in keys:
elt = 'BadNodes.ExcludeExitNodes.BadExit'
LOG.warn(f"Remove ExcludeNodes and move then to {oArgs.bad_nodes}")
print(f"move to the {elt} section as a list")
if 'GuardNodes' in keys:
elt = 'GoodNodes.GuardNodes'
LOG.warn(f"Remove GuardNodes and move then to {oArgs.good_nodes}")
print(f"move to the {elt} section as a list")
if 'ExcludeNodes' in keys:
elt = 'BadNodes.ExcludeNodes.BadExit'
LOG.warn(f"Remove ExcludeNodes and move then to {oArgs.bad_nodes}")
print(f"move to the {elt} section as a list")
if 'ControlSocket' not in keys and os.path.exists('/run/tor/control'):
LOG.info('Add ControlSocket /run/tor/control for us')
print('ControlSocket /run/tor/control GroupWritable RelaxDirModeCheck')
if 'UseMicrodescriptors' not in keys or keys['UseMicrodescriptors'] != '1':
LOG.info('Add UseMicrodescriptors 0 for us')
print('UseMicrodescriptors 0')
if 'AutomapHostsSuffixes' not in keys:
LOG.info('Add AutomapHostsSuffixes for onions')
print('AutomapHostsSuffixes .exit,.onion')
if 'AutoMapHostsOnResolve' not in keys:
LOG.info('Add AutoMapHostsOnResolve for onions')
print('AutoMapHostsOnResolve 1')
if 'VirtualAddrNetworkIPv4' not in keys:
LOG.info('Add VirtualAddrNetworkIPv4 for onions')
print('VirtualAddrNetworkIPv4 172.16.0.0/12')
return 0

60
trustor_poc.py
View File

@ -8,6 +8,7 @@ import requests
from stem.control import Controller
from stem.util.tor_tools import *
from urllib.parse import urlparse
from urllib3.util.retry import Retry
try:
# unbound is not on pypi
@ -29,13 +30,8 @@ try:
except:
TorContactInfoParser = None
# for now we support max_depth = 0 only
# this PoC version has no support for recursion
# https://github.com/nusenu/tor-relay-operator-ids-trust-information#trust-information-consumers
supported_max_depths = ['0']
# https://github.com/nusenu/ContactInfo-Information-Sharing-Specification#ciissversion
accepted_ciissversions = ['2']
class TrustorError(Exception): pass
# https://stackoverflow.com/questions/2532053/validate-a-hostname-string
# FIXME this check allows non-fqdn names
@ -47,7 +43,6 @@ def is_valid_hostname(hostname):
allowed = re.compile("(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)
return all(allowed.match(x) for x in hostname.split("."))
def read_local_trust_config(trust_config):
'''
reads a local configuration file containing trusted domains
@ -55,6 +50,12 @@ def read_local_trust_config(trust_config):
'''
result = []
# for now we support max_depth = 0 only
# this PoC version has no support for recursion
# https://github.com/nusenu/tor-relay-operator-ids-trust-information#trust-information-consumers
supported_max_depths = ['0']
if (os.path.isfile(trust_config)):
f = open(trust_config)
for line in f:
@ -150,6 +151,8 @@ def find_validation_candidates(controller, trusted_domains=[],validation_cache=[
# https://github.com/nusenu/ContactInfo-Information-Sharing-Specification#proof
accepted_proof_types = ['uri-rsa','dns-rsa']
# https://github.com/nusenu/ContactInfo-Information-Sharing-Specification#ciissversion
accepted_ciissversions = ['2']
result = {}
@ -202,7 +205,7 @@ def find_validation_candidates(controller, trusted_domains=[],validation_cache=[
result[domain] = {prooftype : [fingerprint]}
return result
def lDownloadUrlFps(domain, sCAfile, timeout=30, host='127.0.0.1', port=9050):
def oDownloadUrl(domain, sCAfile, timeout=30, host='127.0.0.1', port=9050):
uri="https://"+domain+"/.well-known/tor-relay/rsa-fingerprint.txt"
# socks proxy used for outbound web requests (for validation of proofs)
proxy = {'https': 'socks5h://' +host +':' +str(port)}
@ -216,28 +219,28 @@ def lDownloadUrlFps(domain, sCAfile, timeout=30, host='127.0.0.1', port=9050):
# urllib3.connection WARNING Certificate did not match expected hostname:
head = requests.head(uri, timeout=timeout, proxies=proxy, headers=headers)
except Exception as e:
LOG.warn(f"HTTP HEAD request failed for {uri} {e}")
head = None
return []
if head.status_code != 200:
return []
raise TrustorError(f"HTTP HEAD request failed for {uri} {e}")
if head.status_code >= 300:
raise TrustorError(f"HTTP Errorcode {head.status_code}")
if not head.headers['Content-Type'].startswith('text/plain'):
return []
assert os.path.exists(sCAfile), sCAfile
raise TrustorError(f"HTTP Content-Type != text/plain")
if not os.path.exists(sCAfile):
raise TrustorError(f"File not found CAfile {sCAfile}")
try:
from https_adapter import HTTPSAdapter
except Exception as e:
LOG.warn(f"Could not import HTTPSAdapter {e}")
HTTPSAdapter = None
HTTPSAdapter = None
try:
with requests.sessions.Session() as session:
if HTTPSAdapter:
retries = Retry(connect=3, read=2, redirect=0)
# FixMe: upgrade to TLS1.3
session.mount("https://", HTTPSAdapter(pool_maxsize=1,
max_retries=3,))
fullfile = session.request(method="get", url=uri,
max_retries=retries,))
oReqResp = session.request(method="get", url=uri,
proxies=proxy, timeout=timeout,
headers=headers,
allow_redirects=False,
@ -245,16 +248,21 @@ def lDownloadUrlFps(domain, sCAfile, timeout=30, host='127.0.0.1', port=9050):
)
except:
LOG.warn("HTTP GET request failed for %s" % uri)
return []
if fullfile.status_code != 200 or not fullfile.headers['Content-Type'].startswith('text/plain'):
return []
raise
if oReqResp.status_code != 200:
raise TrustorError(f"HTTP Errorcode {head.status_code}")
if not oReqResp.headers['Content-Type'].startswith('text/plain'):
raise TrustorError(f"HTTP Content-Type != text/plain")
#check for redirects (not allowed as per spec)
if fullfile.url != uri:
LOG.error('Redirect detected %s vs %s (final)' % (uri, fullfile.url))
return []
if oReqResp.url != uri:
LOG.error(f'Redirect detected %s vs %s (final)' % (uri, oReqResp.url))
raise TrustorError(f'Redirect detected %s vs %s (final)' % (uri, oReqResp.url))
return oReqResp
well_known_content = fullfile.text.upper().strip().split('\n')
def lDownloadUrlFps(domain, sCAfile, timeout=30, host='127.0.0.1', port=9050):
o = oDownloadUrl(domain, sCAfile, timeout=timeout, host=host, port=port)
well_known_content = o.text.upper().strip().split('\n')
well_known_content = [i for i in well_known_content if i and len(i) == 40]
return well_known_content