Changed to sets

This commit is contained in:
emdee 2022-11-09 18:43:54 +00:00
parent fdab13e065
commit ae22d14437
3 changed files with 56 additions and 51 deletions

View File

@ -146,6 +146,7 @@ lKNOWN_NODNS = [
'nx42.de', 'nx42.de',
'ormycloud.org', 'ormycloud.org',
'plied-privacy.net', 'plied-privacy.net',
'redacted.org',
'rification-for-nusenu.net', 'rification-for-nusenu.net',
'sv.ch', 'sv.ch',
'thingtohide.nl', 'thingtohide.nl',
@ -179,8 +180,7 @@ def lYamlBadNodes(sFile,
if not yaml: return l if not yaml: return l
if os.path.exists(sFile): if os.path.exists(sFile):
with open(sFile, 'rt') as oFd: with open(sFile, 'rt') as oFd:
o = yaml.safe_load(oFd) oBAD_NODES = yaml.safe_load(oFd)
oBAD_NODES = o
# BROKEN # BROKEN
# root = 'ExcludeNodes' # root = 'ExcludeNodes'
@ -190,10 +190,10 @@ def lYamlBadNodes(sFile,
global lKNOWN_NODNS global lKNOWN_NODNS
root = 'ExcludeDomains' root = 'ExcludeDomains'
if root not in o[oBAD_ROOT] or not o[oBAD_ROOT][root]: if root not in oBAD_NODES[oBAD_ROOT] or not oBAD_NODES[oBAD_ROOT][root]:
o[oBAD_ROOT][root] = lKNOWN_NODNS oBAD_NODES[oBAD_ROOT][root] = lKNOWN_NODNS
else: else:
lKNOWN_NODNS = o[oBAD_ROOT][root] lKNOWN_NODNS = oBAD_NODES[oBAD_ROOT][root]
return l return l
oGOOD_NODES = {} oGOOD_NODES = {}
@ -261,10 +261,9 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
if aTRUST_DB_INDEX and fp in aTRUST_DB_INDEX.keys(): if aTRUST_DB_INDEX and fp in aTRUST_DB_INDEX.keys():
aCachedContact = aTRUST_DB_INDEX[fp] aCachedContact = aTRUST_DB_INDEX[fp]
if aCachedContact['email'] = a['email']: if aCachedContact['email'] == a['email']:
return aCachedContact return aCachedContact
if 'url' not in keys: if 'url' not in keys:
if 'uri' not in keys: if 'uri' not in keys:
a['url'] = '' a['url'] = ''
@ -281,15 +280,16 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
# domain should be a unique key for contacts # domain should be a unique key for contacts
domain = a['url'][8:].strip('/') domain = a['url'][8:].strip('/')
if domain in lKNOWN_NODNS:
LOG.warn(f"{domain} in lKNOWN_NODNS")
return {}
try: try:
ip = sTorResolve(domain) ip = sTorResolve(domain)
except Exception as e: except Exception as e:
ip = ''
if ip == '':
try: try:
lpair = getaddrinfo(domain, 443) lpair = getaddrinfo(domain, 443)
except (socket.gaierror, ) as e:
LOG.debug("{e}")
lpair = None
lKNOWN_NODNS.append(domain)
except Exception as e: except Exception as e:
LOG.warn("{e}") LOG.warn("{e}")
lpair = None lpair = None
@ -479,31 +479,35 @@ def oMainArgparser(_=None):
help="Write the proof data of the included nodes to a YAML file") help="Write the proof data of the included nodes to a YAML file")
return parser return parser
def vwrite_badnodes(oArgs, oBAD_NODES): def vwrite_badnodes(oArgs, oBAD_NODES, slen):
if oArgs.bad_nodes: if oArgs.bad_nodes:
tmp = oArgs.bad_nodes +'.tmp' tmp = oArgs.bad_nodes +'.tmp'
bak = oArgs.bad_nodes +'.bak' bak = oArgs.bad_nodes +'.bak'
with open(tmp, 'wt') as oFYaml: with open(tmp, 'wt') as oFYaml:
yaml.dump(oBAD_NODES, indent=2, stream=oFYaml) yaml.dump(oBAD_NODES, indent=2, stream=oFYaml)
LOG.info(f"Wrote {len(list(oBAD_NODES.keys()))} to {oArgs.bad_nodes}") LOG.info(f"Wrote {slen} to {oArgs.bad_nodes}")
oFYaml.close() oFYaml.close()
if os.path.exists(oArgs.bad_nodes): if os.path.exists(oArgs.bad_nodes):
os.rename(oArgs.bad_nodes, bak) os.rename(oArgs.bad_nodes, bak)
os.rename(tmp, oArgs.bad_nodes) os.rename(tmp, oArgs.bad_nodes)
def vwrite_goodnodes(oArgs, oGOOD_NODES): def vwrite_goodnodes(oArgs, oGOOD_NODES, slen):
if oArgs.good_nodes: if oArgs.good_nodes:
tmp = oArgs.good_nodes +'.tmp' tmp = oArgs.good_nodes +'.tmp'
bak = oArgs.good_nodes +'.bak' bak = oArgs.good_nodes +'.bak'
with open(tmp, 'wt') as oFYaml: with open(tmp, 'wt') as oFYaml:
yaml.dump(oGOOD_NODES, indent=2, stream=oFYaml) yaml.dump(oGOOD_NODES, indent=2, stream=oFYaml)
LOG.info(f"Wrote {len(list(oGOOD_NODES.keys()))} good nodes to {oArgs.good_nodes}") LOG.info(f"Wrote {slen} good nodes to {oArgs.good_nodes}")
oFYaml.close() oFYaml.close()
if os.path.exists(oArgs.good_nodes): if os.path.exists(oArgs.good_nodes):
os.rename(oArgs.good_nodes, bak) os.rename(oArgs.good_nodes, bak)
os.rename(tmp, oArgs.good_nodes) os.rename(tmp, oArgs.good_nodes)
def iMain(lArgs): def iMain(lArgs):
global aTRUST_DB
global aTRUST_DB_INDEX
global lKNOWN_NODNS
parser = oMainArgparser() parser = oMainArgparser()
oArgs = parser.parse_args(lArgs) oArgs = parser.parse_args(lArgs)
@ -515,9 +519,6 @@ def iMain(lArgs):
if sFile and os.path.exists(sFile): if sFile and os.path.exists(sFile):
icheck_torrc(sFile, oArgs) icheck_torrc(sFile, oArgs)
global aTRUST_DB
global aTRUST_DB_INDEX
sFile = oArgs.proof_output sFile = oArgs.proof_output
if sFile and os.path.exists(sFile): if sFile and os.path.exists(sFile):
try: try:
@ -525,14 +526,14 @@ def iMain(lArgs):
aTRUST_DB = yaml.safe_load(oFd) aTRUST_DB = yaml.safe_load(oFd)
# reverse lookup of fps to contacts # reverse lookup of fps to contacts
# but... # but...
for k,v in aTRUST_DB: for k,v in aTRUST_DB.items():
aTRUST_DB_INDEX[k] = v aTRUST_DB_INDEX[k] = v
if 'fps' in aTRUST_DB[k].keys(): if 'fps' in aTRUST_DB[k].keys():
for fp in aTRUST_DB[k]['fps']: for fp in aTRUST_DB[k]['fps']:
aTRUST_DB_INDEX[fp] = v aTRUST_DB_INDEX[fp] = v
except Exception as e: except Exception as e:
LOG.warn(f"Error reading YAML TrustDB {sFile} {e}") LOG.exception(f"Error reading YAML TrustDB {sFile} {e}")
if os.path.exists(oArgs.proxy_ctl): if os.path.exists(oArgs.proxy_ctl):
controller = oMakeController(sSock=oArgs.proxy_ctl) controller = oMakeController(sSock=oArgs.proxy_ctl)
@ -576,8 +577,6 @@ def iMain(lArgs):
LOG.info(f"lYamlBadNodes {len(exit_excludelist)}") LOG.info(f"lYamlBadNodes {len(exit_excludelist)}")
relays = controller.get_server_descriptors()
tProofGoodFps = set() tProofGoodFps = set()
iDnsContact = 0 iDnsContact = 0
lBadContactUrls = [] lBadContactUrls = []
@ -586,6 +585,8 @@ def iMain(lArgs):
aProofUri = {} aProofUri = {}
lConds = oArgs.contact.split(',') lConds = oArgs.contact.split(',')
iR = 0 iR = 0
relays = controller.get_server_descriptors()
for relay in relays: for relay in relays:
iR += 1 iR += 1
if not is_valid_fingerprint(relay.fingerprint): if not is_valid_fingerprint(relay.fingerprint):
@ -611,26 +612,35 @@ def iMain(lArgs):
tProofGoodFps.add(relay.fingerprint) tProofGoodFps.add(relay.fingerprint)
continue continue
if relay.contact and b'dns-rsa' in relay.contact.lower(): if type(relay.contact) == bytes:
# dunno
relay.contact = str(relay.contact, 'UTF-8') relay.contact = str(relay.contact, 'UTF-8')
c = relay.contact.lower()
i = c.find('url:') if ('Empty' in lConds and not relay.contact) or \
if i >=0: ('NoEmail' in lConds and relay.contact and not '@' in relay.contact):
c = c[i+4:]
i = c.find(' ')
if i >=0:
c = c[:i]
if c in lKNOWN_NODNS:
LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {c} {sofar}")
exit_excludelist.append(relay.fingerprint) exit_excludelist.append(relay.fingerprint)
continue continue
LOG.info(f"skipping 'dns-rsa' {relay.fingerprint}.{c} {sofar}") if not relay.contact:
iDnsContact += 1 # should be unreached 'Empty' should always be in lConds
continue continue
if relay.contact and b'proof:uri-rsa' in relay.contact.lower(): c = relay.contact.lower()
relay.contact = str(relay.contact, 'UTF-8') i = c.find('url:')
if i >=0: c = c[i+4:]
i = c.find(' ')
if i >=0: c = c[:i]
domain = c.replace('https://', '').replace('http://', '').strip('/')
if domain in lKNOWN_NODNS:
LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {domain} {sofar}")
exit_excludelist.append(relay.fingerprint)
elif 'dns-rsa' in relay.contact.lower():
LOG.info(f"skipping 'dns-rsa' {relay.fingerprint}.{domain} {sofar}")
iDnsContact += 1
elif 'proof:uri-rsa' in relay.contact.lower():
a = aParseContact(relay.contact, relay.fingerprint) a = aParseContact(relay.contact, relay.fingerprint)
if not a: if not a:
LOG.warn(f"{relay.fingerprint} did not parse {sofar}") LOG.warn(f"{relay.fingerprint} did not parse {sofar}")
@ -657,7 +667,7 @@ def iMain(lArgs):
host=oArgs.proxy_host, host=oArgs.proxy_host,
port=oArgs.proxy_port) port=oArgs.proxy_port)
if not b['fps'] or not b['url']: if not b or not 'fps' in b or not b['fps'] or not b['url']:
LOG.warn(f"{relay.fingerprint} did NOT VERIFY {sofar}") LOG.warn(f"{relay.fingerprint} did NOT VERIFY {sofar}")
# If it's giving contact info that doesnt check out # If it's giving contact info that doesnt check out
# it could be a bad exit with fake contact info # it could be a bad exit with fake contact info
@ -687,11 +697,6 @@ def iMain(lArgs):
with open(proof_output_tmp, 'wt') as oFYaml: with open(proof_output_tmp, 'wt') as oFYaml:
yaml.dump(aProofUri, indent=2, stream=oFYaml) yaml.dump(aProofUri, indent=2, stream=oFYaml)
oFYaml.close() oFYaml.close()
continue
if ('Empty' in lConds and not relay.contact) or \
('NoEmail' in lConds and relay.contact and not b'@' in relay.contact):
exit_excludelist.append(relay.fingerprint)
exit_excludelist = list(set(exit_excludelist).difference(set(lGoodOverrideSet))) exit_excludelist = list(set(exit_excludelist).difference(set(lGoodOverrideSet)))
@ -721,13 +726,12 @@ def iMain(lArgs):
global oBAD_NODES global oBAD_NODES
oBAD_NODES[oBAD_ROOT]['ExcludeNodes']['BadExit'] = exit_excludelist oBAD_NODES[oBAD_ROOT]['ExcludeNodes']['BadExit'] = exit_excludelist
global lKNOWN_NODNS
oBAD_NODES[oBAD_ROOT]['ExcludeDomains'] = lKNOWN_NODNS oBAD_NODES[oBAD_ROOT]['ExcludeDomains'] = lKNOWN_NODNS
vwrite_badnodes(oArgs, oBAD_NODES) vwrite_badnodes(oArgs, oBAD_NODES, str(len(exit_excludelist)))
global oGOOD_NODES global oGOOD_NODES
oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = tProofGoodFps oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = tProofGoodFps
vwrite_goodnodes(oArgs, oGOOD_NODES) vwrite_goodnodes(oArgs, oGOOD_NODES, str(len(tProofGoodFps)))
retval = 0 retval = 0
try: try:

View File

@ -326,5 +326,6 @@ def match_hostname(cert, hostname):
raise CertificateError( raise CertificateError(
"no appropriate commonName or subjectAltName fields were found" "no appropriate commonName or subjectAltName fields were found"
) )
from urllib3.util.ssl_match_hostname = match_hostname
urllib3.util.ssl_match_hostname = match_hostname

View File

@ -153,8 +153,8 @@ def getaddrinfo(sHost, sPort):
lPair = lElts[0][-1] lPair = lElts[0][-1]
assert len(lPair) == 2, repr(lPair) assert len(lPair) == 2, repr(lPair)
assert type(lPair[1]) == int, repr(lPair) assert type(lPair[1]) == int, repr(lPair)
except Exception as e: except (socket.gaierror, OSError, BaseException) as e:
LOG.exception(e) LOG.error(e)
return None return None
return lPair return lPair