Added lookupdns.py
This commit is contained in:
emdee
parent
2ea65cc181
commit
94c0834092
@ -198,7 +198,7 @@ def lYamlGoodNodes(sFile='/etc/tor/torrc-goodnodes.yaml'):
|
||||
# yq '.Nodes.IntroductionPoints|.[]' < /etc/tor/torrc-goodnodes.yaml
|
||||
return l
|
||||
|
||||
def bdomain_is_bad(domain):
|
||||
def bdomain_is_bad(domain, fp):
|
||||
global lKNOWN_NODNS
|
||||
if domain in lKNOWN_NODNS: return True
|
||||
if domain in lMAYBE_NODNS:
|
||||
@ -209,8 +209,9 @@ def bdomain_is_bad(domain):
|
||||
lMAYBE_NODNS.remove(domain)
|
||||
return True
|
||||
|
||||
if '@' in domain:
|
||||
LOG.warn(f"@ in domain {domain}")
|
||||
for elt in '@(){}$!':
|
||||
if elt in domain:
|
||||
LOG.warn(f"{elt} in domain {domain}")
|
||||
return True
|
||||
return False
|
||||
|
||||
@ -273,10 +274,11 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
|
||||
|
||||
# domain should be a unique key for contacts
|
||||
domain = a['url'][8:]
|
||||
if bdomain_is_bad(domain):
|
||||
if bdomain_is_bad(domain, fp):
|
||||
LOG.warn(f"{domain} is bad from {a['url']}")
|
||||
LOG.info(f"{domain} is bad from {a}")
|
||||
LOG.debug(f"{fp} is bad from {a}")
|
||||
return a
|
||||
|
||||
ip = zResolveDomain(domain)
|
||||
if ip == '':
|
||||
aFP_EMAIL[fp] = a['email']
|
||||
@ -357,7 +359,7 @@ def aParseContactYaml(contact, fp):
|
||||
LOG.debug(f"{fp} {a}")
|
||||
return a
|
||||
key = ''
|
||||
for elt in lets:
|
||||
for elt in lelts:
|
||||
if key == '':
|
||||
key = elt
|
||||
continue
|
||||
@ -424,22 +426,22 @@ def vsetup_logging(log_level, logfile=''):
|
||||
LOG.info(f"SSetting log_level to {log_level!s}")
|
||||
|
||||
logging._levelToName = {
|
||||
CRITICAL: 'CRITICAL',
|
||||
ERROR: 'ERROR',
|
||||
WARNING: 'WARN',
|
||||
INFO: 'INFO',
|
||||
DEBUG: 'DEBUG',
|
||||
NOTSET: 'NOTSET',
|
||||
logging.CRITICAL: 'CRITICAL',
|
||||
logging.ERROR: 'ERROR',
|
||||
logging.WARNING: 'WARN',
|
||||
logging.INFO: 'INFO',
|
||||
logging.DEBUG: 'DEBUG',
|
||||
logging.NOTSET: 'NOTSET',
|
||||
}
|
||||
logging._nameToLevel = {
|
||||
'CRITICAL': CRITICAL,
|
||||
'FATAL': FATAL,
|
||||
'ERROR': ERROR,
|
||||
'WARN': WARNING,
|
||||
'WARNING': WARNING,
|
||||
'INFO': INFO,
|
||||
'DEBUG': DEBUG,
|
||||
'NOTSET': NOTSET,
|
||||
'CRITICAL': logging.CRITICAL,
|
||||
'FATAL': logging.FATAL,
|
||||
'ERROR': logging.ERROR,
|
||||
'WARN': logging.WARNING,
|
||||
'WARNING': logging.WARNING,
|
||||
'INFO': logging.INFO,
|
||||
'DEBUG': logging.DEBUG,
|
||||
'NOTSET': logging.NOTSET,
|
||||
}
|
||||
|
||||
def oMainArgparser(_=None):
|
||||
@ -498,7 +500,7 @@ def oMainArgparser(_=None):
|
||||
help="Seconds to wait for Tor to booststrap")
|
||||
parser.add_argument('--points_timeout', type=int, default=0,
|
||||
help="Timeout for getting introduction points - must be long >120sec. 0 means disabled looking for IPs")
|
||||
parser.add_argument('--log_level', type=int, default=10,
|
||||
parser.add_argument('--log_level', type=int, default=20,
|
||||
help="10=debug 20=info 30=warn 40=error")
|
||||
parser.add_argument('--bad_sections', type=str,
|
||||
default='MyBadExit',
|
||||
@ -524,13 +526,13 @@ def vwrite_badnodes(oArgs, oBAD_NODES, slen):
|
||||
os.rename(oArgs.bad_nodes, bak)
|
||||
os.rename(tmp, oArgs.bad_nodes)
|
||||
|
||||
def vwrite_goodnodes(oArgs, oGOOD_NODES, slen):
|
||||
def vwrite_goodnodes(oArgs, oGOOD_NODES, ilen):
|
||||
if oArgs.good_nodes:
|
||||
tmp = oArgs.good_nodes +'.tmp'
|
||||
bak = oArgs.good_nodes +'.bak'
|
||||
with open(tmp, 'wt') as oFYaml:
|
||||
yaml.dump(oGOOD_NODES, indent=2, stream=oFYaml)
|
||||
LOG.info(f"Wrote {slen} good nodes to {oArgs.good_nodes}")
|
||||
LOG.info(f"Wrote {ilen} good relays to {oArgs.good_nodes}")
|
||||
oFYaml.close()
|
||||
if os.path.exists(oArgs.good_nodes):
|
||||
os.rename(oArgs.good_nodes, bak)
|
||||
@ -628,7 +630,7 @@ def iMain(lArgs):
|
||||
LOG.info(f"Preloaded {len(texclude_set)} bad fps")
|
||||
|
||||
ttrust_db_index = aTRUST_DB_INDEX.keys()
|
||||
tdns_contacts = set()
|
||||
tdns_urls = set()
|
||||
iFakeContact = 0
|
||||
iTotalContacts = 0
|
||||
aBadContacts = {}
|
||||
@ -644,7 +646,7 @@ def iMain(lArgs):
|
||||
continue
|
||||
relay.fingerprint = relay.fingerprint.upper()
|
||||
|
||||
sofar = f"G:{len(aTRUST_DB.keys())} U:{len(tdns_contacts)} F:{iFakeContact} BF:{len(texclude_set)} GF:{len(ttrust_db_index)} TC:{iTotalContacts} #{iR}"
|
||||
sofar = f"G:{len(aTRUST_DB.keys())} U:{len(tdns_urls)} F:{iFakeContact} BF:{len(texclude_set)} GF:{len(ttrust_db_index)} TC:{iTotalContacts} #{iR}"
|
||||
if not relay.exit_policy.is_exiting_allowed():
|
||||
if sEXCLUDE_EXIT_KEY == 'ExcludeNodes':
|
||||
pass # LOG.debug(f"{relay.fingerprint} not an exit {sofar}")
|
||||
@ -671,10 +673,11 @@ def iMain(lArgs):
|
||||
continue
|
||||
iTotalContacts += 1
|
||||
|
||||
fp = relay.fingerprint
|
||||
if relay.contact and not 'url:' in relay.contact:
|
||||
LOG.info(f"{relay.fingerprint} skipping bad contact - no url: {sofar}")
|
||||
LOG.debug(f"{relay.fingerprint} {relay.contact} {sofar}")
|
||||
texclude_set.add(relay.fingerprint)
|
||||
LOG.info(f"{fp} skipping bad contact - no url: {sofar}")
|
||||
LOG.debug(f"{fp} {relay.contact} {sofar}")
|
||||
texclude_set.add(fp)
|
||||
continue
|
||||
|
||||
c = relay.contact.lower()
|
||||
@ -688,17 +691,17 @@ def iMain(lArgs):
|
||||
i = c.find('/')
|
||||
if i >=0: c = c[:i]
|
||||
domain = c
|
||||
if domain and bdomain_is_bad(domain):
|
||||
LOG.info(f"{relay.fingerprint} skipping bad {domain} {sofar}")
|
||||
LOG.debug(f"{relay.fingerprint} {relay.contact} {sofar}")
|
||||
texclude_set.add(relay.fingerprint)
|
||||
if domain and bdomain_is_bad(domain, fp):
|
||||
LOG.info(f"{fp} skipping bad {domain} {sofar}")
|
||||
LOG.debug(f"{fp} {relay.contact} {sofar}")
|
||||
texclude_set.add(fp)
|
||||
continue
|
||||
|
||||
if domain:
|
||||
ip = zResolveDomain(domain)
|
||||
if not ip:
|
||||
LOG.warn(f"{relay.fingerprint} {domain} did not resolve {sofar}")
|
||||
texclude_set.add(relay.fingerprint)
|
||||
LOG.warn(f"{fp} {domain} did not resolve {sofar}")
|
||||
texclude_set.add(fp)
|
||||
lKNOWN_NODNS.append(domain)
|
||||
iFakeContact += 1
|
||||
continue
|
||||
@ -706,7 +709,7 @@ def iMain(lArgs):
|
||||
if 'dns-rsa' in relay.contact.lower():
|
||||
target = f"{relay.fingerprint}.{domain}"
|
||||
LOG.info(f"skipping 'dns-rsa' {target} {sofar}")
|
||||
tdns_contacts.add(target)
|
||||
tdns_urls.add(target)
|
||||
|
||||
elif 'proof:uri-rsa' in relay.contact.lower():
|
||||
a = aParseContact(relay.contact, relay.fingerprint)
|
||||
@ -769,6 +772,8 @@ def iMain(lArgs):
|
||||
|
||||
LOG.info(f"Filtered {len(twhitelist_set)} whitelisted relays")
|
||||
texclude_set = texclude_set.difference(twhitelist_set)
|
||||
# accept the dns-rsa urls for now until we test them
|
||||
texclude_set = texclude_set.difference(tdns_urls)
|
||||
LOG.info(f"{len(list(aTRUST_DB.keys()))} good contacts out of {iTotalContacts}")
|
||||
|
||||
if oArgs.proof_output and aTRUST_DB:
|
||||
@ -785,7 +790,7 @@ def iMain(lArgs):
|
||||
with open(oArgs.torrc_output, 'wt') as oFTorrc:
|
||||
oFTorrc.write(f"{sEXCLUDE_EXIT_KEY} {','.join(texclude_set)}\n")
|
||||
oFTorrc.write(f"{sINCLUDE_EXIT_KEY} {','.join(aTRUST_DB_INDEX.keys())}\n")
|
||||
oFTorrc.write(f"{sINCLUDE_GUARD_KEY} {','.join(o[oGOOD_ROOT]['GuardNodes'])}\n")
|
||||
oFTorrc.write(f"{sINCLUDE_GUARD_KEY} {','.join(oGOOD_NODES[oGOOD_ROOT]['GuardNodes'])}\n")
|
||||
LOG.info(f"Wrote tor configuration to {oArgs.torrc_output}")
|
||||
oFTorrc.close()
|
||||
|
||||
@ -801,7 +806,7 @@ def iMain(lArgs):
|
||||
|
||||
oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = list(aTRUST_DB_INDEX.keys())
|
||||
# GuardNodes are readonl
|
||||
vwrite_goodnodes(oArgs, oGOOD_NODES, str(len(ttrust_db_index)))
|
||||
vwrite_goodnodes(oArgs, oGOOD_NODES, len(aTRUST_DB_INDEX.keys()))
|
||||
retval = 0
|
||||
try:
|
||||
logging.getLogger('stem').setLevel(30)
|
||||
@ -838,7 +843,7 @@ def iMain(lArgs):
|
||||
LOG.errro(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor")
|
||||
retval += 1
|
||||
|
||||
LOG.info("dns-rsa domains:\n{'\n'.join(tdns_contacts)}")
|
||||
sys.stdout.write("dns-rsa domains:\n" +'\n'.join(tdns_urls) +'\n')
|
||||
return retval
|
||||
|
||||
except InvalidRequest as e:
|
||||
|
||||
@ -27,9 +27,8 @@ LOG = logging.getLogger()
|
||||
|
||||
bHAVE_TORR = shutil.which('tor-resolve')
|
||||
|
||||
# maybe we should check these each time but we
|
||||
# got them by sorting bad relays in the wild
|
||||
# we'll keep a copy here
|
||||
# we check these each time but we got them by sorting bad relays
|
||||
# in the wild we'll keep a copy here so we can avoid restesting
|
||||
yKNOWN_NODNS = """
|
||||
---
|
||||
- 0x0.is
|
||||
@ -50,6 +49,7 @@ yKNOWN_NODNS = """
|
||||
- or.wowplanet.de
|
||||
- ormycloud.org
|
||||
- plied-privacy.net
|
||||
- rivacysvcs.net
|
||||
- redacted.org
|
||||
- rification-for-nusenu.net
|
||||
- rofl.cat
|
||||
|
||||
@ -140,7 +140,11 @@ def get_controller(address='127.0.0.1', port=9151, password=''):
|
||||
|
||||
return controller
|
||||
|
||||
def find_validation_candidates(controller, trusted_domains=[],validation_cache=[],accept_all=False):
|
||||
def find_validation_candidates(controller,
|
||||
trusted_domains=[],
|
||||
validation_cache=[],
|
||||
CAfile='/etc/ssl/certs/ca-certificates.crt',
|
||||
accept_all=False):
|
||||
'''
|
||||
connect to a tor client via controlport and return a dict of all
|
||||
not yet validated fingerprints per trusted operators
|
||||
@ -529,6 +533,7 @@ def configure_tor(controller, trusted_fingerprints, exitonly=True):
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
CAfile = '/etc/ssl/certs/ca-certificates.crt'
|
||||
trust_config = 'trust_config'
|
||||
assert os.path.exists(trust_config)
|
||||
trusted_domains = read_local_trust_config(trust_config)
|
||||
@ -546,7 +551,8 @@ if __name__ == '__main__':
|
||||
|
||||
r = find_validation_candidates(controller,
|
||||
validation_cache=trusted_fingerprints,
|
||||
trusted_domains=trusted_domains)
|
||||
trusted_domains=trusted_domains,
|
||||
CAfile=CAfile)
|
||||
validate_proofs(r, validation_cache_file,
|
||||
timeout=timeout,
|
||||
host=controller_address,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user