emdee/toxygen_wrapper
1
0
Fork 0
Code Issues Pull Requests Actions Projects Wiki Activity

refactor out onions

Browse Source
This commit is contained in:
emdee 2022-11-13 14:34:02 +00:00
parent 0b4eda648e
commit b7f6d09761
2 changed files with 40 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
wrapper_tests/support_http.py
View File

@ -23,7 +23,7 @@ lNO_PROXY = ['localhost', '127.0.0.1']
CONNECT_TIMEOUT = 20.0 CONNECT_TIMEOUT = 20.0
def bAreWeConnected(): def bAreWeConnected():
# FixMe: Linux # FixMe: Linux only
sFile = f"/proc/{os.getpid()}/net/route" sFile = f"/proc/{os.getpid()}/net/route"
if not os.path.isfile(sFile): return None if not os.path.isfile(sFile): return None
i = 0 i = 0
@ -75,7 +75,7 @@ def download_url(url, settings=None):
settings = pick_up_proxy_from_environ() settings = pick_up_proxy_from_environ()
if pycurl: if pycurl:
LOG.debug('nodes loading with pycurl: ' + str(url)) LOG.debug('Downloading with pycurl: ' + str(url))
buffer = BytesIO() buffer = BytesIO()
c = pycurl.Curl() c = pycurl.Curl()
c.setopt(c.URL, url) c.setopt(c.URL, url)
@ -114,12 +114,12 @@ def download_url(url, settings=None):
LOG.info('nodes loaded with pycurl: ' + str(url)) LOG.info('nodes loaded with pycurl: ' + str(url))
return result return result
except Exception as ex: except Exception as ex:
LOG.error('TOX nodes loading error with pycurl: ' + str(ex)) LOG.error('TOX Downloading error with pycurl: ' + str(ex))
LOG.error('\n' + traceback.format_exc()) LOG.error('\n' + traceback.format_exc())
# drop through # drop through
if requests: if requests:
LOG.debug('nodes loading with requests: ' + str(url)) LOG.debug('Downloading with requests: ' + str(url))
try: try:
headers = dict() headers = dict()
headers['Content-Type'] = 'application/json' headers['Content-Type'] = 'application/json'
@ -144,11 +144,11 @@ def download_url(url, settings=None):
LOG.info('nodes loaded with requests: ' + str(url)) LOG.info('nodes loaded with requests: ' + str(url))
return result return result
except Exception as ex: except Exception as ex:
LOG.error('TOX nodes loading error with requests: ' + str(ex)) LOG.error('TOX Downloading error with requests: ' + str(ex))
# drop through # drop through
if not settings['proxy_type']: # no proxy if not settings['proxy_type']: # no proxy
LOG.debug('nodes loading with urllib no proxy: ' + str(url)) LOG.debug('Downloading with urllib no proxy: ' + str(url))
try: try:
req = urllib.request.Request(url) req = urllib.request.Request(url)
req.add_header('Content-Type', 'application/json') req.add_header('Content-Type', 'application/json')
@ -157,7 +157,7 @@ def download_url(url, settings=None):
LOG.info('nodes loaded with no proxy: ' + str(url)) LOG.info('nodes loaded with no proxy: ' + str(url))
return result return result
except Exception as ex: except Exception as ex:
LOG.error('TOX nodes loading ' + str(ex)) LOG.error('TOX Downloading ' + str(ex))
return '' return ''
return '' return ''

189
wrapper_tests/support_testing.py
View File

@ -40,6 +40,7 @@ except ImportError:
get_user_config_path = None get_user_config_path = None
from wrapper_tests.support_http import bAreWeConnected from wrapper_tests.support_http import bAreWeConnected
from wrapper_tests.support_onions import sTorResolve
# LOG=util.log # LOG=util.log
global LOG global LOG
@ -146,6 +147,16 @@ def ignoreStderr():
os.dup2(old_stderr, 2) os.dup2(old_stderr, 2)
os.close(old_stderr) os.close(old_stderr)
def clean_booleans(oArgs):
for key in lBOOLEANS:
if not hasattr(oArgs, key): continue
val = getattr(oArgs, key)
if type(val) == bool: continue
if val in ['False', 'false', '0']:
setattr(oArgs, key, False)
else:
setattr(oArgs, key, True)
def on_log(iTox, level, filename, line, func, message, *data): def on_log(iTox, level, filename, line, func, message, *data):
# LOG.debug(repr((level, filename, line, func, message,))) # LOG.debug(repr((level, filename, line, func, message,)))
tox_log_cb(level, filename, line, func, message) tox_log_cb(level, filename, line, func, message)
@ -303,6 +314,10 @@ def oMainArgparser(_=None, iMode=2):
def vSetupLogging(oArgs): def vSetupLogging(oArgs):
global LOG global LOG
logging._defaultFormatter = logging.Formatter(datefmt='%m-%d %H:%M:%S')
logging._defaultFormatter.default_time_format = '%m-%d %H:%M:%S'
logging._defaultFormatter.default_msec_format = ''
add = None add = None
kwargs = dict(level=oArgs.loglevel, kwargs = dict(level=oArgs.loglevel,
format='%(levelname)-8s %(message)s') format='%(levelname)-8s %(message)s')
@ -317,10 +332,11 @@ def vSetupLogging(oArgs):
# https://pypi.org/project/coloredlogs/ # https://pypi.org/project/coloredlogs/
aKw = dict(level=oArgs.loglevel, aKw = dict(level=oArgs.loglevel,
logger=LOG, logger=LOG,
stream=sys.stdout,
fmt='%(name)s %(levelname)s %(message)s' fmt='%(name)s %(levelname)s %(message)s'
) )
coloredlogs.install(**aKw) coloredlogs.install(**aKw)
if add: if oArgs.logfile:
oHandler = logging.FileHandler(oArgs.logfile) oHandler = logging.FileHandler(oArgs.logfile)
LOG.addHandler(oHandler) LOG.addHandler(oHandler)
else: else:
@ -329,9 +345,6 @@ def vSetupLogging(oArgs):
oHandler = logging.StreamHandler(sys.stdout) oHandler = logging.StreamHandler(sys.stdout)
LOG.addHandler(oHandler) LOG.addHandler(oHandler)
logging._defaultFormatter = logging.Formatter(datefmt='%m-%d %H:%M:%S')
logging._defaultFormatter.default_time_format = '%m-%d %H:%M:%S'
logging._defaultFormatter.default_msec_format = ''
LOG.info(f"Setting loglevel to {oArgs.loglevel!s}") LOG.info(f"Setting loglevel to {oArgs.loglevel!s}")
@ -522,35 +535,9 @@ def bootstrap_local(self, elts, lToxes):
LOG.info(f'bootstraping local') LOG.info(f'bootstraping local')
return bootstrap_udp(self, elts, lToxes) return bootstrap_udp(self, elts, lToxes)
def sDNSClean(l): def lDNSClean(l):
# list(set(l).difference(lDEAD_BS)) # [elt for elt in l if elt not in lDEAD_BS]
return [elt for elt in l if elt not in lDEAD_BS] return list(set(l).difference(lDEAD_BS))
oSTEM_CONTROLER = None
def oGetStemController(log_level=10, sock_or_pair='/var/run/tor/control'):
from stem import StreamStatus
from stem.control import EventType, Controller
import getpass
global oSTEM_CONTROLER
if oSTEM_CONTROLER: return oSTEM_CONTROLER
from stem.util.log import Runlevel
Runlevel = log_level
if os.path.exists(sock_or_pair):
controller = Controller.from_socket_file(path=sock_or_pair)
else:
if ':' in sock_or_pair:
port = sock_or_pair.split(':')[1]
else:
port = sock_or_pair
controller = Controller.from_port(port=iPort)
sys.stdout.flush()
p = getpass.unix_getpass(prompt='Controller Password: ', stream=sys.stderr)
controller.authenticate(p)
oSTEM_CONTROLER = controller
LOG.debug(f"{controller}")
return oSTEM_CONTROLER
def lExitExcluder(oArgs, iPort=9051): def lExitExcluder(oArgs, iPort=9051):
""" """
@ -590,118 +577,6 @@ def lExitExcluder(oArgs, iPort=9051):
LOG.exception('ExcludeExitNodes ' +str(e)) LOG.exception('ExcludeExitNodes ' +str(e))
return exit_excludelist return exit_excludelist
def sMapaddressResolv(target, iPort=9051):
if not stem:
LOG.warn('please install the stem Python package')
return ''
try:
controller = oGetStemController(log_level=10)
map_dict = {"0.0.0.0": target}
map_ret = controller.map_address(map_dict)
return map_ret
except Exception as e:
LOG.exception(e)
return ''
def lIntroductionPoints(target, iPort=9051):
if stem == False: return ''
from stem import StreamStatus
from stem.control import EventType, Controller
import getpass
l = []
try:
controller = oGetStemController(log_level=10)
desc = controller.get_hidden_service_descriptor(target)
l = desc.introduction_points()
if l:
LOG.warn(f"{elt} NO introduction points for {target}\n")
return l
LOG.debug(f"{elt} len(l) introduction points for {target}")
for introduction_point in l:
l.append('%s:%s => %s' % (introduction_point.address,
introduction_point.port,
introduction_point.identifier))
except Exception as e:
LOG.exception(e)
return l
def sTorResolve(target,
verbose=False,
sHost='127.0.0.1',
iPort=9050,
SOCK_TIMEOUT_SECONDS=10.0,
SOCK_TIMEOUT_TRIES=3,
):
MAX_INFO_RESPONSE_PACKET_LENGTH = 8
seb = b"\o004\o360\o000\o000\o000\o000\o000\o001\o000"
seb = b"\x04\xf0\x00\x00\x00\x00\x00\x01\x00"
seb += bytes(target, 'US-ASCII') + b"\x00"
assert len(seb) == 10+len(target), str(len(seb))+repr(seb)
# LOG.debug(f"0 Sending {len(seb)} to The TOR proxy {seb}")
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((sHost, iPort))
sock.settimeout(SOCK_TIMEOUT_SECONDS)
oRet = sock.sendall(seb)
i = 0
data = ''
while i < SOCK_TIMEOUT_TRIES:
i += 1
time.sleep(3)
lReady = select.select([sock.fileno()], [], [],
SOCK_TIMEOUT_SECONDS)
if not lReady[0]: continue
try:
flags=socket.MSG_WAITALL
data = sock.recv(MAX_INFO_RESPONSE_PACKET_LENGTH, flags)
except socket.timeout:
LOG.warn("4 The TOR proxy " \
+repr((sHost, iPort)) \
+" didnt reply in " + str(SOCK_TIMEOUT_SECONDS) + " sec."
+" #" +str(i))
except Exception as e:
LOG.error("4 The TOR proxy " \
+repr((sHost, iPort)) \
+" errored with " + str(e)
+" #" +str(i))
sock.close()
raise SystemExit(4)
else:
if len(data) > 0: break
if len(data) == 0:
if i > SOCK_TIMEOUT_TRIES:
sLabel = "5 No reply #"
else:
sLabel = "5 No data #"
LOG.info(sLabel +f"{i} from {sHost} {iPort}" )
sock.close()
raise SystemExit(5)
assert len(data) >= 8
packet_sf = data[1]
if packet_sf == 90:
# , "%d" % packet_sf
assert f"{packet_sf}" == "90", f"packet_sf = {packet_sf}"
return f"{data[4]}.{data[5]}.{data[6]}.{data[7]}"
else:
# 91
LOG.warn(f"tor-resolve failed for {target} from {sHost} {iPort}" )
os.system(f"tor-resolve -4 {target} > /tmp/e 2>/dev/null")
# os.system("strace tor-resolve -4 "+target+" 2>&1|grep '^sen\|^rec'")
return ''
def sDNSLookup(host): def sDNSLookup(host):
ipv = 0 ipv = 0
if host in lDEAD_BS: if host in lDEAD_BS:
@ -778,20 +653,21 @@ def bootstrap_good(lelts, lToxes):
return bootstrap_udp(lelts, lToxes) return bootstrap_udp(lelts, lToxes)
def bootstrap_udp(lelts, lToxes): def bootstrap_udp(lelts, lToxes):
lelts = lDNSClean(lelts)
LOG.debug(f'DHT bootstraping {len(lelts)}') LOG.debug(f'DHT bootstraping {len(lelts)}')
for elt in lToxes: for elt in lToxes:
for largs in sDNSClean(lelts): random.shuffle(lelts)
for largs in lelts:
host, port, key = largs host, port, key = largs
if host in lDEAD_BS: continue
ip = sDNSLookup(host) ip = sDNSLookup(host)
if not ip: if not ip:
LOG.warn(f'bootstrap_udp to {host} did not resolve') LOG.warn(f'bootstrap_udp to {host} did not resolve')
continue continue
assert len(key) == 64, key
if type(port) == str: if type(port) == str:
port = int(port) port = int(port)
try: try:
assert len(key) == 64, key
oRet = elt.bootstrap(ip, oRet = elt.bootstrap(ip,
port, port,
key) key)
@ -809,11 +685,11 @@ def bootstrap_udp(lelts, lToxes):
pass pass
def bootstrap_tcp(lelts, lToxes): def bootstrap_tcp(lelts, lToxes):
for elt in lToxes: lelts = lDNSClean(lelts)
LOG.debug(f'Relay bootstapping {len(lelts)}') for oTox in lToxes:
for largs in sDNSClean(lelts): random.shuffle(lelts)
host, port, key = largs LOG.info(f'bootstrap_tcp bootstapping {[l[0] for l in lelts]}')
if host in lDEAD_BS: continue for (host, port, key,) in lelts:
ip = sDNSLookup(host) ip = sDNSLookup(host)
if not ip: if not ip:
LOG.warn(f'bootstrap_tcp to {host} did not resolve {ip}') LOG.warn(f'bootstrap_tcp to {host} did not resolve {ip}')
@ -823,11 +699,12 @@ def bootstrap_tcp(lelts, lToxes):
l = lIntroductionPoints(host) l = lIntroductionPoints(host)
if not l: if not l:
LOG.warn(f'bootstrap_tcp to {host} has no introduction points') LOG.warn(f'bootstrap_tcp to {host} has no introduction points')
assert len(key) == 64, key continue
if type(port) == str: if type(port) == str:
port = int(port) port = int(port)
try: try:
oRet = elt.add_tcp_relay(ip, assert len(key) == 64, key
oRet = oTox.add_tcp_relay(ip,
port, port,
key) key)
except Exception as e: except Exception as e:
@ -835,7 +712,7 @@ def bootstrap_tcp(lelts, lToxes):
continue continue
if not oRet: if not oRet:
LOG.warn(f'bootstrap_tcp failed to {host} : ' +str(oRet)) LOG.warn(f'bootstrap_tcp failed to {host} : ' +str(oRet))
elif elt.self_get_connection_status() != TOX_CONNECTION['NONE']: elif oTox.self_get_connection_status() != TOX_CONNECTION['NONE']:
LOG.info(f'bootstrap_tcp to {host} connected') LOG.info(f'bootstrap_tcp to {host} connected')
break break
else: else: