From 14c71563088dba0edd63dd7d035f22b762ceeadd Mon Sep 17 00:00:00 2001 From: emdee Date: Sun, 23 Oct 2022 22:14:35 +0000 Subject: [PATCH] First refactor for tox_irc_sync --- README.md | 4 +- wrapper/tox.py | 8 +- wrapper_tests/support_testing.py | 270 +++++++++++++++++++++---------- wrapper_tests/tests_wrapper.py | 18 ++- 4 files changed, 206 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index 62e9db1..fbfe629 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ Others include: This was the basis for the TokTok/py-toxcore-c code until recently. To our point of view, the ability of CTYPEs to follow code in the -debugger is crucial. +debugger is a crucial advantage. Work on this project is suspended until the -[MultiDevice](https://git.plastiras.org/emdee/tox_profile/wiki/MultiDevice-Announcements-POC) problem is solved. Fork me! +[MultiDevice](https://git.macaw.me/emdee/tox_profile/wiki/MultiDevice-Announcements-POC) problem is solved. Fork me! diff --git a/wrapper/tox.py b/wrapper/tox.py index 33dfd23..a463ec1 100644 --- a/wrapper/tox.py +++ b/wrapper/tox.py @@ -416,7 +416,7 @@ class Tox: # ----------------------------------------------------------------------------------------------------------------- def self_get_toxid(self, address=None): - return self_get_address(self, address=None) + return self.self_get_address(address) def self_get_address(self, address=None): """ @@ -2661,7 +2661,7 @@ class Tox: raise ToxError(f"group_send_private_message {TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE[error.value]}") return result - def group_send_message(self, group_number, type, message): + def group_send_message(self, group_number, type_, message): """ Send a text chat message to the group. @@ -2673,7 +2673,7 @@ class Tox: then reassemble the fragments. Messages may not be empty. :param group_number: The group number of the group the message is intended for. - :param type: Message type (normal, action, ...). + :param type_: Message type (normal, action, ...). :param message: A non-NULL pointer to the first element of a byte array containing the message text. :return True on success. @@ -2687,7 +2687,7 @@ class Tox: # bool tox_group_send_message(const Tox *tox, uint32_t group_number, Tox_Message_Type type, const uint8_t *message, size_t length, uint32_t *message_id, Tox_Err_Group_Send_Message *error) result = Tox.libtoxcore.tox_group_send_message(self._tox_pointer, group_number, - type, + type_, message, len(message), # dunno diff --git a/wrapper_tests/support_testing.py b/wrapper_tests/support_testing.py index aed1904..08cf0d2 100644 --- a/wrapper_tests/support_testing.py +++ b/wrapper_tests/support_testing.py @@ -25,6 +25,10 @@ try: # https://pypi.org/project/coloredlogs/ except ImportError as e: coloredlogs = False +try: + import stem +except ImportError as e: + stem = False import wrapper from wrapper.toxcore_enums_and_consts import TOX_CONNECTION, TOX_USER_STATUS @@ -56,7 +60,7 @@ else: connection_opts={'IO': 'TCP', 'PORT': 6666} intf = Mserver.ServerInterface(connection_opts=connection_opts) dbg_opts = { 'interface': intf } - print('Starting TCP server listening on port 6666.') + print(f'Starting TCP server listening on port 6666.') debug(dbg_opts=dbg_opts) return @@ -90,6 +94,7 @@ sTOX_VERSION = "1000002018" bHAVE_NMAP = shutil.which('nmap') bHAVE_JQ = shutil.which('jq') bHAVE_BASH = shutil.which('bash') +bHAVE_TORR = shutil.which('tor-resolve') lDEAD_BS = [ # [notice] Have tried resolving or connecting to address @@ -98,7 +103,7 @@ lDEAD_BS = [ '172.93.52.70', 'tox.abilinski.com', # Failed to resolve "tox3.plastiras.org". - "tox3.plastiras.org", +# "tox3.plastiras.org", ] @@ -273,8 +278,8 @@ def oMainArgparser(_=None): parser.add_argument('--nodes_json', type=str, default='') parser.add_argument('--network', type=str, - choices=['old', 'new', 'local', 'newlocal'], - default='new') + choices=['old', 'main', 'local'], + default='main') parser.add_argument('--download_nodes_url', type=str, default='https://nodes.tox.chat/json') parser.add_argument('--logfile', default=logfile, @@ -292,24 +297,61 @@ def oMainArgparser(_=None): help='Sleep method - one of qt, gevent , time') return parser -def vOargsToxPreamble(oArgs, Tox, ToxTest): +def vSetupLogging(oArgs): + global LOG kwargs = dict(level=oArgs.loglevel, format='%(levelname)-8s %(message)s') if oArgs.logfile: kwargs['filename'] = oArgs.logfile - logging.basicConfig(**kwargs) + + if coloredlogs: + # https://pypi.org/project/coloredlogs/ + coloredlogs.install(level=oArgs.loglevel, + logger=LOG, + # %(asctime)s,%(msecs)03d %(hostname)s [%(process)d] + fmt='%(name)s %(levelname)s %(message)s' + ) + else: + logging.basicConfig(**kwargs) + + logging._defaultFormatter = logging.Formatter(datefmt='%m-%d %H:%M:%S') + logging._defaultFormatter.default_time_format = '%m-%d %H:%M:%S' + logging._defaultFormatter.default_msec_format = '' + LOG.info(f"Setting loglevel to {oArgs.loglevel!s}") + if oArgs.logfile: + assert os.path.exists(oArgs.logfile) - methods = set([x for x in dir(Tox) if not x[0].isupper() - and not x[0] == '_']) - docs = "".join([getattr(ToxTest, x).__doc__ for x in dir(ToxTest) - if getattr(ToxTest, x).__doc__ is not None]) - tested = set(re.findall(r't:(.*?)\n', docs)) - not_tested = methods.difference(tested) +def setup_logging(oArgs): + global LOG + if coloredlogs: + aKw = dict(level=oArgs.loglevel, + logger=LOG, + fmt='%(name)s %(levelname)s %(message)s') + if oArgs.logfile: + oFd = open(oArgs.logfile, 'wt') + setattr(oArgs, 'log_oFd', oFd) + aKw['stream'] = oFd + coloredlogs.install(**aKw) +# logging._defaultFormatter = coloredlogs.Formatter(datefmt='%m-%d %H:%M:%S') + if oArgs.logfile: + oHandler = logging.StreamHandler(stream=sys.stdout) + LOG.addHandler(oHandler) + else: + aKw = dict(level=oArgs.loglevel, + format='%(name)s %(levelname)-4s %(message)s') + if oArgs.logfile: + aKw['filename'] = oArgs.logfile + logging.basicConfig(**aKw) - logging.info('Test Coverage: %.2f%%' % (len(tested) * 100.0 / len(methods))) - if len(not_tested): - logging.info('Not tested:\n %s' % "\n ".join(sorted(list(not_tested)))) + + logging._defaultFormatter = logging.Formatter(datefmt='%m-%d %H:%M:%S') + logging._defaultFormatter.default_time_format = '%m-%d %H:%M:%S' + logging._defaultFormatter.default_msec_format = '' + + LOG.setLevel(oArgs.loglevel) + LOG.trace = lambda l: LOG.log(0, repr(l)) + LOG.info(f"Setting loglevel to {oArgs.loglevel!s}") def signal_handler(num, f): from trepan.interfaces import server as Mserver @@ -374,6 +416,8 @@ DEFAULT_NODES_COUNT = 8 global aNODES aNODES = {} +import functools +# @functools.lru_cache(maxsize=12) def generate_nodes(oArgs=None, nodes_count=DEFAULT_NODES_COUNT, ipv='ipv4', @@ -381,11 +425,15 @@ def generate_nodes(oArgs=None, global aNODES sKey = ipv sKey += ',0' if udp_not_tcp else ',1' - if sKey in aNODES: return aNODES[sKey] + if sKey in aNODES and aNODES[sKey]: + return aNODES[sKey] sFile = _get_nodes_path(oArgs=oArgs) - aNODES[sKey] = generate_nodes_from_file(sFile, - nodes_count=nodes_count, - ipv=ipv, udp_not_tcp=udp_not_tcp) + assert os.path.exists(sFile), sFile + lNodes = generate_nodes_from_file(sFile, + nodes_count=nodes_count, + ipv=ipv, udp_not_tcp=udp_not_tcp) + assert lNodes + aNODES[sKey] = lNodes return aNODES[sKey] aNODES_CACHE = {} @@ -399,7 +447,7 @@ I had a conversation with @irungentoo on IRC about whether we really need to cal """ global aNODES_CACHE - key = sFile +',' +ipv + key = ipv key += ',0' if udp_not_tcp else ',1' if key in aNODES_CACHE: sorted_nodes = aNODES_CACHE[key] @@ -419,18 +467,20 @@ I had a conversation with @irungentoo on IRC about whether we really need to cal if udp_not_tcp: nodes = [(node[ipv], node['port'], node['public_key'],) for node in json_nodes if node[ipv] != 'NONE' \ - and node["status_udp"] in [True, "true"] + and node["status_udp"] in [True, "true"] ] else: nodes = [] elts = [(node[ipv], node['tcp_ports'], node['public_key'],) \ for node in json_nodes if node[ipv] != 'NONE' \ - and node['last_ping'] > 0 - and node["status_tcp"] in [True, "true"] + and node["status_tcp"] in [True, "true"] ] - for (ipv4, ports, public_key,) in elts: + for (ipv, ports, public_key,) in elts: for port in ports: - nodes += [(ipv4, port, public_key)] + nodes += [(ipv, port, public_key)] + if not nodes: + LOG.warn(f'empty generate_nodes from {sFile} {json_nodes!r}') + return [] sorted_nodes = nodes aNODES_CACHE[key] = sorted_nodes @@ -457,62 +507,132 @@ def bootstrap_local(self, elts, lToxes): else: iRet = os.system("netstat -nle4|grep -q :33") if iRet > 0: - LOG.warn('bootstraping local No local DHT running') - LOG.info('bootstraping local') + LOG.warn(f'bootstraping local No local DHT running') + LOG.info(f'bootstraping local') return bootstrap_good(self, elts, lToxes) +def sDNSLookup(host): + ipv = 0 + if host in lDEAD_BS: + LOG.warn(f"address skipped because in lDEAD_BS {host}") + return '' +# return host + try: + s = host.replace('.','') + int(s) + except: + try: + s = host.replace(':','') + int(s) + except: pass + else: + ipv = 6 + else: + ipv = 4 + + if ipv > 0: + LOG.debug(f"address is {ipv} {host}") + return host + + if host.endswith('.tox') or host.endswith('.tox.onion'): + if not bHAVE_TORR: + LOG.warn(f"onion address skipped because no tor-resolve {host}") + return '' + try: + sOut = f"/tmp/TR{os.getpid()}.log" + i = os.system(f"tor-resolve -4 {host} > {sOUT}") + if not i: + LOG.warn(f"onion address skipped because tor-resolve on {host}") + return '' + ip = open(sOut, 'rt').read() + if ip.endswith('failed.'): + LOG.warn(f"onion address skipped because tor-resolve failed on {host}") + return '' + LOG.debug(f"onion address tor-resolve {ip} on {host}") + return ip + except: + pass + else: + try: + ip = socket.gethostbyname(host) + except: + LOG.warn(f"address skipped because socket.gethostbyname failed on {host}") + return '' + LOG.debug(f'{host} {ip}') + if ip == '': + try: + sOut = f"/tmp/TR{os.getpid()}.log" + i = os.system(f"dig {host}|grep ^{host}|sed -e 's/.* //'> {sOUT}") + if not i: + LOG.warn(f"onion address skipped because tor-resolve on {host}") + return '' + ip = open(sOut, 'rt').read().strip() + LOG.debug(f"address dig {ip} on {host}") + return ip + except: + ip = host + return ip + +def bootstrap_udp(lelts, lToxes): + return bootstrap_good(lelts, lToxes) + def bootstrap_good(lelts, lToxes): - LOG.info('bootstraping udp') for elt in lToxes: + LOG.info(f'UDP bootstrapping {len(lelts)}') for largs in lelts: host, port, key = largs - if largs[0] in lDEAD_BS: continue - try: - host = socket.gethostbyname(largs[0]) - except: - continue + ip = sDNSLookup(host) + if not ip: + LOG.warn(f'bootstrap_good to {host} did not resolve') + ip = host assert len(key) == 64, key if type(port) == str: port = int(port) try: - oRet = elt.bootstrap(host, + oRet = elt.bootstrap(ip, port, - largs[2]) + key) except Exception as e: - LOG.error('bootstrap to ' +host +':' +str(largs[1]) \ + LOG.error(f'bootstrap to {host}:' +str(largs[1]) \ +' ' +str(e)) continue if not oRet: - LOG.warn('bootstrap failed to ' +host +' : ' +str(oRet)) + LOG.warn(f'bootstrap failed to {host} : ' +str(oRet)) elif elt.self_get_connection_status() != TOX_CONNECTION['NONE']: - LOG.info('bootstrap to ' +host +' connected') + LOG.info(f'bootstrap to {host} connected') break else: - LOG.debug('bootstrap to ' +host +' not connected') + # LOG.debug(f'bootstrap to {host} not connected') + pass def bootstrap_tcp(lelts, lToxes): - LOG.info('bootstraping tcp') for elt in lToxes: + LOG.info(f'TCP bootstapping {len(lelts)}') for largs in lelts: - if largs[0] in lDEAD_BS: continue + host, port, key = largs + ip = sDNSLookup(host) + if not ip: + LOG.warn(f'bootstrap_tcp to {host} did not resolve {ip}') +# continue + ip = host + assert len(key) == 64, key + if type(port) == str: + port = int(port) try: - host = socket.gethostbyname(largs[0]) - except: - continue - try: - oRet = elt.add_tcp_relay(host, - int(largs[1]), - largs[2]) + oRet = elt.add_tcp_relay(ip, + port, + key) except Exception as e: - LOG.error('bootstrap_tcp to ' +largs[0] +' : ' +str(e)) + LOG.error(f'bootstrap_tcp to {host} : ' +str(e)) continue if not oRet: - LOG.warn('bootstrap_tcp failed to ' +largs[0] +' : ' +str(oRet)) + LOG.warn(f'bootstrap_tcp failed to {host} : ' +str(oRet)) elif elt.self_get_connection_status() != TOX_CONNECTION['NONE']: - LOG.info('bootstrap_tcp to ' +largs[0] +' connected') + LOG.info(f'bootstrap_tcp to {host} connected') break else: - LOG.debug('bootstrap_tcp to ' +largs[0] +' not connected') + # LOG.debug(f'bootstrap_tcp to {host} not connected') + pass def bootstrap_iNmapInfo(lElts, oArgs, bIS_LOCAL=False, iNODES=iNODES): if not bIS_LOCAL and not bAreWeConnected(): @@ -528,12 +648,19 @@ def bootstrap_iNmapInfo(lElts, oArgs, bIS_LOCAL=False, iNODES=iNODES): env = os.environ lRetval = [] for elts in lElts[:iNODES]: - if elts[0] in lDEAD_BS: continue + host, port, key = elts + ip = sDNSLookup(host) + if not ip: + LOG.info('bootstrap_iNmapInfo to {host} did not resolve') + continue + assert len(key) == 64, key + if type(port) == str: + port = int(port) iRet = -1 try: - iRet = iNmapInfo(protocol, *elts) + iRet = iNmapInfo(protocol, ip, port, key) if iRet != 0: - LOG.warn('iNmapInfo to ' +repr(elts[0]) +' retval=' +str(iRet)) + LOG.warn('iNmapInfo to ' +repr(host) +' retval=' +str(iRet)) lRetval += [False] else: LOG.info(f'bootstrap_iNmapInfo ' @@ -544,42 +671,11 @@ def bootstrap_iNmapInfo(lElts, oArgs, bIS_LOCAL=False, iNODES=iNODES): ) lRetval += [True] except Exception as e: - LOG.error('iNmapInfo to ' +repr(elts[0]) +' : ' +str(e) \ + LOG.error('iNmapInfo to {host} : ' +str(e) \ +'\n' + traceback.format_exc()) lRetval += [False] return any(lRetval) -def setup_logging(oArgs): - global LOG - if coloredlogs: - aKw = dict(level=oArgs.loglevel, - logger=LOG, - fmt='%(name)s %(levelname)s %(message)s') - if False and oArgs.logfile: - oFd = open(oArgs.logfile, 'wt') - setattr(oArgs, 'log_oFd', oFd) - aKw['stream'] = oFd - coloredlogs.install(**aKw) -# logging._defaultFormatter = coloredlogs.Formatter(datefmt='%m-%d %H:%M:%S') - if oArgs.logfile: - oHandler = logging.StreamHandler(stream=sys.stdout) - LOG.addHandler(oHandler) - else: - aKw = dict(level=oArgs.loglevel, - format='%(name)s %(levelname)-4s %(message)s') - if oArgs.logfile: - aKw['filename'] = oArgs.logfile - logging.basicConfig(**aKw) - - - logging._defaultFormatter = logging.Formatter(datefmt='%m-%d %H:%M:%S') - logging._defaultFormatter.default_time_format = '%m-%d %H:%M:%S' - logging._defaultFormatter.default_msec_format = '' - - LOG.setLevel(oArgs.loglevel) - LOG.trace = lambda l: LOG.log(0, repr(l)) - LOG.info(f"Setting loglevel to {oArgs.loglevel!s}") - def caseFactory(cases): """We want the tests run in order.""" if len(cases) > 1: diff --git a/wrapper_tests/tests_wrapper.py b/wrapper_tests/tests_wrapper.py index a20725d..b21872c 100644 --- a/wrapper_tests/tests_wrapper.py +++ b/wrapper_tests/tests_wrapper.py @@ -21,7 +21,7 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # -"""Originanly from https://github.com/oxij/PyTox c-toxcore-02 branch +"""Originaly from https://github.com/oxij/PyTox c-toxcore-02 branch which itself was forked from https://github.com/aitjcize/PyTox/ Modified to work with @@ -1610,6 +1610,22 @@ class ToxSuite(unittest.TestCase): else: LOG.info("passed test_tox_savedata") +def vOargsToxPreamble(oArgs, Tox, ToxTest): + + ts.vSetupLogging() + + methods = set([x for x in dir(Tox) if not x[0].isupper() + and not x[0] == '_']) + docs = "".join([getattr(ToxTest, x).__doc__ for x in dir(ToxTest) + if getattr(ToxTest, x).__doc__ is not None]) + + tested = set(re.findall(r't:(.*?)\n', docs)) + not_tested = methods.difference(tested) + + logging.info('Test Coverage: %.2f%%' % (len(tested) * 100.0 / len(methods))) + if len(not_tested): + logging.info('Not tested:\n %s' % "\n ".join(sorted(list(not_tested)))) + ### def iMain(oArgs):