emdee/tox_profile
2
3
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: emdee:b3d46abb917c66ef1027f7b5696255be57c11ea0
Branches Tags
emdee:main
..
compare: emdee:d24657a1b8550ca2b0cd2551feb53c88e1ce5aa4
Branches Tags
emdee:main

No commits in common. "b3d46abb917c66ef1027f7b5696255be57c11ea0" and "d24657a1b8550ca2b0cd2551feb53c88e1ce5aa4" have entirely different histories.
b3d46abb91 ... d24657a1b8

1 changed files with 152 additions and 213 deletions
Show Stats Expand all files Collapse all files

271
tox_profile.py
View File

@ -60,53 +60,42 @@ commands, or the filename of the nodes file for the nodes command.
# originally from: # originally from:
# https://stackoverflow.com/questions/30901873/what-format-are-tox-files-stored-in # https://stackoverflow.com/questions/30901873/what-format-are-tox-files-stored-in
import argparse
import json
import logging
import os
import shutil
import struct
import sys import sys
import os
import time import time
import warnings import struct
from socket import inet_ntop, AF_INET6, AF_INET
import logging
import argparse
from pprint import pprint from pprint import pprint
from socket import AF_INET, AF_INET6, inet_ntop import shutil
import json
import warnings
warnings.filterwarnings('ignore') warnings.filterwarnings('ignore')
from wrapper_tests import support_testing as ts
try: try:
# https://pypi.org/project/msgpack/ # https://pypi.org/project/msgpack/
import msgpack import msgpack
except ImportError as e: # noqa except ImportError as e:
msgpack = None msgpack = None
try: try:
import yaml import yaml
except ImportError as e: # noqa except ImportError as e:
yaml = None yaml = None
try:
import stem
except ImportError as e: # noqa
stem = None
try:
import nmap
except ImportError as e: # noqa
nmap = None
try: try:
# https://pypi.org/project/coloredlogs/ # https://pypi.org/project/coloredlogs/
import coloredlogs import coloredlogs
if 'COLOREDLOGS_LEVEL_STYLES' not in os.environ: if 'COLOREDLOGS_LEVEL_STYLES' not in os.environ:
os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red' os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red'
except ImportError as e: # noqa except ImportError as e:
coloredlogs = False coloredlogs = False
try: try:
# https://git.plastiras.org/emdee/toxygen_wrapper # https://git.plastiras.org/emdee/toxygen_wrapper
from wrapper.toxencryptsave import ToxEncryptSave from wrapper.toxencryptsave import ToxEncryptSave
from wrapper_tests import support_testing as ts from wrapper_tests.support_http import download_url, bAreWeConnected
from wrapper_tests.support_http import bAreWeConnected, download_url
from wrapper_tests.support_testing import sTorResolve from wrapper_tests.support_testing import sTorResolve
from wrapper_tests import support_testing as ts
except ImportError as e: except ImportError as e:
print(f"Import Warning {e}") print(f"Import Warning {e}")
print("Download toxygen_wrapper to deal with encrypted tox files, from:") print("Download toxygen_wrapper to deal with encrypted tox files, from:")
@ -123,18 +112,6 @@ except ImportError as e:
LOG = logging.getLogger('TSF') LOG = logging.getLogger('TSF')
def LOG_error(a): print('EROR> '+a)
def LOG_warn(a): print('WARN> '+a)
def LOG_info(a):
bVERBOSE = hasattr(__builtins__, 'oArgs') and oArgs.log_level <= 20
if bVERBOSE: print('INFO> '+a)
def LOG_debug(a):
bVERBOSE = hasattr(__builtins__, 'oArgs') and oArgs.log_level <= 10-1
if bVERBOSE: print('DBUG> '+a)
def LOG_trace(a):
bVERBOSE = hasattr(__builtins__, 'oArgs') and oArgs.log_level < 10
if bVERBOSE: print('TRAC> '+a)
# Fix for Windows # Fix for Windows
sDIR = os.environ.get('TMPDIR', '/tmp') sDIR = os.environ.get('TMPDIR', '/tmp')
sTOX_VERSION = "1000002018" sTOX_VERSION = "1000002018"
@ -154,7 +131,6 @@ LOG.trace = trace
global bOUT, aOUT, sENC global bOUT, aOUT, sENC
aOUT = {} aOUT = {}
bOUT = b'' bOUT = b''
lLABELS = []
sENC = sys.getdefaultencoding() # 'utf-8' sENC = sys.getdefaultencoding() # 'utf-8'
lNULLS = ['', '[]', 'null'] lNULLS = ['', '[]', 'null']
lNONES = ['', '-', 'NONE'] lNONES = ['', '-', 'NONE']
@ -283,7 +259,7 @@ Length Contents
4: 'Friend online' 4: 'Friend online'
} }
slen = 1+32+1024+1+2+128+2+1007+1+2+1+3+4+8 # 2216 slen = 1+32+1024+1+2+128+2+1007+1+2+1+3+4+8 # 2216
assert length % slen == 0, length assert length % slen == 0
lIN = [] lIN = []
for i in range(length // slen): for i in range(length // slen):
delta = i*slen delta = i*slen
@ -540,7 +516,7 @@ def lProcessDHTnodes(state, index, length, result, label="DHTnode"):
return lIN return lIN
def process_chunk(index, state, oArgs=None): def process_chunk(index, state, oArgs=None):
global bOUT, aOUT, lLABELS global bOUT, aOUT
global sENC global sENC
length = struct.unpack_from("<I", state, index)[0] length = struct.unpack_from("<I", state, index)[0]
@ -561,7 +537,6 @@ def process_chunk(index, state, oArgs=None):
LOG.trace(f"PROCESS_CHUNK {label} index={index} bOUT={len(bOUT)} delta={diff} length={length}") LOG.trace(f"PROCESS_CHUNK {label} index={index} bOUT={len(bOUT)} delta={diff} length={length}")
if data_type == MESSENGER_STATE_TYPE_NOSPAMKEYS: if data_type == MESSENGER_STATE_TYPE_NOSPAMKEYS:
lLABELS += [label]
nospam = bin_to_hex(result[0:4]) nospam = bin_to_hex(result[0:4])
public_key = bin_to_hex(result[4:36]) public_key = bin_to_hex(result[4:36])
private_key = bin_to_hex(result[36:68]) private_key = bin_to_hex(result[36:68])
@ -590,7 +565,6 @@ def process_chunk(index, state, oArgs=None):
LOG.info(f"{label} {key} EDITED to {val}") LOG.info(f"{label} {key} EDITED to {val}")
elif data_type == MESSENGER_STATE_TYPE_DHT: elif data_type == MESSENGER_STATE_TYPE_DHT:
lLABELS += [label]
LOG.debug(f"process_chunk {label} length={length}") LOG.debug(f"process_chunk {label} length={length}")
if length > 4: if length > 4:
lIN = lProcessDHTnodes(state, index, length, result, "DHTnode") lIN = lProcessDHTnodes(state, index, length, result, "DHTnode")
@ -609,8 +583,7 @@ def process_chunk(index, state, oArgs=None):
LOG.info(f"{label} {key} EDITED to {val}") LOG.info(f"{label} {key} EDITED to {val}")
elif data_type == MESSENGER_STATE_TYPE_FRIENDS: elif data_type == MESSENGER_STATE_TYPE_FRIENDS:
lLABELS += [label] LOG.info(f"{label} {length // 2216} FRIENDS {length % 2216}")
LOG.info(f"{label} {length // 2216} friends mod={length % 2216}")
if length > 0: if length > 0:
lIN = lProcessFriends(state, index, length, result) lIN = lProcessFriends(state, index, length, result)
else: else:
@ -619,7 +592,6 @@ def process_chunk(index, state, oArgs=None):
aOUT.update({label: lIN}) aOUT.update({label: lIN})
elif data_type == MESSENGER_STATE_TYPE_NAME: elif data_type == MESSENGER_STATE_TYPE_NAME:
lLABELS += [label]
name = str(result, sENC) name = str(result, sENC)
LOG.info(f"{label} Nick_name = " +name) LOG.info(f"{label} Nick_name = " +name)
aIN = {"Nick_name": name} aIN = {"Nick_name": name}
@ -632,7 +604,6 @@ def process_chunk(index, state, oArgs=None):
LOG.info(f"{label} {key} EDITED to {val}") LOG.info(f"{label} {key} EDITED to {val}")
elif data_type == MESSENGER_STATE_TYPE_STATUSMESSAGE: elif data_type == MESSENGER_STATE_TYPE_STATUSMESSAGE:
lLABELS += [label]
mess = str(result, sENC) mess = str(result, sENC)
LOG.info(f"{label} StatusMessage = " +mess) LOG.info(f"{label} StatusMessage = " +mess)
aIN = {"Status_message": mess} aIN = {"Status_message": mess}
@ -645,7 +616,6 @@ def process_chunk(index, state, oArgs=None):
LOG.info(f"{label} {key} EDITED to {val}") LOG.info(f"{label} {key} EDITED to {val}")
elif data_type == MESSENGER_STATE_TYPE_STATUS: elif data_type == MESSENGER_STATE_TYPE_STATUS:
lLABELS += [label]
# 1 uint8_t status (0 = online, 1 = away, 2 = busy) # 1 uint8_t status (0 = online, 1 = away, 2 = busy)
dStatus = {0: 'online', 1: 'away', 2: 'busy'} dStatus = {0: 'online', 1: 'away', 2: 'busy'}
status = struct.unpack_from(">b", state, index)[0] status = struct.unpack_from(">b", state, index)[0]
@ -661,7 +631,6 @@ def process_chunk(index, state, oArgs=None):
LOG.info(f"{label} {key} EDITED to {val}") LOG.info(f"{label} {key} EDITED to {val}")
elif data_type == MESSENGER_STATE_TYPE_GROUPS: elif data_type == MESSENGER_STATE_TYPE_GROUPS:
lLABELS += [label]
if length > 0: if length > 0:
lIN = lProcessGroups(state, index, length, result, label) lIN = lProcessGroups(state, index, length, result, label)
else: else:
@ -670,7 +639,6 @@ def process_chunk(index, state, oArgs=None):
aOUT.update({label: lIN}) aOUT.update({label: lIN})
elif data_type == MESSENGER_STATE_TYPE_TCP_RELAY: elif data_type == MESSENGER_STATE_TYPE_TCP_RELAY:
lLABELS += [label]
if length > 0: if length > 0:
lIN = lProcessNodeInfo(state, index, length, result, "TCPnode") lIN = lProcessNodeInfo(state, index, length, result, "TCPnode")
LOG.info(f"TYPE_TCP_RELAY {len(lIN)} nodes {length} length") LOG.info(f"TYPE_TCP_RELAY {len(lIN)} nodes {length} length")
@ -686,7 +654,6 @@ def process_chunk(index, state, oArgs=None):
LOG.info(f"{label} {key} EDITED to {val}") LOG.info(f"{label} {key} EDITED to {val}")
elif data_type == MESSENGER_STATE_TYPE_PATH_NODE: elif data_type == MESSENGER_STATE_TYPE_PATH_NODE:
lLABELS += [label]
#define NUM_SAVED_PATH_NODES 8 #define NUM_SAVED_PATH_NODES 8
if not length % 8 == 0: if not length % 8 == 0:
# this should be an assert? # this should be an assert?
@ -703,7 +670,6 @@ def process_chunk(index, state, oArgs=None):
LOG.info(f"{label} {key} EDITED to {val}") LOG.info(f"{label} {key} EDITED to {val}")
elif data_type == MESSENGER_STATE_TYPE_CONFERENCES: elif data_type == MESSENGER_STATE_TYPE_CONFERENCES:
lLABELS += [label]
lIN = [] lIN = []
if length > 0: if length > 0:
LOG.debug(f"TODO process_chunk {label} bytes={length}") LOG.debug(f"TODO process_chunk {label} bytes={length}")
@ -714,13 +680,10 @@ def process_chunk(index, state, oArgs=None):
elif data_type != MESSENGER_STATE_TYPE_END: elif data_type != MESSENGER_STATE_TYPE_END:
LOG.error("UNRECOGNIZED datatype={datatype}") LOG.error("UNRECOGNIZED datatype={datatype}")
sys.exit(1) sys.exit(1)
else: else:
LOG.info("END") # That's all folks... LOG.info("END") # That's all folks...
# drop through # drop through
if len(lLABELS) == len(dSTATE_TYPE.keys()) - 1:
LOG.info(f"{len(lLABELS)} sections") # That's all folks...
else:
LOG.warn(f"{10 - len(lLABELS)} sections missing {lLABELS}") # That's all folks...
# We repack as we read: or edit as we parse; simply edit result and length. # We repack as we read: or edit as we parse; simply edit result and length.
# We'll add the results back to bOUT to see if we get what we started with. # We'll add the results back to bOUT to see if we get what we started with.
@ -823,14 +786,14 @@ def lNodesCheckNodes(json_nodes, oArgs, bClean=False):
Checking NODES.json Checking NODES.json
""" """
lErrs = [] lErrs = []
ierrs = 0 iErrs = 0
nth = 0 nth = 0
if bClean: lNew=[] if bClean: lNew=[]
# assert type(json_nodes) == dict # assert type(json_nodes) == dict
bRUNNING_TOR = False bRUNNING_TOR = False
if bHAVE_TOR: if bHAVE_TOR:
iret = os.system("netstat -nle4|grep -q :9050") iRet = os.system("netstat -nle4|grep -q :9050")
if iret == 0: if iRet == 0:
bRUNNING_TOR = True bRUNNING_TOR = True
lOnions = [] lOnions = []
@ -868,8 +831,8 @@ def lNodesCheckNodes(json_nodes, oArgs, bClean=False):
elif True: elif True:
if not node[ipv] in lNONES and ipv == 'ipv4': if not node[ipv] in lNONES and ipv == 'ipv4':
# just ping for now # just ping for now
iret = os.system(f"ping -c 1 {node[ipv]} > /dev/null") iRet = os.system(f"ping -c 1 {node[ipv]} > /dev/null")
if iret == 0: if iRet == 0:
LOG.info(f"Pinged {node[ipv]}") LOG.info(f"Pinged {node[ipv]}")
else: else:
LOG.warn(f"Failed ping {node[ipv]}") LOG.warn(f"Failed ping {node[ipv]}")
@ -895,10 +858,10 @@ def lNodesCheckNodes(json_nodes, oArgs, bClean=False):
# Put the onion address in the location after the country code # Put the onion address in the location after the country code
if len(node["location"]) not in [2, 65]: if len(node["location"]) not in [2, 65]:
LOG.warn(f"location {node['location']} should be a 2 digit country code, or 'code onion'") LOG.warn(f"location {location} should be a 2 digit country code, or 'code onion'")
elif len(node["location"]) == 65 and \ elif len(node["location"]) == 65 and \
not node["location"].endswith('.onion') : not node["location"].endswith('.onion') :
LOG.warn(f"location {node['location']} should be a 2 digit country code 'code onion'") LOG.warn(f"location {location} should be a 2 digit country code 'code onion'")
elif len(node["location"]) == 65 and \ elif len(node["location"]) == 65 and \
node["location"].endswith('.onion') and bHAVE_TOR: node["location"].endswith('.onion') and bHAVE_TOR:
onion = node["location"][3:] onion = node["location"][3:]
@ -932,7 +895,7 @@ def lNodesCheckNodes(json_nodes, oArgs, bClean=False):
if not node['motd']: if not node['motd']:
# LOG.info(f"Maybe put a ToxID: in motd so people can contact you.") # LOG.info(f"Maybe put a ToxID: in motd so people can contact you.")
pass pass
if bClean and nth not in lErrs: if bClean and not nth in lErrs:
lNew+=[new_node] lNew+=[new_node]
nth += 1 nth += 1
@ -962,7 +925,7 @@ def iNodesFileCheck(sProOrNodes, oArgs, bClean=False):
with open(sProOrNodes, 'rt') as fl: with open(sProOrNodes, 'rt') as fl:
json_all = json.loads(fl.read()) json_all = json.loads(fl.read())
json_nodes = json_all['nodes'] json_nodes = json_all['nodes']
except Exception as e: # noqa except Exception as e:
LOG.exception(f"{oArgs.command} error reading {sProOrNodes}") LOG.exception(f"{oArgs.command} error reading {sProOrNodes}")
return 1 return 1
@ -977,17 +940,17 @@ def iNodesFileCheck(sProOrNodes, oArgs, bClean=False):
aOut = dict(last_scan=json_all['last_scan'], aOut = dict(last_scan=json_all['last_scan'],
last_refresh=now, last_refresh=now,
nodes=al) nodes=al)
sout = oArgs.output sOut = oArgs.output
try: try:
LOG.debug(f"iNodesFileClean saving to {sout}") LOG.debug(f"iNodesFileClean saving to {sOut}")
oStream = open(sout, 'wt', encoding=sENC) oStream = open(sOut, 'wt', encoding=sENC)
json.dump(aOut, oStream, indent=oArgs.indent) json.dump(aOut, oStream, indent=oArgs.indent)
if oStream.write('\n') > 0: i = 0 if oStream.write('\n') > 0: i = 0
except Exception as e: # noqa except Exception as e:
LOG.exception(f"iNodesFileClean error dumping JSON to {sout}") LOG.exception(f"iNodesFileClean error dumping JSON to {sOUT}")
return 3 return 3
except Exception as e: # noqa except Exception as e:
LOG.exception(f"iNodesFileCheck error checking JSON") LOG.exception(f"iNodesFileCheck error checking JSON")
i = -2 i = -2
else: else:
@ -1002,67 +965,67 @@ def iNodesFileClean(sProOrNodes):
return 0 return 0
f = "DHTNodes.clean" f = "DHTNodes.clean"
if not oArgs.output: if not oArgs.output:
sout = os.path.join(sDIR, f) sOut = os.path.join(sDIR, f)
else: else:
sout = oArgs.output sOut = oArgs.output
try: try:
LOG.debug(f"iNodesFileClean saving to {sout}") LOG.debug(f"iNodesFileClean saving to {sOut}")
oStream = open(sout, 'wt', encoding=sENC) oStream = open(sOut, 'wt', encoding=sENC)
json.dump(aOUT, oStream, indent=oArgs.indent) json.dump(aOUT, oStream, indent=oArgs.indent)
if oStream.write('\n') > 0: iret = 0 if oStream.write('\n') > 0: iRet = 0
except Exception as e: # noqa except Exception as e:
LOG.exception(f"iNodesFileClean error dumping JSON to {sout}") LOG.exception(f"iNodesFileClean error dumping JSON to {sOUT}")
return 3 return 3
LOG.info(f"{oArgs.info}ing iret={iret} to {oArgs.output}") LOG.info(f"{oArgs.info}ing iRet={iRet} to {oArgs.output}")
return 0 return 0
def iOsSystemNmapUdp(l, oArgs): def iOsSystemNmapUdp(l, oArgs):
ierrs = 0 iErrs = 0
for elt in l: for elt in l:
cmd = f"sudo nmap -Pn -n -sU -p U:{elt['Port']} {elt['Ip']}" cmd = f"sudo nmap -Pn -n -sU -p U:{elt['Port']} {elt['Ip']}"
LOG.debug(f"{oArgs.info} {cmd} to {oArgs.output}") LOG.debug(f"{oArgs.info} {cmd} to {oArgs.output}")
ierrs += os.system(cmd +f" >> {oArgs.output} 2>&1") iErrs += os.system(cmd +f" >> {oArgs.output} 2>&1")
if ierrs: if iErrs:
LOG.warn(f"{oArgs.info} {ierrs} ERRORs to {oArgs.output}") LOG.warn(f"{oArgs.info} {iErrs} ERRORs to {oArgs.output}")
else: else:
LOG.info(f"{oArgs.info} NO errors to {oArgs.output}") LOG.info(f"{oArgs.info} NO errors to {oArgs.output}")
lRet = lParseNapOutput(oArgs.output) lRet = lParseNapOutput(oArgs.output)
if lRet: if lRet:
for sLine in lRet: for sLine in lRet:
LOG.warn(f"{oArgs.nodes} {sLine}") LOG.warn(f"{oArgs.nodes} {sLine}")
ierr = len(lRet) iErr = len(lRet)
ierrs += ierr iErrs += iErr
return ierrs return iErrs
def iOsSystemNmapTcp(l, oArgs): def iOsSystemNmapTcp(l, oArgs):
ierrs = 0 iErrs = 0
LOG.debug(f"{len(l)} nodes to {oArgs.output}") LOG.debug(f"{len(l)} nodes to {oArgs.output}")
for elt in l: for elt in l:
cmd = f"sudo nmap -Pn -n -sT -p T:{elt['Port']} {elt['Ip']}" cmd = f"sudo nmap -Pn -n -sT -p T:{elt['Port']} {elt['Ip']}"
LOG.debug(f"iOsSystemNmapTcp {cmd} to {oArgs.output}") LOG.debug(f"iOsSystemNmapTcp {cmd} to {oArgs.output}")
ierr = os.system(cmd +f" >> {oArgs.output} 2>&1") iErr += os.system(cmd +f" >> {oArgs.output} 2>&1")
if ierr: if iErr:
LOG.warn(f"iOsSystemNmapTcp {ierrs} ERRORs to {oArgs.output}") LOG.warn(f"iOsSystemNmapTcp {iErrs} ERRORs to {oArgs.output}")
else: else:
lRet = lParseNapOutput(oArgs.output) lRet = lParseNapOutput(oArgs.output)
if lRet: if lRet:
for sLine in lRet: for sLine in lRet:
LOG.warn(f"{oArgs.nodes} {sLine}") LOG.warn(f"{oArgs.nodes} {sLine}")
ierr = len(lRet) iErr = len(lRet)
ierrs += ierr iErrs += iErr
return ierrs return iErrs
def vSetupLogging(log_level=logging.DEBUG): def vSetupLogging(loglevel=logging.DEBUG):
global LOG global LOG
if coloredlogs: if coloredlogs:
aKw = dict(level=log_level, aKw = dict(level=loglevel,
logger=LOG, logger=LOG,
fmt='%(name)s %(levelname)s %(message)s') fmt='%(name)s %(levelname)s %(message)s')
coloredlogs.install(**aKw) coloredlogs.install(**aKw)
else: else:
aKw = dict(level=log_level, aKw = dict(level=loglevel,
format='%(name)s %(levelname)-4s %(message)s') format='%(name)s %(levelname)-4s %(message)s')
logging.basicConfig(**aKw) logging.basicConfig(**aKw)
@ -1079,7 +1042,7 @@ def iTestTorConfig(sProOrNodes, oArgs, bClean=False):
for key,val in lONION_CONFIG.items(): for key,val in lONION_CONFIG.items():
for line in val: for line in val:
if line.startswith('#'): continue if line.startswith('#'): continue
if line not in lEtcTorrc: if not line in lEtcTorrc:
print(line) print(line)
# add_mapaddress # add_mapaddress
if bClean == False: if bClean == False:
@ -1094,23 +1057,6 @@ def iTestTorConfig(sProOrNodes, oArgs, bClean=False):
# add_bootstrap # add_bootstrap
return 0 return 0
def iTestTorExits(sProOrNodes, oArgs, bClean=False):
LOG.info(f"iTestTorExits")
# import pdb; pdb.set_trace()
# sProOrNodes
try:
if hasattr(ts, 'oSTEM_CONTROLER') and ts.oSTEM_CONTROLER \
and ts.oSTEM_CONTROLER.is_set('ExcludeExitNodes'):
LOG_info(f"ExcludeExitNodes is set so we cant continue")
return 0
LOG_info(f"ExcludeExitNodes is not set so we can continue")
l = ts.lExitExcluder(iPort=9051)
except Exception as e:
LOG.error(f"ExcludeExitNodes errored {e}")
return 1
return 0
def iTestTorTest(sProOrNodes, oArgs, bClean=False): def iTestTorTest(sProOrNodes, oArgs, bClean=False):
# test_onion # test_onion
# check_mapaddress # check_mapaddress
@ -1118,7 +1064,7 @@ def iTestTorTest(sProOrNodes, oArgs, bClean=False):
LOG.info(f"iTestTorTest {sProOrNodes}") LOG.info(f"iTestTorTest {sProOrNodes}")
for elt in lONION_NODES: for elt in lONION_NODES:
for line in elt['onions']: for line in elt['onions']:
(host, port,) = line.split(':') host,port = line.split(':')
LOG.debug(f"iTestTorTest resolving {host}") LOG.debug(f"iTestTorTest resolving {host}")
ip = ts.sTorResolve(host) ip = ts.sTorResolve(host)
if ip: LOG.info(f"{host} resolved to {ip}") if ip: LOG.info(f"{host} resolved to {ip}")
@ -1153,12 +1099,12 @@ def iMain(sProOrNodes, oArgs):
if oArgs.command == 'decrypt': if oArgs.command == 'decrypt':
assert oArgs.output, "--output required for this command" assert oArgs.output, "--output required for this command"
oStream = open(oArgs.output, 'wb') oStream = open(oArgs.output, 'wb')
iret = oStream.write(bSAVE) iRet = oStream.write(bSAVE)
LOG.info(f"Wrote {iret} to {oArgs.output}") LOG.info(f"Wrote {iRet} to {oArgs.output}")
iret = 0 iRet = 0
elif oArgs.command == 'nodes': elif oArgs.command == 'nodes':
iret = -1 iRet = -1
ep_sec = str(int(time.time())) ep_sec = str(int(time.time()))
json_head = '{"last_scan":' +ep_sec \ json_head = '{"last_scan":' +ep_sec \
+',"last_refresh":' +ep_sec \ +',"last_refresh":' +ep_sec \
@ -1169,7 +1115,7 @@ def iMain(sProOrNodes, oArgs):
with open(oArgs.output, 'wt') as oFd: with open(oArgs.output, 'wt') as oFd:
oFd.write(json_head) oFd.write(json_head)
cmd = f"cat '{sProOrNodes}' | jq '.|with_entries(select(.key|match(\"nodes\"))).nodes[]|select(.status_tcp)|select(.ipv4|match(\".\"))' " cmd = f"cat '{sProOrNodes}' | jq '.|with_entries(select(.key|match(\"nodes\"))).nodes[]|select(.status_tcp)|select(.ipv4|match(\".\"))' "
iret = os.system(cmd +"| sed -e '2,$s/^{/,{/'" +f" >>{oArgs.output}") iRet = os.system(cmd +"| sed -e '2,$s/^{/,{/'" +f" >>{oArgs.output}")
with open(oArgs.output, 'at') as oFd: oFd.write(']}\n') with open(oArgs.output, 'at') as oFd: oFd.write(']}\n')
elif oArgs.nodes == 'select_udp': elif oArgs.nodes == 'select_udp':
@ -1178,7 +1124,7 @@ def iMain(sProOrNodes, oArgs):
with open(oArgs.output, 'wt') as oFd: with open(oArgs.output, 'wt') as oFd:
oFd.write(json_head) oFd.write(json_head)
cmd = f"cat '{sProOrNodes}' | jq '.|with_entries(select(.key|match(\"nodes\"))).nodes[]|select(.status_udp)|select(.ipv4|match(\".\"))' " cmd = f"cat '{sProOrNodes}' | jq '.|with_entries(select(.key|match(\"nodes\"))).nodes[]|select(.status_udp)|select(.ipv4|match(\".\"))' "
iret = os.system(cmd +"| sed -e '2,$s/^{/,{/'" +f" >>{oArgs.output}") iRet = os.system(cmd +"| sed -e '2,$s/^{/,{/'" +f" >>{oArgs.output}")
with open(oArgs.output, 'at') as oFd: oFd.write(']}\n') with open(oArgs.output, 'at') as oFd: oFd.write(']}\n')
elif oArgs.nodes == 'select_version': elif oArgs.nodes == 'select_version':
@ -1188,7 +1134,7 @@ def iMain(sProOrNodes, oArgs):
oFd.write(json_head) oFd.write(json_head)
cmd = f"cat '{sProOrNodes}' | jq '.|with_entries(select(.key|match(\"nodes\"))).nodes[]|select(.status_udp)|select(.version|match(\"{sTOX_VERSION}\"))'" cmd = f"cat '{sProOrNodes}' | jq '.|with_entries(select(.key|match(\"nodes\"))).nodes[]|select(.status_udp)|select(.version|match(\"{sTOX_VERSION}\"))'"
iret = os.system(cmd +"| sed -e '2,$s/^{/,{/'" +f" >>{oArgs.output}") iRet = os.system(cmd +"| sed -e '2,$s/^{/,{/'" +f" >>{oArgs.output}")
with open(oArgs.output, 'at') as oFd: with open(oArgs.output, 'at') as oFd:
oFd.write(']}\n') oFd.write(']}\n')
@ -1200,13 +1146,13 @@ def iMain(sProOrNodes, oArgs):
cmd = sBashFileNmapTcp() cmd = sBashFileNmapTcp()
cmd = f"sudo bash {cmd} < '{sProOrNodes}' >'{oArgs.output}' 2>&1" cmd = f"sudo bash {cmd} < '{sProOrNodes}' >'{oArgs.output}' 2>&1"
LOG.debug(cmd) LOG.debug(cmd)
iret = os.system(cmd) iRet = os.system(cmd)
if iret == 0: if iRet == 0:
lRet = lParseNapOutput(oArgs.output) lRet = lParseNapOutput(oArgs.output)
if lRet: if lRet:
for sLine in lRet: for sLine in lRet:
LOG.warn(f"{oArgs.nodes} {sLine}") LOG.warn(f"{oArgs.nodes} {sLine}")
iret = len(lRet) iRet = len(lRet)
elif oArgs.nodes == 'nmap_udp': elif oArgs.nodes == 'nmap_udp':
assert oArgs.output, "--output required for this command" assert oArgs.output, "--output required for this command"
@ -1217,13 +1163,13 @@ def iMain(sProOrNodes, oArgs):
cmd = vBashFileNmapUdp() cmd = vBashFileNmapUdp()
cmd = f"sudo bash {cmd} < '{sProOrNodes}'" +f" >'{oArgs.output}' 2>&1" cmd = f"sudo bash {cmd} < '{sProOrNodes}'" +f" >'{oArgs.output}' 2>&1"
LOG.debug(cmd) LOG.debug(cmd)
iret = os.system(cmd) iRet = os.system(cmd)
if iret == 0: if iRet == 0:
lRet = lParseNapOutput(oArgs.output) lRet = lParseNapOutput(oArgs.output)
if lRet: if lRet:
for sLine in lRet: for sLine in lRet:
LOG.warn(f"{oArgs.nodes} {sLine}") LOG.warn(f"{oArgs.nodes} {sLine}")
iret = len(lRet) iRet = len(lRet)
elif oArgs.nodes == 'download' and download_url: elif oArgs.nodes == 'download' and download_url:
if not bAreWeConnected(): if not bAreWeConnected():
@ -1232,7 +1178,7 @@ def iMain(sProOrNodes, oArgs):
b = download_url(url) b = download_url(url)
if not b: if not b:
LOG.warn("failed downloading list of nodes") LOG.warn("failed downloading list of nodes")
iret = -1 iRet = -1
else: else:
if oArgs.output: if oArgs.output:
oStream = open(oArgs.output, 'wb') oStream = open(oArgs.output, 'wb')
@ -1240,23 +1186,23 @@ def iMain(sProOrNodes, oArgs):
else: else:
oStream = sys.stdout oStream = sys.stdout
oStream.write(str(b, sENC)) oStream.write(str(b, sENC))
iret = 0 iRet = 0
LOG.info(f"downloaded list of nodes to {oStream}") LOG.info(f"downloaded list of nodes to {oStream}")
elif oArgs.nodes == 'check': elif oArgs.nodes == 'check':
i = iNodesFileCheck(sProOrNodes, oArgs, bClean=False) i = iNodesFileCheck(sProOrNodes, oArgs, bClean=False)
iret = i iRet = i
elif oArgs.nodes == 'clean': elif oArgs.nodes == 'clean':
assert oArgs.output, "--output required for this command" assert oArgs.output, "--output required for this command"
i = iNodesFileCheck(sProOrNodes, oArgs, bClean=True) i = iNodesFileCheck(sProOrNodes, oArgs, bClean=True)
iret = i iRet = i
if iret > 0: if iRet > 0:
LOG.warn(f"{oArgs.nodes} iret={iret} to {oArgs.output}") LOG.warn(f"{oArgs.nodes} iRet={iRet} to {oArgs.output}")
elif iret == 0: elif iRet == 0:
LOG.info(f"{oArgs.nodes} iret={iret} to {oArgs.output}") LOG.info(f"{oArgs.nodes} iRet={iRet} to {oArgs.output}")
elif oArgs.command == 'onions': elif oArgs.command == 'onions':
@ -1264,17 +1210,11 @@ def iMain(sProOrNodes, oArgs):
if oArgs.onions == 'config': if oArgs.onions == 'config':
i = iTestTorConfig(sProOrNodes, oArgs) i = iTestTorConfig(sProOrNodes, oArgs)
iret = i iRet = i
elif oArgs.onions == 'test': elif oArgs.onions == 'test':
i = iTestTorTest(sProOrNodes, oArgs) i = iTestTorTest(sProOrNodes, oArgs)
iret = i iRet = i
elif oArgs.onions == 'exits':
i = iTestTorExits(sProOrNodes, oArgs)
iret = i
else:
RuntimeError(oArgs.onions)
elif oArgs.command in ['info', 'edit']: elif oArgs.command in ['info', 'edit']:
@ -1291,11 +1231,11 @@ def iMain(sProOrNodes, oArgs):
assert bSAVE[:8] == bMARK, "Not a Tox profile" assert bSAVE[:8] == bMARK, "Not a Tox profile"
bOUT = bMARK bOUT = bMARK
iret = 0 iErrs = 0
process_chunk(len(bOUT), bSAVE, oArgs) process_chunk(len(bOUT), bSAVE, oArgs)
if not bOUT: if not bOUT:
LOG.error(f"{oArgs.command} NO bOUT results") LOG.error(f"{oArgs.command} NO bOUT results")
iret = 1 iRet = 1
else: else:
oStream = None oStream = None
LOG.debug(f"command={oArgs.command} len bOUT={len(bOUT)} results") LOG.debug(f"command={oArgs.command} len bOUT={len(bOUT)} results")
@ -1303,15 +1243,15 @@ def iMain(sProOrNodes, oArgs):
if oArgs.command in ['edit'] or oArgs.info in ['save']: if oArgs.command in ['edit'] or oArgs.info in ['save']:
LOG.debug(f"{oArgs.command} saving to {oArgs.output}") LOG.debug(f"{oArgs.command} saving to {oArgs.output}")
oStream = open(oArgs.output, 'wb', encoding=None) oStream = open(oArgs.output, 'wb', encoding=None)
if oStream.write(bOUT) > 0: iret = 0 if oStream.write(bOUT) > 0: iRet = 0
LOG.info(f"{oArgs.info}ed iret={iret} to {oArgs.output}") LOG.info(f"{oArgs.info}ed iRet={iRet} to {oArgs.output}")
elif oArgs.info == 'info': elif oArgs.info == 'info':
pass pass
iret = 0 iRet = 0
elif oArgs.info == 'yaml': elif oArgs.info == 'yaml':
if not yaml: if not yaml:
LOG.warn(f"{oArgs.command} no yaml support") LOG.warn(f"{oArgs.command} no yaml support")
iret = -1 iRet = -1
else: else:
LOG.debug(f"{oArgs.command} saving to {oArgs.output}") LOG.debug(f"{oArgs.command} saving to {oArgs.output}")
oStream = open(oArgs.output, 'wt', encoding=sENC) oStream = open(oArgs.output, 'wt', encoding=sENC)
@ -1322,13 +1262,13 @@ def iMain(sProOrNodes, oArgs):
LOG.warn(f'WARN: {e}') LOG.warn(f'WARN: {e}')
else: else:
oStream.write('\n') oStream.write('\n')
iret = 0 iRet = 0
LOG.info(f"{oArgs.info}ing iret={iret} to {oArgs.output}") LOG.info(f"{oArgs.info}ing iRet={iRet} to {oArgs.output}")
elif oArgs.info == 'json': elif oArgs.info == 'json':
if not yaml: if not yaml:
LOG.warn(f"{oArgs.command} no json support") LOG.warn(f"{oArgs.command} no json support")
iret = -1 iRet = -1
else: else:
LOG.debug(f"{oArgs.command} saving to {oArgs.output}") LOG.debug(f"{oArgs.command} saving to {oArgs.output}")
oStream = open(oArgs.output, 'wt', encoding=sENC) oStream = open(oArgs.output, 'wt', encoding=sENC)
@ -1337,56 +1277,56 @@ def iMain(sProOrNodes, oArgs):
except: except:
LOG.warn("There are somtimes problems with the json info dump of bytes keys: ```TypeError: Object of type bytes is not JSON serializable```") LOG.warn("There are somtimes problems with the json info dump of bytes keys: ```TypeError: Object of type bytes is not JSON serializable```")
oStream.write('\n') > 0 oStream.write('\n') > 0
iret = 0 iRet = 0
LOG.info(f"{oArgs.info}ing iret={iret} to {oArgs.output}") LOG.info(f"{oArgs.info}ing iRet={iRet} to {oArgs.output}")
elif oArgs.info == 'repr': elif oArgs.info == 'repr':
LOG.debug(f"{oArgs.command} saving to {oArgs.output}") LOG.debug(f"{oArgs.command} saving to {oArgs.output}")
oStream = open(oArgs.output, 'wt', encoding=sENC) oStream = open(oArgs.output, 'wt', encoding=sENC)
if oStream.write(repr(bOUT)) > 0: iret = 0 if oStream.write(repr(bOUT)) > 0: iRet = 0
if oStream.write('\n') > 0: iret = 0 if oStream.write('\n') > 0: iRet = 0
LOG.info(f"{oArgs.info}ing iret={iret} to {oArgs.output}") LOG.info(f"{oArgs.info}ing iRet={iRet} to {oArgs.output}")
elif oArgs.info == 'pprint': elif oArgs.info == 'pprint':
LOG.debug(f"{oArgs.command} saving to {oArgs.output}") LOG.debug(f"{oArgs.command} saving to {oArgs.output}")
oStream = open(oArgs.output, 'wt', encoding=sENC) oStream = open(oArgs.output, 'wt', encoding=sENC)
pprint(aOUT, stream=oStream, indent=oArgs.indent, width=80) pprint(aOUT, stream=oStream, indent=oArgs.indent, width=80)
iret = 0 iRet = 0
LOG.info(f"{oArgs.info}ing iret={iret} to {oArgs.output}") LOG.info(f"{oArgs.info}ing iRet={iRet} to {oArgs.output}")
elif oArgs.info == 'nmap_relay': elif oArgs.info == 'nmap_relay':
assert bHAVE_NMAP, "nmap is required for this command" assert bHAVE_NMAP, "nmap is required for this command"
assert oArgs.output, "--output required for this command" assert oArgs.output, "--output required for this command"
if aOUT["TCP_RELAY"]: if aOUT["TCP_RELAY"]:
iret = iOsSystemNmapTcp(aOUT["TCP_RELAY"], oArgs) iRet = iOsSystemNmapTcp(aOUT["TCP_RELAY"], oArgs)
else: else:
LOG.warn(f"{oArgs.info} no TCP_RELAY") LOG.warn(f"{oArgs.info} no TCP_RELAY")
iret = 0 iRet = 0
elif oArgs.info == 'nmap_dht': elif oArgs.info == 'nmap_dht':
assert bHAVE_NMAP, "nmap is required for this command" assert bHAVE_NMAP, "nmap is required for this command"
assert oArgs.output, "--output required for this command" assert oArgs.output, "--output required for this command"
if aOUT["DHT"]: if aOUT["DHT"]:
iret = iOsSystemNmapUdp(aOUT["DHT"], oArgs) iRet = iOsSystemNmapUdp(aOUT["DHT"], oArgs)
else: else:
LOG.warn(f"{oArgs.info} no DHT") LOG.warn(f"{oArgs.info} no DHT")
iret = 0 iRet = 0
elif oArgs.info == 'nmap_path': elif oArgs.info == 'nmap_path':
assert bHAVE_NMAP, "nmap is required for this command" assert bHAVE_NMAP, "nmap is required for this command"
assert oArgs.output, "--output required for this command" assert oArgs.output, "--output required for this command"
if aOUT["PATH_NODE"]: if aOUT["PATH_NODE"]:
iret = iOsSystemNmapUdp(aOUT["PATH_NODE"], oArgs) iRet = iOsSystemNmapUdp(aOUT["PATH_NODE"], oArgs)
else: else:
LOG.warn(f"{oArgs.info} no PATH_NODE") LOG.warn(f"{oArgs.info} no PATH_NODE")
iret = 0 iRet = 0
else: else:
LOG.warn(f"{oArgs.command} UNREGOGNIZED") LOG.warn(f"{oArgs.command} UNREGOGNIZED")
if oStream and oStream != sys.stdout and oStream != sys.stderr: if oStream and oStream != sys.stdout and oStream != sys.stderr:
oStream.close() oStream.close()
return iret return iRet
def oMainArgparser(_=None): def oMainArgparser(_=None):
if not os.path.exists('/proc/sys/net/ipv6'): if not os.path.exists('/proc/sys/net/ipv6'):
@ -1460,7 +1400,6 @@ if __name__ == '__main__':
print('Supported Quads: section,num,key,type ' +sEDIT_HELP) print('Supported Quads: section,num,key,type ' +sEDIT_HELP)
sys.exit(0) sys.exit(0)
__builtins__.oArgs = oArgs
vSetupLogging(oArgs.log_level) vSetupLogging(oArgs.log_level)
i = 0 i = 0
for sProOrNodes in oArgs.lprofile: for sProOrNodes in oArgs.lprofile: