emdee/tox_profile
2
3
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix tox_savefile_test.bash

Browse Source
This commit is contained in:
emdee 2022-10-03 05:35:26 +00:00
parent 525446cc4a
commit 6752347ea6
2 changed files with 89 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
tox_savefile.py
View File

@ -46,6 +46,7 @@ commands, or the filename of the nodes file for the nodes command.
import sys import sys
import os import os
import time
import struct import struct
from socket import inet_ntop, AF_INET6, AF_INET from socket import inet_ntop, AF_INET6, AF_INET
import logging import logging
@ -76,7 +77,7 @@ except ImportError as e:
try: try:
# https://git.plastiras.org/emdee/toxygen_wrapper # https://git.plastiras.org/emdee/toxygen_wrapper
from wrapper.toxencryptsave import ToxEncryptSave from wrapper.toxencryptsave import ToxEncryptSave
from wrapper_tests.support_http import download_url from wrapper_tests.support_http import download_url, bAreWeConnected
except ImportError as e: except ImportError as e:
print(f"Import Error {e}") print(f"Import Error {e}")
print("Download toxygen_wrapper to deal with encrypted tox files, from:") print("Download toxygen_wrapper to deal with encrypted tox files, from:")
@ -95,13 +96,13 @@ sDIR = os.environ.get('TMPDIR', '/tmp')
sTOX_VERSION = "1000002018" sTOX_VERSION = "1000002018"
bHAVE_NMAP = shutil.which('nmap') bHAVE_NMAP = shutil.which('nmap')
bHAVE_JQ = shutil.which('jq') bHAVE_JQ = shutil.which('jq')
bHAVE_BASH = shutil.which('bash')
bMARK = b'\x00\x00\x00\x00\x1f\x1b\xed\x15' bMARK = b'\x00\x00\x00\x00\x1f\x1b\xed\x15'
bDEBUG = 'DEBUG' in os.environ and os.environ['DEBUG'] != 0 bDEBUG = 'DEBUG' in os.environ and os.environ['DEBUG'] != 0
def trace(s): LOG.log(LOG.level, '+ ' +s) def trace(s): LOG.log(LOG.level, '+ ' +s)
LOG.trace = trace LOG.trace = trace
global lOUT, bOUT, aOUT, sENC global bOUT, aOUT, sENC
lOUT = []
aOUT = {} aOUT = {}
bOUT = b'' bOUT = b''
sENC = 'utf-8' sENC = 'utf-8'
@ -417,7 +418,7 @@ def lProcessDHTnodes(state, index, length, result, label="DHTnode"):
return lIN return lIN
def process_chunk(index, state, oArgs=None): def process_chunk(index, state, oArgs=None):
global lOUT, bOUT, aOUT global bOUT, aOUT
global sENC global sENC
length = struct.unpack_from("<I", state, index)[0] length = struct.unpack_from("<I", state, index)[0]
@ -447,23 +448,23 @@ def process_chunk(index, state, oArgs=None):
aIN = {"Nospam": f"{nospam}", aIN = {"Nospam": f"{nospam}",
"Public_key": f"{public_key}", "Public_key": f"{public_key}",
"Private_key": f"{private_key}"} "Private_key": f"{private_key}"}
lOUT += [{label: aIN}]; aOUT.update({label: aIN}) aOUT.update({label: aIN})
elif data_type == MESSENGER_STATE_TYPE_DHT: elif data_type == MESSENGER_STATE_TYPE_DHT:
LOG.debug(f"process_chunk {label} length={length}") LOG.debug(f"process_chunk {label} length={length}")
lIN = lProcessDHTnodes(state, index, length, result) lIN = lProcessDHTnodes(state, index, length, result)
lOUT += [{label: lIN}]; aOUT.update({label: lIN}) aOUT.update({label: lIN})
elif data_type == MESSENGER_STATE_TYPE_FRIENDS: elif data_type == MESSENGER_STATE_TYPE_FRIENDS:
LOG.info(f"{label} {length // 2216} FRIENDS {length % 2216}") LOG.info(f"{label} {length // 2216} FRIENDS {length % 2216}")
lIN = lProcessFriends(state, index, length, result) lIN = lProcessFriends(state, index, length, result)
lOUT += [{label: lIN}]; aOUT.update({label: lIN}) aOUT.update({label: lIN})
elif data_type == MESSENGER_STATE_TYPE_NAME: elif data_type == MESSENGER_STATE_TYPE_NAME:
name = str(result, sENC) name = str(result, sENC)
LOG.info(f"{label} Nick_name = " +name) LOG.info(f"{label} Nick_name = " +name)
aIN = {"Nick_name": name} aIN = {"Nick_name": name}
lOUT += [{label: aIN}]; aOUT.update({label: aIN}) aOUT.update({label: aIN})
if oArgs.command == 'edit' and section == label: if oArgs.command == 'edit' and section == label:
## NAME,.,Nick_name,str ## NAME,.,Nick_name,str
if key == "Nick_name": if key == "Nick_name":
@ -475,7 +476,7 @@ def process_chunk(index, state, oArgs=None):
mess = str(result, sENC) mess = str(result, sENC)
LOG.info(f"{label} StatusMessage = " +mess) LOG.info(f"{label} StatusMessage = " +mess)
aIN = {"Status_message": mess} aIN = {"Status_message": mess}
lOUT += [{label: aIN}]; aOUT.update({label: aIN}) aOUT.update({label: aIN})
if oArgs.command == 'edit' and section == label: if oArgs.command == 'edit' and section == label:
## STATUSMESSAGE,.,Status_message,str ## STATUSMESSAGE,.,Status_message,str
if key == "Status_message": if key == "Status_message":
@ -490,7 +491,7 @@ def process_chunk(index, state, oArgs=None):
status = dStatus[status] status = dStatus[status]
LOG.info(f"{label} = " +status) LOG.info(f"{label} = " +status)
aIN = {f"Online_status": status} aIN = {f"Online_status": status}
lOUT += [{"STATUS": aIN}]; aOUT.update({"STATUS": aIN}) aOUT.update({label: aIN})
if oArgs.command == 'edit' and section == label: if oArgs.command == 'edit' and section == label:
## STATUS,.,Online_status,int ## STATUS,.,Online_status,int
if key == "Online_status": if key == "Online_status":
@ -504,7 +505,7 @@ def process_chunk(index, state, oArgs=None):
else: else:
lIN = [] lIN = []
LOG.info(f"NO {label}") LOG.info(f"NO {label}")
lOUT += [{label: lIN}]; aOUT.update({label: lIN}) aOUT.update({label: lIN})
elif data_type == MESSENGER_STATE_TYPE_TCP_RELAY: elif data_type == MESSENGER_STATE_TYPE_TCP_RELAY:
if length > 0: if length > 0:
@ -512,14 +513,17 @@ def process_chunk(index, state, oArgs=None):
else: else:
lIN = [] lIN = []
LOG.info(f"NO {label}") LOG.info(f"NO {label}")
lOUT += [{label: lIN}]; aOUT.update({label: lIN}) aOUT.update({label: lIN})
elif data_type == MESSENGER_STATE_TYPE_PATH_NODE: elif data_type == MESSENGER_STATE_TYPE_PATH_NODE:
#define NUM_SAVED_PATH_NODES 8 #define NUM_SAVED_PATH_NODES 8
assert length % 8 == 0, length if not length % 8 == 0:
# this should be an assert?
LOG.warn(f"process_chunk {label} mod={length % 8}")
else:
LOG.debug(f"process_chunk {label} bytes={length}") LOG.debug(f"process_chunk {label} bytes={length}")
lIN = lProcessNodeInfo(state, index, length, result, "PATHnode") lIN = lProcessNodeInfo(state, index, length, result, "PATHnode")
lOUT += [{label: lIN}]; aOUT.update({label: lIN}) aOUT.update({label: lIN})
elif data_type == MESSENGER_STATE_TYPE_CONFERENCES: elif data_type == MESSENGER_STATE_TYPE_CONFERENCES:
lIN = [] lIN = []
@ -527,7 +531,7 @@ def process_chunk(index, state, oArgs=None):
LOG.debug(f"TODO process_chunk {label} bytes={length}") LOG.debug(f"TODO process_chunk {label} bytes={length}")
else: else:
LOG.info(f"NO {label}") LOG.info(f"NO {label}")
lOUT += [{label: []}]; aOUT.update({label: []}) aOUT.update({label: []})
elif data_type != MESSENGER_STATE_TYPE_END: elif data_type != MESSENGER_STATE_TYPE_END:
LOG.error("UNRECOGNIZED datatype={datatype}") LOG.error("UNRECOGNIZED datatype={datatype}")
@ -546,8 +550,7 @@ def process_chunk(index, state, oArgs=None):
struct.pack("<H", check) + \ struct.pack("<H", check) + \
result result
if data_type == MESSENGER_STATE_TYPE_END or \ if data_type == MESSENGER_STATE_TYPE_END or index + 8 >= len(state):
index + 8 >= len(state):
diff = len(bSAVE) - len(bOUT) diff = len(bSAVE) - len(bOUT)
if oArgs.command != 'edit' and diff > 0: if oArgs.command != 'edit' and diff > 0:
# if short repacking as we read - tox_profile is padded with nulls # if short repacking as we read - tox_profile is padded with nulls
@ -556,17 +559,6 @@ def process_chunk(index, state, oArgs=None):
process_chunk(new_index, state, oArgs) process_chunk(new_index, state, oArgs)
def bAreWeConnected():
# FixMe: Linux
sFile = f"/proc/{os.getpid()}/net/route"
if not os.path.isfile(sFile): return None
i = 0
for elt in open(sFile, "r").readlines():
if elt.startswith('Iface'): continue
if elt.startswith('lo'): continue
i += 1
return i > 0
sNMAP_TCP = """#!/bin/bash sNMAP_TCP = """#!/bin/bash
ip="" ip=""
declare -a ports declare -a ports
@ -601,6 +593,9 @@ jq '.|with_entries(select(.key|match("nodes"))).nodes[]|select(.status_tcp)|sele
done""" done"""
def vBashFileNmapTcp(): def vBashFileNmapTcp():
assert bHAVE_JQ, "jq is required for this command"
assert bHAVE_NMAP, "nmap is required for this command"
assert bHAVE_BASH, "bash is required for this command"
f = "NmapTcp.bash" f = "NmapTcp.bash"
sFile = os.path.join(sDIR, f) sFile = os.path.join(sDIR, f)
if not os.path.exists(sFile): if not os.path.exists(sFile):
@ -610,6 +605,9 @@ def vBashFileNmapTcp():
return sFile return sFile
def vBashFileNmapUdp(): def vBashFileNmapUdp():
assert bHAVE_JQ, "jq is required for this command"
assert bHAVE_NMAP, "nmap is required for this command"
assert bHAVE_BASH, "bash is required for this command"
f = "NmapUdp.bash" f = "NmapUdp.bash"
sFile = os.path.join(sDIR, f) sFile = os.path.join(sDIR, f)
if not os.path.exists(sFile): if not os.path.exists(sFile):
@ -741,63 +739,71 @@ if __name__ == '__main__':
elif oArgs.command == 'nodes': elif oArgs.command == 'nodes':
iRet = -1 iRet = -1
ep_sec = str(int(time.time()))
json_head = '{"last_scan":' +ep_sec \
+',"last_refresh":' +ep_sec \
+',"nodes":['
if oArgs.nodes == 'select_tcp': if oArgs.nodes == 'select_tcp':
assert oArgs.output, "--output required for this command" assert oArgs.output, "--output required for this command"
assert bHAVE_JQ, "jq is required for this command" assert bHAVE_JQ, "jq is required for this command"
with open(oArgs.output, 'wt') as oFd:
oFd.write(json_head)
cmd = f"cat '{sFile}' | jq '.|with_entries(select(.key|match(\"nodes\"))).nodes[]|select(.status_tcp)|select(.ipv4|match(\".\"))' " cmd = f"cat '{sFile}' | jq '.|with_entries(select(.key|match(\"nodes\"))).nodes[]|select(.status_tcp)|select(.ipv4|match(\".\"))' "
iRet = os.system(cmd +f" > {oArgs.output}") iRet = os.system(cmd +f" >> {oArgs.output}")
with open(oArgs.output, 'at') as oFd:
oFd.write(']}')
elif oArgs.nodes == 'select_udp': elif oArgs.nodes == 'select_udp':
assert oArgs.output, "--output required for this command" assert oArgs.output, "--output required for this command"
assert bHAVE_JQ, "jq is required for this command" assert bHAVE_JQ, "jq is required for this command"
with open(oArgs.output, 'wt') as oFd:
oFd.write(json_head)
cmd = f"cat '{sFile}' | jq '.|with_entries(select(.key|match(\"nodes\"))).nodes[]|select(.status_udp)|select(.ipv4|match(\".\"))' " cmd = f"cat '{sFile}' | jq '.|with_entries(select(.key|match(\"nodes\"))).nodes[]|select(.status_udp)|select(.ipv4|match(\".\"))' "
iRet = os.system(cmd +f" > {oArgs.output}") iRet = os.system(cmd +f" >> {oArgs.output}")
with open(oArgs.output, 'at') as oFd:
oFd.write(']}')
elif oArgs.nodes == 'select_version': elif oArgs.nodes == 'select_version':
assert bHAVE_JQ, "jq is required for this command" assert bHAVE_JQ, "jq is required for this command"
assert oArgs.output, "--output required for this command" assert oArgs.output, "--output required for this command"
with open(oArgs.output, 'wt') as oFd:
oFd.write(json_head)
cmd = f"cat '{sFile}' | jq '.|with_entries(select(.key|match(\"nodes\"))).nodes[]|select(.status_udp)|select(.version|match(\"{sTOX_VERSION}\"))' " cmd = f"cat '{sFile}' | jq '.|with_entries(select(.key|match(\"nodes\"))).nodes[]|select(.status_udp)|select(.version|match(\"{sTOX_VERSION}\"))' "
iRet = os.system(cmd +f" > {oArgs.output}") iRet = os.system(cmd +f" >> {oArgs.output}")
with open(oArgs.output, 'at') as oFd:
oFd.write(']}')
elif oArgs.nodes == 'nmap_tcp' and bHAVE_NMAP: elif oArgs.nodes == 'nmap_tcp':
assert bHAVE_JQ, "jq is required for this command"
assert oArgs.output, "--output required for this command" assert oArgs.output, "--output required for this command"
if not bAreWeConnected(): if not bAreWeConnected():
LOG.error(f"{oArgs.nodes} not connected") LOG.warn(f"{oArgs.nodes} we are not connected")
iRet = -1
else:
cmd = vBashFileNmapTcp() cmd = vBashFileNmapTcp()
iRet = os.system(f"bash {cmd} < '{sFile}'" +f" >'{oArgs.output}'") iRet = os.system(f"bash {cmd} < '{sFile}'" +f" >'{oArgs.output}'")
elif oArgs.nodes == 'nmap_udp' and bHAVE_NMAP: elif oArgs.nodes == 'nmap_udp':
assert bHAVE_JQ, "jq is required for this command"
assert oArgs.output, "--output required for this command" assert oArgs.output, "--output required for this command"
if not bAreWeConnected(): if not bAreWeConnected():
LOG.error(f"{oArgs.nodes} not connected") LOG.warn(f"{oArgs.nodes} we are not connected")
iRet = -1
else:
cmd = vBashFileNmapUdp() cmd = vBashFileNmapUdp()
iRet = os.system(f"bash {cmd} < '{sFile}'" +f" >'{oArgs.output}'") iRet = os.system(f"bash {cmd} < '{sFile}'" +f" >'{oArgs.output}'")
elif oArgs.nodes == 'download' and download_url: elif oArgs.nodes == 'download' and download_url:
if not bAreWeConnected(): if not bAreWeConnected():
LOG.error(f"{oArgs.nodes} not connected") LOG.warn(f"{oArgs.nodes} we are not connected")
iRet = -1
else:
url = oArgs.download_nodes_url url = oArgs.download_nodes_url
bSAVE = download_url(url) b = download_url(url)
if not bSAVE: if not bSAVE:
LOG.warn("failed downloading list of nodes") LOG.warn("failed downloading list of nodes")
iRet = -1 iRet = -1
else: else:
if oArgs.output: if oArgs.output:
oStream = open(oArgs.output, 'rb') oStream = open(oArgs.output, 'rb')
oStream.write(bSAVE) oStream.write(b)
else: else:
oStream = sys.stdout oStream = sys.stdout
oStream.write(str(bSAVE, sENC)) oStream.write(str(b, sENC))
iRet = -1 iRet = -1
LOG.info(f"downloaded list of nodes saved to {oStream}") LOG.info(f"downloaded list of nodes to {oStream}")
if iRet > 0: if iRet > 0:
LOG.warn(f"{oArgs.nodes} iRet={iRet} to {oArgs.output}") LOG.warn(f"{oArgs.nodes} iRet={iRet} to {oArgs.output}")

26
tox_savefile_test.bash
View File

@ -1,4 +1,4 @@
#!/bin/sh -e #!/bin/sh
# -*- mode: sh; fill-column: 75; tab-width: 8; coding: utf-8-unix -*- # -*- mode: sh; fill-column: 75; tab-width: 8; coding: utf-8-unix -*-
PREFIX=/o/var/local/src PREFIX=/o/var/local/src
@ -13,8 +13,8 @@ WRAPPER=$PREFIX/toxygen_wrapper
ERROR() { echo ERROR $* ; } ERROR() { echo ERROR $* ; }
} }
set -- -e # set -- -e
target=$PREFIX/tox_profile/logging_tox_savefile.py target=$PREFIX/tox_profile/tox_savefile.py
[ -s $target ] || exit 1 [ -s $target ] || exit 1
tox=$HOME/.config/tox/toxic_profile.tox tox=$HOME/.config/tox/toxic_profile.tox
@ -43,7 +43,10 @@ $EXE $target --command decrypt --output /tmp/toxic_profile.bin $tox || exit 11
tox=/tmp/toxic_profile.bin tox=/tmp/toxic_profile.bin
INFO info $tox INFO info $tox
$EXE $target --command info --info info $tox 2>/tmp/toxic_profile.info || exit 13 $EXE $target --command info --info info $tox 2>/tmp/toxic_profile.info || {
ERROR $EXE $target --command info --info info $tox
exit 13
}
[ -s /tmp/toxic_profile.info ] || exit 14 [ -s /tmp/toxic_profile.info ] || exit 14
INFO /tmp/toxic_profile.save INFO /tmp/toxic_profile.save
@ -86,10 +89,12 @@ the_base=`echo $the_tox | sed -e 's/.save$//' -e 's/.json$//'`
$EXE $target --command nodes --nodes $nmap \ $EXE $target --command nodes --nodes $nmap \
--output $the_base.$nmap.json $the_tox || exit 31 --output $the_base.$nmap.json $the_tox || exit 31
[ -s $the_base.$nmap.json ] || exit 32 [ -s $the_base.$nmap.json ] || exit 32
[ $nmap = select_tcp ] && \
grep '"status_tcp": false' $the_base.select_tcp.json && exit 33
[ $nmap = select_udp ] && \
grep '"status_udp": false' $the_base.select_udp.json && exit 34
done done
grep '"status_tcp": false' $the_base.select_tcp.json && exit 33
grep '"status_udp": false' $the_base.select_udp.json && exit 34
ls -l /tmp/toxic_profile.* /tmp/toxic_nodes.* ls -l /tmp/toxic_profile.* /tmp/toxic_nodes.*
@ -100,9 +105,16 @@ the_base=`echo $the_tox | sed -e 's/.save$//' -e 's/.tox$//'`
[ "$HAVE_JQ" = 0 ] || \ [ "$HAVE_JQ" = 0 ] || \
[ "$HAVE_NMAP" = 0 ] || \ [ "$HAVE_NMAP" = 0 ] || \
for nmap in nmap_tcp nmap_udp nmap_onion ; do for nmap in nmap_tcp nmap_udp nmap_onion ; do
# [ $nmap = select_tcp ] && continue
# [ $nmap = select_udp ] && continue
INFO $the_base.$nmap INFO $the_base.$nmap
$EXE $target --command info --info $nmap \ $EXE $target --command info --info $nmap \
--output $the_base.$nmap $the_tox.json || exit 40 --output $the_base.$nmap $the_tox.json || {
# select_tcp may be empty and jq errors
# exit 41
WARN $the_base.$nmap.json
continue
}
[ -s $the_base.$nmap.json ] || exit 41 [ -s $the_base.$nmap.json ] || exit 41
done done