updated docstring
This commit is contained in:
emdee
parent
28d1d34dbd
commit
6cf32758c1
@ -106,13 +106,6 @@ try:
|
|||||||
except:
|
except:
|
||||||
ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None
|
ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None
|
||||||
|
|
||||||
try:
|
|
||||||
if 'COLOREDLOGS_LEVEL_STYLES' not in os.environ:
|
|
||||||
os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red'
|
|
||||||
# https://pypi.org/project/coloredlogs/
|
|
||||||
import coloredlogs
|
|
||||||
except ImportError as e:
|
|
||||||
coloredlogs = False
|
|
||||||
|
|
||||||
global LOG
|
global LOG
|
||||||
import logging
|
import logging
|
||||||
@ -120,7 +113,7 @@ import warnings
|
|||||||
warnings.filterwarnings('ignore')
|
warnings.filterwarnings('ignore')
|
||||||
LOG = logging.getLogger()
|
LOG = logging.getLogger()
|
||||||
|
|
||||||
import requests
|
from support_phantompy import vsetup_logging
|
||||||
from trustor_poc import oDownloadUrlUrllib3 as oDownloadUrl
|
from trustor_poc import oDownloadUrlUrllib3 as oDownloadUrl
|
||||||
from trustor_poc import idns_validate, TrustorError
|
from trustor_poc import idns_validate, TrustorError
|
||||||
from support_onions import icheck_torrc, bAreWeConnected, lIntroductionPoints, zResolveDomain, vwait_for_controller, yKNOWN_NODNS
|
from support_onions import icheck_torrc, bAreWeConnected, lIntroductionPoints, zResolveDomain, vwait_for_controller, yKNOWN_NODNS
|
||||||
@ -383,67 +376,6 @@ def aParseContact(contact, fp):
|
|||||||
a = yaml.safe_load(oFd)
|
a = yaml.safe_load(oFd)
|
||||||
return a
|
return a
|
||||||
|
|
||||||
def vsetup_logging(log_level, logfile=''):
|
|
||||||
global LOG
|
|
||||||
add = True
|
|
||||||
|
|
||||||
# stem fucks up logging
|
|
||||||
from stem.util import log
|
|
||||||
logging.getLogger('stem').setLevel(30)
|
|
||||||
|
|
||||||
logging._defaultFormatter = logging.Formatter(datefmt='%m-%d %H:%M:%S')
|
|
||||||
logging._defaultFormatter.default_time_format = '%m-%d %H:%M:%S'
|
|
||||||
logging._defaultFormatter.default_msec_format = ''
|
|
||||||
|
|
||||||
kwargs = dict(level=log_level,
|
|
||||||
force=True,
|
|
||||||
format='%(levelname)s %(message)s')
|
|
||||||
|
|
||||||
if logfile:
|
|
||||||
add = logfile.startswith('+')
|
|
||||||
sub = logfile.startswith('-')
|
|
||||||
if add or sub:
|
|
||||||
logfile = logfile[1:]
|
|
||||||
kwargs['filename'] = logfile
|
|
||||||
|
|
||||||
if coloredlogs:
|
|
||||||
# https://pypi.org/project/coloredlogs/
|
|
||||||
aKw = dict(level=log_level,
|
|
||||||
logger=LOG,
|
|
||||||
stream=sys.stdout if add else None,
|
|
||||||
fmt='%(levelname)s %(message)s'
|
|
||||||
)
|
|
||||||
coloredlogs.install(**aKw)
|
|
||||||
if logfile:
|
|
||||||
oHandler = logging.FileHandler(logfile)
|
|
||||||
LOG.addHandler(oHandler)
|
|
||||||
LOG.info(f"CSetting log_level to {log_level!s}")
|
|
||||||
else:
|
|
||||||
logging.basicConfig(**kwargs)
|
|
||||||
if add and logfile:
|
|
||||||
oHandler = logging.StreamHandler(sys.stdout)
|
|
||||||
LOG.addHandler(oHandler)
|
|
||||||
LOG.info(f"SSetting log_level to {log_level!s}")
|
|
||||||
|
|
||||||
logging._levelToName = {
|
|
||||||
logging.CRITICAL: 'CRITICAL',
|
|
||||||
logging.ERROR: 'ERROR',
|
|
||||||
logging.WARNING: 'WARN',
|
|
||||||
logging.INFO: 'INFO',
|
|
||||||
logging.DEBUG: 'DEBUG',
|
|
||||||
logging.NOTSET: 'NOTSET',
|
|
||||||
}
|
|
||||||
logging._nameToLevel = {
|
|
||||||
'CRITICAL': logging.CRITICAL,
|
|
||||||
'FATAL': logging.FATAL,
|
|
||||||
'ERROR': logging.ERROR,
|
|
||||||
'WARN': logging.WARNING,
|
|
||||||
'WARNING': logging.WARNING,
|
|
||||||
'INFO': logging.INFO,
|
|
||||||
'DEBUG': logging.DEBUG,
|
|
||||||
'NOTSET': logging.NOTSET,
|
|
||||||
}
|
|
||||||
|
|
||||||
def oMainArgparser(_=None):
|
def oMainArgparser(_=None):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@ -508,7 +440,8 @@ def oMainArgparser(_=None):
|
|||||||
parser.add_argument('--white_services', type=str,
|
parser.add_argument('--white_services', type=str,
|
||||||
default='',
|
default='',
|
||||||
help="comma sep. list of onions to whitelist their introduction points - BROKEN")
|
help="comma sep. list of onions to whitelist their introduction points - BROKEN")
|
||||||
parser.add_argument('--torrc_output', type=str, default='',
|
parser.add_argument('--torrc_output', type=str,
|
||||||
|
default=os.path.join(ETC_DIR, 'torrc.new'),
|
||||||
help="Write the torrc configuration to a file")
|
help="Write the torrc configuration to a file")
|
||||||
parser.add_argument('--proof_output', type=str, default=os.path.join(ETC_DIR, 'proof.yaml'),
|
parser.add_argument('--proof_output', type=str, default=os.path.join(ETC_DIR, 'proof.yaml'),
|
||||||
help="Write the proof data of the included nodes to a YAML file")
|
help="Write the proof data of the included nodes to a YAML file")
|
||||||
|
|||||||
79
lookupdns.py
79
lookupdns.py
@ -1,79 +0,0 @@
|
|||||||
#!/usr/local/bin/python3.sh
|
|
||||||
# -*-mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import os
|
|
||||||
import traceback
|
|
||||||
|
|
||||||
from phantompy import Render
|
|
||||||
|
|
||||||
global LOG
|
|
||||||
import logging
|
|
||||||
import warnings
|
|
||||||
warnings.filterwarnings('ignore')
|
|
||||||
LOG = logging.getLogger()
|
|
||||||
|
|
||||||
class LookFor(Render):
|
|
||||||
|
|
||||||
def __init__(self, url, outfile, jsfile=None):
|
|
||||||
self.uri = url
|
|
||||||
Render.__init__(self, url, outfile, jsfile)
|
|
||||||
|
|
||||||
def ilookfor(self, html):
|
|
||||||
import json
|
|
||||||
marker = '<pre style="word-wrap: break-word; white-space: pre-wrap;">'
|
|
||||||
if marker not in html: return ''
|
|
||||||
i = html.find(marker) + len(marker)
|
|
||||||
html = html[i:]
|
|
||||||
assert html[0] == '{', html
|
|
||||||
i = html.find('</pre')
|
|
||||||
html = html[:i]
|
|
||||||
assert html[-1] == '}', html
|
|
||||||
LOG.debug(f"Found {len(html)} json")
|
|
||||||
o = json.loads(html)
|
|
||||||
if "Answer" not in o.keys() or type(o["Answer"]) != list:
|
|
||||||
LOG.warn(f"FAIL {self.uri}")
|
|
||||||
return 1
|
|
||||||
for elt in o["Answer"]:
|
|
||||||
assert type(elt) == dict, elt
|
|
||||||
assert 'type' in elt, elt
|
|
||||||
if elt['type'] != 16: continue
|
|
||||||
assert 'data' in elt, elt
|
|
||||||
if elt['data'] == 'we-run-this-tor-relay':
|
|
||||||
LOG.info(f"OK {self.uri}")
|
|
||||||
return 0
|
|
||||||
LOG.warn(f"BAD {self.uri}")
|
|
||||||
return 2
|
|
||||||
|
|
||||||
def _html_callback(self, *args):
|
|
||||||
"""print(self, QPrinter, Callable[[bool], None])"""
|
|
||||||
if type(args[0]) is str:
|
|
||||||
self._save(args[0])
|
|
||||||
i = self.ilookfor(args[0])
|
|
||||||
self._exit(i)
|
|
||||||
|
|
||||||
def _save(self, html):
|
|
||||||
sfile = self.outfile.replace('.pdf','.out')
|
|
||||||
# CompleteHtmlSaveFormat SingleHtmlSaveFormat MimeHtmlSaveFormat
|
|
||||||
with open(sfile, 'wt') as ofd:
|
|
||||||
ofd.write(html)
|
|
||||||
LOG.debug(f"Saved {sfile}")
|
|
||||||
|
|
||||||
def _loadFinished(self, result):
|
|
||||||
LOG.debug("phantom.py: Loading finished!")
|
|
||||||
self.toHtml(self._html_callback)
|
|
||||||
|
|
||||||
def main():
|
|
||||||
if (len(sys.argv) < 3):
|
|
||||||
LOG.info("USAGE: lookupdns.py <url> <pdf-file> [<javascript-file>]")
|
|
||||||
else:
|
|
||||||
url = sys.argv[1]
|
|
||||||
outfile = sys.argv[2]
|
|
||||||
jsfile = sys.argv[3] if len(sys.argv) > 3 else None
|
|
||||||
r = LookFor(url, outfile, jsfile)
|
|
||||||
|
|
||||||
sys.exit(0)
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
|
|
||||||
260
phantompy.py
260
phantompy.py
@ -1,260 +0,0 @@
|
|||||||
#!/usr/local/bin/python3.sh
|
|
||||||
# -*-mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*-
|
|
||||||
# https://gist.github.com/michaelfranzl/91f0cc13c56120391b949f885643e974/raw/a0601515e7a575bc4c7d4d2a20973b29b6c6f2df/phantom.py
|
|
||||||
"""
|
|
||||||
# phantom.py
|
|
||||||
|
|
||||||
Simple but fully scriptable headless QtWebKit browser using PyQt5 in Python3,
|
|
||||||
specialized in executing external JavaScript and generating PDF files. A lean
|
|
||||||
replacement for other bulky headless browser frameworks.
|
|
||||||
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
If you have a display attached:
|
|
||||||
|
|
||||||
./phantom.py <url> <pdf-file> [<javascript-file>]
|
|
||||||
|
|
||||||
If you don't have a display attached (i.e. on a remote server):
|
|
||||||
|
|
||||||
xvfb-run ./phantom.py <url> <pdf-file> [<javascript-file>]
|
|
||||||
|
|
||||||
Arguments:
|
|
||||||
|
|
||||||
<url> Can be a http(s) URL or a path to a local file
|
|
||||||
<pdf-file> Path and name of PDF file to generate
|
|
||||||
[<javascript-file>] (optional) Path and name of a JavaScript file to execute
|
|
||||||
|
|
||||||
|
|
||||||
## Features
|
|
||||||
|
|
||||||
* Generate a PDF screenshot of the web page after it is completely loaded.
|
|
||||||
* Optionally execute a local JavaScript file specified by the argument
|
|
||||||
<javascript-file> after the web page is completely loaded, and before
|
|
||||||
the PDF is generated.
|
|
||||||
* console.log's will be printed to stdout.
|
|
||||||
* Easily add new features by changing the source code of this script, without
|
|
||||||
compiling C++ code. For more advanced applications, consider attaching
|
|
||||||
PyQt objects/methods to WebKit's JavaScript space by using
|
|
||||||
`QWebFrame::addToJavaScriptWindowObject()`.
|
|
||||||
|
|
||||||
If you execute an external <javascript-file>, phantom.py has no way of knowing
|
|
||||||
when that script has finished doing its work. For this reason, the external
|
|
||||||
script should execute `console.log("__PHANTOM_PY_DONE__");` when done. This will
|
|
||||||
trigger the PDF generation, after which phantom.py will exit. If no
|
|
||||||
`__PHANTOM_PY_DONE__` string is seen on the console for 10 seconds, phantom.py
|
|
||||||
will exit without doing anything. This behavior could be implemented more
|
|
||||||
elegantly without console.log's but it is the simplest solution.
|
|
||||||
|
|
||||||
It is important to remember that since you're just running WebKit, you can use
|
|
||||||
everything that WebKit supports, including the usual JS client libraries, CSS,
|
|
||||||
CSS @media types, etc.
|
|
||||||
|
|
||||||
|
|
||||||
## Dependencies
|
|
||||||
|
|
||||||
* Python3
|
|
||||||
* PyQt5
|
|
||||||
* xvfb (optional for display-less machines)
|
|
||||||
|
|
||||||
Installation of dependencies in Debian Stretch is easy:
|
|
||||||
|
|
||||||
apt-get install xvfb python3-pyqt5 python3-pyqt5.qtwebkit
|
|
||||||
|
|
||||||
Finding the equivalent for other OSes is an exercise that I leave to you.
|
|
||||||
|
|
||||||
|
|
||||||
## Examples
|
|
||||||
|
|
||||||
Given the following file /tmp/test.html
|
|
||||||
|
|
||||||
<html>
|
|
||||||
<body>
|
|
||||||
<p>foo <span id="id1">foo</span> <span id="id2">foo</span></p>
|
|
||||||
</body>
|
|
||||||
<script>
|
|
||||||
document.getElementById('id1').innerHTML = "bar";
|
|
||||||
</script>
|
|
||||||
</html>
|
|
||||||
|
|
||||||
... and the following file /tmp/test.js:
|
|
||||||
|
|
||||||
document.getElementById('id2').innerHTML = "baz";
|
|
||||||
console.log("__PHANTOM_PY_DONE__");
|
|
||||||
|
|
||||||
... and running this script (without attached display) ...
|
|
||||||
|
|
||||||
xvfb-run python3 phantom.py /tmp/test.html /tmp/out.pdf /tmp/test.js
|
|
||||||
|
|
||||||
... you will get a PDF file /tmp/out.pdf with the contents "foo bar baz".
|
|
||||||
|
|
||||||
Note that the second occurrence of "foo" has been replaced by the web page's own
|
|
||||||
script, and the third occurrence of "foo" by the external JS file.
|
|
||||||
|
|
||||||
|
|
||||||
## License
|
|
||||||
|
|
||||||
Copyright 2017 Michael Karl Franzl
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
||||||
this software and associated documentation files (the "Software"), to deal in
|
|
||||||
the Software without restriction, including without limitation the rights to
|
|
||||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
|
||||||
of the Software, and to permit persons to whom the Software is furnished to do
|
|
||||||
so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
|
||||||
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
|
||||||
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
|
||||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
||||||
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import os
|
|
||||||
import traceback
|
|
||||||
import atexit
|
|
||||||
from PyQt5.QtCore import QUrl
|
|
||||||
from PyQt5.QtCore import QTimer
|
|
||||||
from PyQt5.QtWidgets import QApplication
|
|
||||||
from PyQt5.QtPrintSupport import QPrinter
|
|
||||||
from PyQt5.QtWebEngineWidgets import QWebEnginePage as QWebPage
|
|
||||||
|
|
||||||
global LOG
|
|
||||||
import logging
|
|
||||||
import warnings
|
|
||||||
warnings.filterwarnings('ignore')
|
|
||||||
LOG = logging.getLogger()
|
|
||||||
|
|
||||||
def prepare():
|
|
||||||
sfile = '/tmp/test.js'
|
|
||||||
if not os.path.exists(sfile):
|
|
||||||
with open(sfile, 'wt') as ofd:
|
|
||||||
ofd.write("""
|
|
||||||
document.getElementById('id2').innerHTML = "baz";
|
|
||||||
console.log("__PHANTOM_PY_DONE__");
|
|
||||||
""")
|
|
||||||
sys.stderr.write(f"wrote {sfile} ")
|
|
||||||
sfile = '/tmp/test.html'
|
|
||||||
if not os.path.exists(sfile):
|
|
||||||
with open(sfile, 'wt') as ofd:
|
|
||||||
ofd.write("""
|
|
||||||
<html>
|
|
||||||
<body>
|
|
||||||
<p>foo <span id="id1">foo</span> <span id="id2">foo</span></p>
|
|
||||||
</body>
|
|
||||||
<script>
|
|
||||||
document.getElementById('id1').innerHTML = "bar";
|
|
||||||
</script>
|
|
||||||
</html>
|
|
||||||
""")
|
|
||||||
sys.stderr.write(f"wrote {sfile} ")
|
|
||||||
sys.stderr.write("\n")
|
|
||||||
|
|
||||||
class Render(QWebPage):
|
|
||||||
def __init__(self, url, outfile, jsfile=None):
|
|
||||||
self.app = QApplication(sys.argv)
|
|
||||||
|
|
||||||
QWebPage.__init__(self)
|
|
||||||
|
|
||||||
self.jsfile = jsfile
|
|
||||||
self.outfile = outfile
|
|
||||||
|
|
||||||
qurl = QUrl.fromUserInput(url)
|
|
||||||
|
|
||||||
LOG.debug(f"phantom.py: URL= {qurl} OUTFILE={outfile} JSFILE= {jsfile)")
|
|
||||||
|
|
||||||
# The PDF generation only happens when the special string __PHANTOM_PY_DONE__
|
|
||||||
# is sent to console.log(). The following JS string will be executed by
|
|
||||||
# default, when no external JavaScript file is specified.
|
|
||||||
self.js_contents = "setTimeout(function() { console.log('__PHANTOM_PY_DONE__') }, 5000);";
|
|
||||||
|
|
||||||
if jsfile:
|
|
||||||
try:
|
|
||||||
f = open(self.jsfile)
|
|
||||||
self.js_contents = f.read()
|
|
||||||
f.close()
|
|
||||||
except:
|
|
||||||
LOG.error(traceback.format_exc())
|
|
||||||
self._exit(10)
|
|
||||||
|
|
||||||
self.loadFinished.connect(self._loadFinished)
|
|
||||||
self.load(qurl)
|
|
||||||
self.javaScriptConsoleMessage = self._onConsoleMessage
|
|
||||||
|
|
||||||
if False:
|
|
||||||
# Run for a maximum of 10 seconds
|
|
||||||
watchdog = QTimer()
|
|
||||||
watchdog.setSingleShot(True)
|
|
||||||
watchdog.timeout.connect(lambda: self._exit(9))
|
|
||||||
watchdog.start(10000)
|
|
||||||
|
|
||||||
self.app.exec_()
|
|
||||||
|
|
||||||
def _onConsoleMessage(self, *args):
|
|
||||||
if len(args) > 3:
|
|
||||||
level, txt, lineno, filename = args
|
|
||||||
else:
|
|
||||||
level = 1
|
|
||||||
txt, lineno, filename = args
|
|
||||||
LOG.debug(f"CONSOLE {lineno} {txt} {filename}")
|
|
||||||
if "__PHANTOM_PY_DONE__" in txt:
|
|
||||||
# If we get this magic string, it means that the external JS is done
|
|
||||||
self._print()
|
|
||||||
if "__PHANTOM_PY_EXIT__" in txt:
|
|
||||||
self._exit(0)
|
|
||||||
|
|
||||||
def _loadFinished(self, result):
|
|
||||||
LOG.debug(f"phantom.py: Evaluating JS from {self.jsfile}")
|
|
||||||
self.runJavaScript("document.documentElement.contentEditable=true")
|
|
||||||
self.runJavaScript(self.js_contents)
|
|
||||||
|
|
||||||
def _printer_callback(self, *args):
|
|
||||||
"""print(self, QPrinter, Callable[[bool], None])"""
|
|
||||||
# print(f"_printer_callback {self.outfile} {args}")
|
|
||||||
if args[0] is False:
|
|
||||||
i = 1
|
|
||||||
else:
|
|
||||||
i = 0
|
|
||||||
self._exit(i)
|
|
||||||
|
|
||||||
def _print(self):
|
|
||||||
printer = QPrinter()
|
|
||||||
printer.setPageMargins(10, 10, 10, 10, QPrinter.Millimeter)
|
|
||||||
printer.setPaperSize(QPrinter.A4)
|
|
||||||
printer.setCreator("phantom.py by Michael Karl Franzl")
|
|
||||||
printer.setOutputFormat(QPrinter.PdfFormat);
|
|
||||||
printer.setOutputFileName(self.outfile);
|
|
||||||
self.print(printer, self._printer_callback)
|
|
||||||
LOG.debug("phantom.py: Printed")
|
|
||||||
|
|
||||||
def _exit(self, val):
|
|
||||||
LOG.debug(f"phantom.py: Exiting with val {val}")
|
|
||||||
|
|
||||||
# Run for a maximum of 10 seconds
|
|
||||||
watchdog = QTimer()
|
|
||||||
watchdog.setSingleShot(True)
|
|
||||||
watchdog.timeout.connect(lambda: sys.exit(val))
|
|
||||||
watchdog.start(10000)
|
|
||||||
self.app.exit(val)
|
|
||||||
atexit._clear()
|
|
||||||
sys.exit(val)
|
|
||||||
|
|
||||||
def main():
|
|
||||||
if (len(sys.argv) < 3):
|
|
||||||
LOG.info("USAGE: ./phantom.py <url> <pdf-file> [<javascript-file>]")
|
|
||||||
else:
|
|
||||||
url = sys.argv[1]
|
|
||||||
outfile = sys.argv[2]
|
|
||||||
jsfile = sys.argv[3] if len(sys.argv) > 3 else None
|
|
||||||
r = Render(url, outfile, jsfile)
|
|
||||||
|
|
||||||
sys.exit(0)
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
|
|
||||||
Loading…
Reference in New Issue
Block a user