Compare commits
10 Commits
b0a18addb3
...
48a4b37b76
Author | SHA1 | Date | |
---|---|---|---|
|
48a4b37b76 | ||
|
bdedba8d11 | ||
|
146cd71281 | ||
|
81a5e66b60 | ||
|
97db0946da | ||
|
12ff9b924e | ||
|
c6a7d839d9 | ||
|
1d92e0ec65 | ||
|
1cb4e53cce | ||
|
71672da7af |
47
.github/workflows/test.yml
vendored
Normal file
47
.github/workflows/test.yml
vendored
Normal file
@ -0,0 +1,47 @@
|
|||||||
|
name: test
|
||||||
|
|
||||||
|
on: [push]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
ci:
|
||||||
|
name: Python-${{ matrix.python }} ${{ matrix.qt.qt_api }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
qt:
|
||||||
|
- package: PyQt5
|
||||||
|
qt_api: "pyqt5"
|
||||||
|
- package: PyQt6
|
||||||
|
qt_api: "pyqt6"
|
||||||
|
- package: PySide2
|
||||||
|
qt_api: "pyside2"
|
||||||
|
- package: PySide6
|
||||||
|
qt_api: "pyside6"
|
||||||
|
python: [3.6, 3.7, 3.8, 3.9]
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v1
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v2
|
||||||
|
with:
|
||||||
|
python-version: ${{ matrix.python }}
|
||||||
|
architecture: x64
|
||||||
|
- name: Install pipenv
|
||||||
|
run: |
|
||||||
|
python -m pip install --upgrade pipenv wheel
|
||||||
|
- name: Install dependencies
|
||||||
|
run: |
|
||||||
|
pipenv install --dev
|
||||||
|
pipenv run pip install ${{ matrix.qt.package }} pytest
|
||||||
|
- name: Install Libxcb dependencies
|
||||||
|
run: |
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install '^libxcb.*-dev' libx11-xcb-dev libglu1-mesa-dev libxrender-dev libxi-dev libxkbcommon-dev libxkbcommon-x11-dev
|
||||||
|
- name: Run headless test
|
||||||
|
uses: GabrielBB/xvfb-action@v1
|
||||||
|
env:
|
||||||
|
QT_API: ${{ matrix.qt.qt_api }}
|
||||||
|
with:
|
||||||
|
run: pipenv run py.test --forked -v
|
16
Pipfile
Normal file
16
Pipfile
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
[[source]]
|
||||||
|
url = "https://pypi.org/simple"
|
||||||
|
name = "pypi"
|
||||||
|
verify_ssl = true
|
||||||
|
|
||||||
|
[dev-packages]
|
||||||
|
atomicwrites = "*"
|
||||||
|
pytest = "*"
|
||||||
|
pytest-forked = "*"
|
||||||
|
pytest-raises = "*"
|
||||||
|
|
||||||
|
[packages]
|
||||||
|
|
||||||
|
[dev-packages.phantomjs]
|
||||||
|
editable = true
|
||||||
|
path = "."
|
24
README.md
24
README.md
@ -4,8 +4,9 @@ A simple replacement for phantomjs using PyQt.
|
|||||||
|
|
||||||
This code is based on a brilliant idea of
|
This code is based on a brilliant idea of
|
||||||
[Michael Franzl](https://gist.github.com/michaelfranzl/91f0cc13c56120391b949f885643e974/raw/a0601515e7a575bc4c7d4d2a20973b29b6c6f2df/phantom.py)
|
[Michael Franzl](https://gist.github.com/michaelfranzl/91f0cc13c56120391b949f885643e974/raw/a0601515e7a575bc4c7d4d2a20973b29b6c6f2df/phantom.py)
|
||||||
that he wrote up in his
|
that he wrote up in his blog:
|
||||||
[blog](https://blog.michael.franzl.name/2017/10/16/phantom-py/index.html)
|
* https://blog.michael.franzl.name/2017/10/16/phantomjs-alternative-write-short-pyqt-scripts-instead-phantom-py/
|
||||||
|
* https://blog.michael.franzl.name/2017/10/16/phantom-py/
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
@ -26,17 +27,22 @@ way of knowing when that script has finished doing its work. For this
|
|||||||
reason, the external script should execute at the end
|
reason, the external script should execute at the end
|
||||||
```console.log("__PHANTOM_PY_DONE__");``` when done. This will trigger
|
```console.log("__PHANTOM_PY_DONE__");``` when done. This will trigger
|
||||||
the PDF generation or the file saving, after which phantompy will exit.
|
the PDF generation or the file saving, after which phantompy will exit.
|
||||||
|
If you do not want to run any javascipt file, this trigger is provided
|
||||||
|
in the code by default.
|
||||||
|
|
||||||
It is important to remember that since you're just running WebKit, you can
|
It is important to remember that since you're just running WebKit, you can
|
||||||
use everything that WebKit supports, including the usual JS client
|
use everything that WebKit supports, including the usual JS client
|
||||||
libraries, CSS, CSS @media types, etc.
|
libraries, CSS, CSS @media types, etc.
|
||||||
|
|
||||||
|
Qt picks up proxies from the environment, so this will respect
|
||||||
|
```https_proxy``` or ```http_proxy``` if set.
|
||||||
|
|
||||||
## Dependencies
|
## Dependencies
|
||||||
|
|
||||||
* Python3
|
* Python3
|
||||||
* PyQt5 (this should work with PySide2 and PyQt6 - let us know.)
|
* PyQt5 (this should work with PySide2 and PyQt6 - let us know.)
|
||||||
* [qasnyc](https://github.com/CabbageDevelopment/qasync) for the
|
* [qasnyc](https://github.com/CabbageDevelopment/qasync) for the
|
||||||
standalone program ```qasync_lookup.py```
|
standalone program ```qasync_phantompy.py```
|
||||||
|
|
||||||
## Standalone
|
## Standalone
|
||||||
|
|
||||||
@ -57,10 +63,10 @@ for the PyQt ```app.exec``` and the exiting of the program.
|
|||||||
We've decided to use the best of the shims that merge the Python
|
We've decided to use the best of the shims that merge the Python
|
||||||
```asyncio``` and Qt event loops:
|
```asyncio``` and Qt event loops:
|
||||||
[qasyc](https://github.com/CabbageDevelopment/qasync). This is seen as
|
[qasyc](https://github.com/CabbageDevelopment/qasync). This is seen as
|
||||||
the successor to the sorta abandoned[quamash](https://github.com/harvimt/quamash).
|
the successor to the sorta abandoned [quamash](https://github.com/harvimt/quamash).
|
||||||
The code is based on a
|
The code is based on a
|
||||||
[comment](https://github.com/CabbageDevelopment/qasync/issues/35#issuecomment-1315060043)
|
[comment](https://github.com/CabbageDevelopment/qasync/issues/35#issuecomment-1315060043)
|
||||||
by [Alex Marcha](https://github.com/hosaka) who's excellent code helped me.
|
by [Alex March](https://github.com/hosaka) who's excellent code helped me.
|
||||||
As this is my first use of ```asyncio``` and ```qasync``` I may have
|
As this is my first use of ```asyncio``` and ```qasync``` I may have
|
||||||
introduced some errors and it may be improved on, but it works, and
|
introduced some errors and it may be improved on, but it works, and
|
||||||
it not a monolithic Qt program, so it can be used as a library.
|
it not a monolithic Qt program, so it can be used as a library.
|
||||||
@ -73,9 +79,11 @@ The standalone program is ```quash_phantompy.py```
|
|||||||
### Arguments
|
### Arguments
|
||||||
|
|
||||||
```
|
```
|
||||||
<url> Can be a http(s) URL or a path to a local file
|
--js_input (optional) Path and name of a JavaScript file to execute on the HTML
|
||||||
<pdf-file> Path and name of PDF file to generate
|
--html_output <html-file> (optional) Path a HTML output file to generate after JS is applied
|
||||||
[<javascript-file>] (optional) Path and name of a JavaScript file to execute
|
--pdf_output <pdf-file> (optional) Path and name of PDF file to generate after JS is applied
|
||||||
|
--log_level 10=debug 20=info 30=warn 40=error
|
||||||
|
html_or_url - required argument, a http(s) URL or a path to a local file.
|
||||||
```
|
```
|
||||||
Setting ```DEBUG=1``` in the environment will give debugging messages
|
Setting ```DEBUG=1``` in the environment will give debugging messages
|
||||||
on ```stderr```.
|
on ```stderr```.
|
||||||
|
22
appveyor.yml
Normal file
22
appveyor.yml
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
environment:
|
||||||
|
matrix:
|
||||||
|
- PYTHON: "C:\\Python36"
|
||||||
|
- PYTHON: "C:\\Python37"
|
||||||
|
- PYTHON: "C:\\Python38"
|
||||||
|
- PYTHON: "C:\\Python39"
|
||||||
|
|
||||||
|
init:
|
||||||
|
- set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH%
|
||||||
|
|
||||||
|
install:
|
||||||
|
- pip install pipenv
|
||||||
|
- pipenv install --dev
|
||||||
|
- pipenv run pip install PyQt5 PySide2
|
||||||
|
# FIX: colorama not installed by pipenv
|
||||||
|
- pipenv run pip install colorama
|
||||||
|
|
||||||
|
build: off
|
||||||
|
|
||||||
|
test_script:
|
||||||
|
- set QT_API=PyQt5&& pipenv run py.test -v
|
||||||
|
- set QT_API=PySide2&& pipenv run py.test -v
|
3
phantompy/__init__.py
Normal file
3
phantompy/__init__.py
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
# -*-mode: python; indent-tabs-mode: nil; py-indent-offset: 2; coding: utf-8 -*-
|
||||||
|
|
||||||
|
__version__ = "0.1.0"
|
@ -1,10 +1,13 @@
|
|||||||
#!/usr/local/bin/python3.sh
|
#!/usr/local/bin/python3.sh
|
||||||
# -*-mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*
|
# -*-mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*
|
||||||
|
|
||||||
from qasync_phantompy import iMain
|
from __future__ import absolute_import
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from .qasync_phantompy import iMain
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from support_phantompy import vsetup_logging
|
from .support_phantompy import vsetup_logging
|
||||||
d = int(os.environ.get('DEBUG', 0))
|
d = int(os.environ.get('DEBUG', 0))
|
||||||
if d > 0:
|
if d > 0:
|
||||||
vsetup_logging(10, stream=sys.stderr)
|
vsetup_logging(10, stream=sys.stderr)
|
||||||
@ -13,4 +16,5 @@ try:
|
|||||||
vsetup_logging(log_level, logfile='', stream=sys.stderr)
|
vsetup_logging(log_level, logfile='', stream=sys.stderr)
|
||||||
except: pass
|
except: pass
|
||||||
|
|
||||||
iMain(sys.argv[1:], bgui=False)
|
if __name__ == '__main__':
|
||||||
|
iMain(sys.argv[1:])
|
84
phantompy/lookupdns.py
Normal file
84
phantompy/lookupdns.py
Normal file
@ -0,0 +1,84 @@
|
|||||||
|
#!/usr/local/bin/python3.sh
|
||||||
|
# -*-mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*
|
||||||
|
|
||||||
|
"""
|
||||||
|
Looks for urls https://dns.google/resolve?
|
||||||
|
https://dns.google/resolve?name=domain.name&type=TXT&cd=true&do=true
|
||||||
|
and parses them to extract a magic field.
|
||||||
|
|
||||||
|
A good example of how you can parse json embedded in HTML with phantomjs.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
|
||||||
|
from phantompy import Render
|
||||||
|
|
||||||
|
global LOG
|
||||||
|
import logging
|
||||||
|
import warnings
|
||||||
|
warnings.filterwarnings('ignore')
|
||||||
|
LOG = logging.getLogger()
|
||||||
|
|
||||||
|
class LookFor(Render):
|
||||||
|
|
||||||
|
def __init__(self, app, do_print=True, do_save=False):
|
||||||
|
app.lfps = []
|
||||||
|
self._app = app
|
||||||
|
self.do_print = do_print
|
||||||
|
self.do_save = do_save
|
||||||
|
self.progress = 0
|
||||||
|
self.we_run_this_tor_relay = None
|
||||||
|
Render.__init__(self, app, do_print, do_save)
|
||||||
|
|
||||||
|
def _exit(self, val):
|
||||||
|
Render._exit(self, val)
|
||||||
|
self.percent = 100
|
||||||
|
LOG.debug(f"phantom.py: Exiting with val {val}")
|
||||||
|
i = self.uri.find('name=')
|
||||||
|
fp = self.uri[i+5:]
|
||||||
|
i = fp.find('.')
|
||||||
|
fp = fp[:i]
|
||||||
|
# threadsafe?
|
||||||
|
self._app.lfps.append(fp)
|
||||||
|
|
||||||
|
def _html_callback(self, *args):
|
||||||
|
"""print(self, QPrinter, Callable[[bool], None])"""
|
||||||
|
if type(args[0]) is str:
|
||||||
|
self._save(args[0])
|
||||||
|
i = self.ilookfor(args[0])
|
||||||
|
self._onConsoleMessage(i, "__PHANTOM_PY_SAVED__", 0 , '')
|
||||||
|
|
||||||
|
def ilookfor(self, html):
|
||||||
|
import json
|
||||||
|
marker = '<pre style="word-wrap: break-word; white-space: pre-wrap;">'
|
||||||
|
if marker not in html: return -1
|
||||||
|
i = html.find(marker) + len(marker)
|
||||||
|
html = html[i:]
|
||||||
|
assert html[0] == '{', html
|
||||||
|
i = html.find('</pre')
|
||||||
|
html = html[:i]
|
||||||
|
assert html[-1] == '}', html
|
||||||
|
LOG.debug(f"Found {len(html)} json")
|
||||||
|
o = json.loads(html)
|
||||||
|
if "Answer" not in o.keys() or type(o["Answer"]) != list:
|
||||||
|
LOG.warn(f"FAIL {self.uri}")
|
||||||
|
return 1
|
||||||
|
for elt in o["Answer"]:
|
||||||
|
assert type(elt) == dict, elt
|
||||||
|
assert 'type' in elt, elt
|
||||||
|
if elt['type'] != 16: continue
|
||||||
|
assert 'data' in elt, elt
|
||||||
|
if elt['data'] == 'we-run-this-tor-relay':
|
||||||
|
LOG.info(f"OK {self.uri}")
|
||||||
|
self.we_run_this_tor_relay = True
|
||||||
|
return 0
|
||||||
|
self.we_run_this_tor_relay = False
|
||||||
|
LOG.warn(f"BAD {self.uri}")
|
||||||
|
return 2
|
||||||
|
|
||||||
|
def _loadFinished(self, result):
|
||||||
|
LOG.debug(f"phantom.py: Loading finished {self.uri}")
|
||||||
|
self.toHtml(self._html_callback)
|
||||||
|
|
@ -13,17 +13,18 @@ replacement for other bulky headless browser frameworks.
|
|||||||
|
|
||||||
If you have a display attached:
|
If you have a display attached:
|
||||||
|
|
||||||
./phantom.py <url> <pdf-file> [<javascript-file>]
|
./phantom.py [--pdf_output <pdf-file>] [--js_input <javascript-file>] <url-or-html-file>
|
||||||
|
|
||||||
If you don't have a display attached (i.e. on a remote server):
|
If you don't have a display attached (i.e. on a remote server), you can use
|
||||||
|
xvfb-run, or don't add --show_gui - it should work without a display.
|
||||||
xvfb-run ./phantom.py <url> <pdf-file> [<javascript-file>]
|
|
||||||
|
|
||||||
Arguments:
|
Arguments:
|
||||||
|
|
||||||
|
[--pdf_output <pdf-file>] (optional) Path and name of PDF file to generate
|
||||||
|
[--html_output <html-file>] (optional) Path and name of HTML file to generate
|
||||||
|
[--js_input <javascript-file>] (optional) Path and name of a JavaScript file to execute
|
||||||
|
--log_level 10=debug 20=info 30=warn 40=error
|
||||||
<url> Can be a http(s) URL or a path to a local file
|
<url> Can be a http(s) URL or a path to a local file
|
||||||
<pdf-file> Path and name of PDF file to generate
|
|
||||||
[<javascript-file>] (optional) Path and name of a JavaScript file to execute
|
|
||||||
|
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
@ -55,6 +56,9 @@ CSS @media types, etc.
|
|||||||
|
|
||||||
* Python3
|
* Python3
|
||||||
* PyQt5
|
* PyQt5
|
||||||
|
* [qasnyc](https://github.com/CabbageDevelopment/qasync) for the
|
||||||
|
standalone program ```qasnyc_phantompy.py```
|
||||||
|
|
||||||
* xvfb (optional for display-less machines)
|
* xvfb (optional for display-less machines)
|
||||||
|
|
||||||
Installation of dependencies in Debian Stretch is easy:
|
Installation of dependencies in Debian Stretch is easy:
|
||||||
@ -114,23 +118,20 @@ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|||||||
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import sys
|
import importlib
|
||||||
import os
|
import os
|
||||||
import traceback
|
import sys # noqa
|
||||||
import atexit
|
|
||||||
import time
|
|
||||||
|
|
||||||
from PyQt5.QtCore import QUrl
|
from qasync import QtModuleName
|
||||||
from PyQt5.QtCore import QTimer
|
from qasync.QtCore import QUrl
|
||||||
from PyQt5.QtWidgets import QApplication
|
|
||||||
from PyQt5.QtPrintSupport import QPrinter
|
|
||||||
from PyQt5.QtWebEngineWidgets import QWebEnginePage
|
|
||||||
|
|
||||||
from support_phantompy import vsetup_logging
|
QPrinter = importlib.import_module(QtModuleName + ".QtPrintSupport.QPrinter", package=QtModuleName)
|
||||||
|
QWebEnginePage = importlib.import_module(QtModuleName + ".QtWebEngineWidgets.QWebEnginePage", package=QtModuleName)
|
||||||
|
|
||||||
global LOG
|
global LOG
|
||||||
import logging
|
import logging
|
||||||
import warnings
|
import warnings
|
||||||
|
|
||||||
warnings.filterwarnings('ignore')
|
warnings.filterwarnings('ignore')
|
||||||
LOG = logging.getLogger()
|
LOG = logging.getLogger()
|
||||||
|
|
||||||
@ -167,28 +168,31 @@ class Render(QWebEnginePage):
|
|||||||
self.percent = 0
|
self.percent = 0
|
||||||
self.uri = None
|
self.uri = None
|
||||||
self.jsfile = None
|
self.jsfile = None
|
||||||
self.outfile = None
|
self.htmlfile = None
|
||||||
|
self.pdffile = None
|
||||||
QWebEnginePage.__init__(self)
|
QWebEnginePage.__init__(self)
|
||||||
|
|
||||||
def run(self, url, outfile, jsfile):
|
def run(self, url, pdffile, htmlfile, jsfile):
|
||||||
self._app.lstart.append(id(self))
|
self._app.lstart.append(id(self))
|
||||||
self.percent = 10
|
self.percent = 10
|
||||||
self.uri = url
|
self.uri = url
|
||||||
self.jsfile = jsfile
|
self.jsfile = jsfile
|
||||||
self.outfile = outfile
|
self.htmlfile = htmlfile
|
||||||
LOG.debug(f"phantom.py: URL={url} OUTFILE={outfile} JSFILE={jsfile}")
|
self.pdffile = pdffile
|
||||||
|
self.outfile = pdffile or htmlfile
|
||||||
|
LOG.debug(f"phantom.py: URL={url} htmlfile={htmlfile} pdffile={pdffile} JSFILE={jsfile}")
|
||||||
qurl = QUrl.fromUserInput(url)
|
qurl = QUrl.fromUserInput(url)
|
||||||
|
|
||||||
# The PDF generation only happens when the special string __PHANTOM_PY_DONE__
|
# The PDF generation only happens when the special string __PHANTOM_PY_DONE__
|
||||||
# is sent to console.log(). The following JS string will be executed by
|
# is sent to console.log(). The following JS string will be executed by
|
||||||
# default, when no external JavaScript file is specified.
|
# default, when no external JavaScript file is specified.
|
||||||
self.js_contents = "setTimeout(function() { console.log('__PHANTOM_PY_DONE__') }, 5000);";
|
self.js_contents = "setTimeout(function() { console.log('__PHANTOM_PY_DONE__') }, 5000);"
|
||||||
|
|
||||||
if jsfile:
|
if jsfile:
|
||||||
try:
|
try:
|
||||||
with open(self.jsfile, 'rt') as f:
|
with open(self.jsfile, 'rt') as f:
|
||||||
self.js_contents = f.read()
|
self.js_contents = f.read()
|
||||||
except Exception as e:
|
except Exception as e: # noqa
|
||||||
LOG.exception(f"error reading jsfile {self.jsfile}")
|
LOG.exception(f"error reading jsfile {self.jsfile}")
|
||||||
|
|
||||||
self.loadFinished.connect(self._loadFinished)
|
self.loadFinished.connect(self._loadFinished)
|
||||||
@ -223,6 +227,7 @@ class Render(QWebEnginePage):
|
|||||||
self._exit(level)
|
self._exit(level)
|
||||||
|
|
||||||
def _loadFinished(self, result):
|
def _loadFinished(self, result):
|
||||||
|
# RenderProcessTerminationStatus ?
|
||||||
self.percent = 30
|
self.percent = 30
|
||||||
LOG.info(f"phantom.py: _loadFinished {result} {self.percent}")
|
LOG.info(f"phantom.py: _loadFinished {result} {self.percent}")
|
||||||
LOG.debug(f"phantom.py: Evaluating JS from {self.jsfile}")
|
LOG.debug(f"phantom.py: Evaluating JS from {self.jsfile}")
|
||||||
@ -233,10 +238,10 @@ class Render(QWebEnginePage):
|
|||||||
"""print(self, QPrinter, Callable[[bool], None])"""
|
"""print(self, QPrinter, Callable[[bool], None])"""
|
||||||
if type(args[0]) is str:
|
if type(args[0]) is str:
|
||||||
self._save(args[0])
|
self._save(args[0])
|
||||||
self._onConsoleMessage(0, "__PHANTOM_PY_SAVED__", 0 , '')
|
self._onConsoleMessage(0, "__PHANTOM_PY_SAVED__", 0, '')
|
||||||
|
|
||||||
def _save(self, html):
|
def _save(self, html):
|
||||||
sfile = self.outfile.replace('.pdf','.html')
|
sfile = self.htmlfile
|
||||||
# CompleteHtmlSaveFormat SingleHtmlSaveFormat MimeHtmlSaveFormat
|
# CompleteHtmlSaveFormat SingleHtmlSaveFormat MimeHtmlSaveFormat
|
||||||
with open(sfile, 'wt') as ofd:
|
with open(sfile, 'wt') as ofd:
|
||||||
ofd.write(html)
|
ofd.write(html)
|
||||||
@ -244,20 +249,19 @@ class Render(QWebEnginePage):
|
|||||||
|
|
||||||
def _printer_callback(self, *args):
|
def _printer_callback(self, *args):
|
||||||
"""print(self, QPrinter, Callable[[bool], None])"""
|
"""print(self, QPrinter, Callable[[bool], None])"""
|
||||||
# print(f"_printer_callback {self.outfile} {args}")
|
|
||||||
if args[0] is False:
|
if args[0] is False:
|
||||||
i = 1
|
i = 1
|
||||||
else:
|
else:
|
||||||
i = 0
|
i = 0
|
||||||
self._onConsoleMessage(i, "__PHANTOM_PY_PRINTED__", 0 , '')
|
self._onConsoleMessage(i, "__PHANTOM_PY_PRINTED__", 0, '')
|
||||||
|
|
||||||
def _print(self):
|
def _print(self):
|
||||||
sfile = self.outfile.replace('.html', '.pdf')
|
sfile = self.pdffile
|
||||||
printer = QPrinter()
|
printer = QPrinter()
|
||||||
printer.setPageMargins(10, 10, 10, 10, QPrinter.Millimeter)
|
printer.setPageMargins(10, 10, 10, 10, QPrinter.Millimeter)
|
||||||
printer.setPaperSize(QPrinter.A4)
|
printer.setPaperSize(QPrinter.A4)
|
||||||
printer.setCreator("phantom.py by Michael Karl Franzl")
|
printer.setCreator("phantom.py by Michael Karl Franzl")
|
||||||
printer.setOutputFormat(QPrinter.PdfFormat);
|
printer.setOutputFormat(QPrinter.PdfFormat)
|
||||||
printer.setOutputFileName(sfile)
|
printer.setOutputFileName(sfile)
|
||||||
self.print(printer, self._printer_callback)
|
self.print(printer, self._printer_callback)
|
||||||
LOG.debug("phantom.py: Printed")
|
LOG.debug("phantom.py: Printed")
|
||||||
@ -267,26 +271,3 @@ class Render(QWebEnginePage):
|
|||||||
LOG.debug(f"phantom.py: Exiting with val {val}")
|
LOG.debug(f"phantom.py: Exiting with val {val}")
|
||||||
# threadsafe?
|
# threadsafe?
|
||||||
self._app.ldone.append(self.uri)
|
self._app.ldone.append(self.uri)
|
||||||
|
|
||||||
def omain(app, largs):
|
|
||||||
if (len(largs) < 2):
|
|
||||||
LOG.info("USAGE: ./phantom.py <url> <pdf-file> [<javascript-file>]")
|
|
||||||
return -1
|
|
||||||
|
|
||||||
url = largs[0]
|
|
||||||
outfile = largs[1]
|
|
||||||
jsfile = largs[2] if len(largs) > 2 else None
|
|
||||||
ilen = 1
|
|
||||||
|
|
||||||
r = Render(app, do_print=False, do_save=True)
|
|
||||||
r.run(url, outfile, jsfile)
|
|
||||||
for i in range(1, 120):
|
|
||||||
app.processEvents()
|
|
||||||
print(f"{app.ldone} {i}")
|
|
||||||
if len(app.ldone) == ilen:
|
|
||||||
print(f"{app.ldone} found {ilen}")
|
|
||||||
app.exit()
|
|
||||||
return r
|
|
||||||
time.sleep(1)
|
|
||||||
return r
|
|
||||||
|
|
@ -1,25 +1,30 @@
|
|||||||
#!/usr/local/bin/python3.sh
|
#!/usr/local/bin/python3.sh
|
||||||
# -*-mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*
|
# -*-mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*
|
||||||
|
|
||||||
import sys
|
|
||||||
import os
|
|
||||||
import qasync
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import time
|
import os
|
||||||
import random
|
import sys
|
||||||
|
|
||||||
from PyQt5 import QtWidgets
|
# let qasync figure out what Qt we are using - we dont care
|
||||||
from PyQt5.QtWidgets import (QProgressBar, QWidget, QVBoxLayout)
|
from qasync import QApplication, QEventLoop, QtWidgets
|
||||||
|
|
||||||
from phantompy import Render
|
from phantompy import Render
|
||||||
|
# if you want an example of looking for things in downloaded HTML:
|
||||||
# from lookupdns import LookFor as Render
|
# from lookupdns import LookFor as Render
|
||||||
|
from support_phantompy import omain_argparser, vsetup_logging
|
||||||
|
|
||||||
global LOG
|
global LOG
|
||||||
import logging
|
import logging
|
||||||
import warnings
|
import warnings
|
||||||
|
|
||||||
warnings.filterwarnings('ignore')
|
warnings.filterwarnings('ignore')
|
||||||
LOG = logging.getLogger()
|
LOG = logging.getLogger()
|
||||||
|
|
||||||
|
try:
|
||||||
|
import shtab
|
||||||
|
except:
|
||||||
|
shtab = None
|
||||||
|
|
||||||
class Widget(QtWidgets.QWidget):
|
class Widget(QtWidgets.QWidget):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
QtWidgets.QWidget.__init__(self)
|
QtWidgets.QWidget.__init__(self)
|
||||||
@ -27,7 +32,7 @@ class Widget(QtWidgets.QWidget):
|
|||||||
box = QtWidgets.QHBoxLayout()
|
box = QtWidgets.QHBoxLayout()
|
||||||
self.setLayout(box)
|
self.setLayout(box)
|
||||||
box.addWidget(self._label)
|
box.addWidget(self._label)
|
||||||
self.progress = QProgressBar()
|
self.progress = QtWidgets.QProgressBar()
|
||||||
self.progress.setRange(0, 99)
|
self.progress.setRange(0, 99)
|
||||||
box.addWidget(self.progress)
|
box.addWidget(self.progress)
|
||||||
|
|
||||||
@ -37,13 +42,17 @@ class Widget(QtWidgets.QWidget):
|
|||||||
self.progress.setValue(int(text))
|
self.progress.setValue(int(text))
|
||||||
|
|
||||||
class ContextManager:
|
class ContextManager:
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
self._seconds = 0
|
self._seconds = 0
|
||||||
|
|
||||||
async def __aenter__(self):
|
async def __aenter__(self):
|
||||||
LOG.debug("ContextManager enter")
|
LOG.debug("ContextManager enter")
|
||||||
return self
|
return self
|
||||||
|
|
||||||
async def __aexit__(self, *args):
|
async def __aexit__(self, *args):
|
||||||
LOG.debug("ContextManager exit")
|
LOG.debug("ContextManager exit")
|
||||||
|
|
||||||
async def tick(self):
|
async def tick(self):
|
||||||
await asyncio.sleep(1)
|
await asyncio.sleep(1)
|
||||||
self._seconds += 1
|
self._seconds += 1
|
||||||
@ -63,12 +72,25 @@ async def main(widget, app, ilen):
|
|||||||
app.exit()
|
app.exit()
|
||||||
# raise asyncio.CancelledError
|
# raise asyncio.CancelledError
|
||||||
return
|
return
|
||||||
LOG.debug(f"{app.ldone} {perc} {seconds}")
|
LOG.debug(f"{app.ldone} {seconds}")
|
||||||
except asyncio.CancelledError as ex:
|
except asyncio.CancelledError as ex: # noqa
|
||||||
LOG.debug("Task cancelled")
|
LOG.debug("Task cancelled")
|
||||||
|
|
||||||
def iMain(largs, bgui=True):
|
def iMain(largs):
|
||||||
app = QtWidgets.QApplication([])
|
parser = omain_argparser()
|
||||||
|
if shtab:
|
||||||
|
shtab.add_argument_to(parser, ["-s", "--print-completion"]) # magic!
|
||||||
|
oargs = parser.parse_args(largs)
|
||||||
|
bgui = oargs.show_gui
|
||||||
|
|
||||||
|
try:
|
||||||
|
d = int(os.environ.get('DEBUG', 0))
|
||||||
|
if d > 0:
|
||||||
|
oargs.log_level = 10
|
||||||
|
vsetup_logging(oargs.log_level, logfile='', stream=sys.stderr)
|
||||||
|
except: pass
|
||||||
|
|
||||||
|
app = QApplication([])
|
||||||
app.lstart = []
|
app.lstart = []
|
||||||
if bgui:
|
if bgui:
|
||||||
widget = Widget()
|
widget = Widget()
|
||||||
@ -77,17 +99,19 @@ def iMain(largs, bgui=True):
|
|||||||
else:
|
else:
|
||||||
widget = None
|
widget = None
|
||||||
|
|
||||||
loop = qasync.QEventLoop(app)
|
loop = QEventLoop(app)
|
||||||
asyncio.set_event_loop(loop)
|
asyncio.set_event_loop(loop)
|
||||||
|
|
||||||
largs = sys.argv[1:]
|
url = oargs.html_url
|
||||||
url = largs[0]
|
htmlfile = oargs.html_output
|
||||||
outfile = largs[1]
|
pdffile = oargs.html_output
|
||||||
jsfile = largs[2] if len(largs) > 2 else None
|
jsfile = oargs.js_input
|
||||||
# run only starts the url loading
|
# run only starts the url loading
|
||||||
r = Render(app, do_print=False, do_save=True)
|
r = Render(app,
|
||||||
|
do_print=True if pdffile else False,
|
||||||
|
do_save=True if htmlfile else False)
|
||||||
uri = url.strip()
|
uri = url.strip()
|
||||||
r.run(uri, outfile, jsfile)
|
r.run(uri, pdffile, htmlfile, jsfile)
|
||||||
LOG.debug(f"{r.percent} {app.lstart}")
|
LOG.debug(f"{r.percent} {app.lstart}")
|
||||||
|
|
||||||
LOG.info(f"queued {len(app.lstart)} urls")
|
LOG.info(f"queued {len(app.lstart)} urls")
|
||||||
@ -101,15 +125,4 @@ def iMain(largs, bgui=True):
|
|||||||
loop.run_until_complete(asyncio.gather(*tasks))
|
loop.run_until_complete(asyncio.gather(*tasks))
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
try:
|
iMain(sys.argv[1:])
|
||||||
from exclude_badExits import vsetup_logging
|
|
||||||
d = int(os.environ.get('DEBUG', 0))
|
|
||||||
if d > 0:
|
|
||||||
vsetup_logging(10, stream=sys.stderr)
|
|
||||||
else:
|
|
||||||
vsetup_logging(20, stream=sys.stderr)
|
|
||||||
vsetup_logging(log_level, logfile='', stream=sys.stderr)
|
|
||||||
except: pass
|
|
||||||
|
|
||||||
iMain(sys.argv[1:], bgui=False)
|
|
||||||
|
|
@ -1,20 +1,22 @@
|
|||||||
#!/usr/local/bin/python3.sh
|
#!/usr/local/bin/python3.sh
|
||||||
# -*-mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*
|
# -*-mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*
|
||||||
|
|
||||||
import sys
|
import argparse
|
||||||
import os
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if 'COLOREDLOGS_LEVEL_STYLES' not in os.environ:
|
if 'COLOREDLOGS_LEVEL_STYLES' not in os.environ:
|
||||||
os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red'
|
os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red'
|
||||||
# https://pypi.org/project/coloredlogs/
|
# https://pypi.org/project/coloredlogs/
|
||||||
import coloredlogs
|
import coloredlogs
|
||||||
except ImportError as e:
|
except ImportError:
|
||||||
coloredlogs = False
|
coloredlogs = False
|
||||||
|
|
||||||
global LOG
|
global LOG
|
||||||
import logging
|
import logging
|
||||||
import warnings
|
import warnings
|
||||||
|
|
||||||
warnings.filterwarnings('ignore')
|
warnings.filterwarnings('ignore')
|
||||||
LOG = logging.getLogger()
|
LOG = logging.getLogger()
|
||||||
|
|
||||||
@ -23,7 +25,7 @@ def vsetup_logging(log_level, logfile='', stream=sys.stdout):
|
|||||||
add = True
|
add = True
|
||||||
|
|
||||||
# stem fucks up logging
|
# stem fucks up logging
|
||||||
from stem.util import log
|
# from stem.util import log
|
||||||
logging.getLogger('stem').setLevel(30)
|
logging.getLogger('stem').setLevel(30)
|
||||||
|
|
||||||
logging._defaultFormatter = logging.Formatter(datefmt='%m-%d %H:%M:%S')
|
logging._defaultFormatter = logging.Formatter(datefmt='%m-%d %H:%M:%S')
|
||||||
@ -79,3 +81,37 @@ def vsetup_logging(log_level, logfile='', stream=sys.stdout):
|
|||||||
'NOTSET': logging.NOTSET,
|
'NOTSET': logging.NOTSET,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def omain_argparser(_=None):
|
||||||
|
|
||||||
|
try:
|
||||||
|
from OpenSSL import SSL
|
||||||
|
lCAfs = SSL._CERTIFICATE_FILE_LOCATIONS
|
||||||
|
except:
|
||||||
|
lCAfs = []
|
||||||
|
|
||||||
|
CAfs = []
|
||||||
|
for elt in lCAfs:
|
||||||
|
if os.path.exists(elt):
|
||||||
|
CAfs.append(elt)
|
||||||
|
if not CAfs:
|
||||||
|
CAfs = ['']
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(add_help=True,
|
||||||
|
epilog=__doc__)
|
||||||
|
parser.add_argument('--https_cafile', type=str,
|
||||||
|
help="Certificate Authority file (in PEM) (unused)",
|
||||||
|
default=CAfs[0])
|
||||||
|
parser.add_argument('--log_level', type=int, default=20,
|
||||||
|
help="10=debug 20=info 30=warn 40=error")
|
||||||
|
parser.add_argument('--js_input', type=str, default='',
|
||||||
|
help="Operate on the HTML file with javascript")
|
||||||
|
parser.add_argument('--html_output', type=str, default='',
|
||||||
|
help="Write loaded and javascripted result to a HTML file")
|
||||||
|
parser.add_argument('--pdf_output', type=str, default='',
|
||||||
|
help="Write loaded and javascripted result to a PDF file")
|
||||||
|
parser.add_argument('--show_gui', type=bool, default=False, store_action=True,
|
||||||
|
help="show a progress meter that doesn't work")
|
||||||
|
parser.add_argument('html_url', type=str, nargs='?',
|
||||||
|
required=True,
|
||||||
|
help='html file or url')
|
||||||
|
return parser
|
58
setup.cfg
Normal file
58
setup.cfg
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
[metadata]
|
||||||
|
classifiers =
|
||||||
|
License :: OSI Approved
|
||||||
|
License :: OSI Approved :: BSD 1-clause
|
||||||
|
Intended Audience :: Web Developers
|
||||||
|
Operating System :: Microsoft :: Windows
|
||||||
|
Operating System :: POSIX :: BSD :: FreeBSD
|
||||||
|
Operating System :: POSIX :: Linux
|
||||||
|
Programming Language :: Python :: 3 :: Only
|
||||||
|
Programming Language :: Python :: 3.6
|
||||||
|
Programming Language :: Python :: 3.7
|
||||||
|
Programming Language :: Python :: 3.8
|
||||||
|
Programming Language :: Python :: 3.9
|
||||||
|
Programming Language :: Python :: Implementation :: CPython
|
||||||
|
Framework :: AsyncIO
|
||||||
|
|
||||||
|
[options]
|
||||||
|
zip_safe = false
|
||||||
|
python_requires = ~=3.6
|
||||||
|
packages = find:
|
||||||
|
include_package_data = false
|
||||||
|
install_requires =
|
||||||
|
qasync
|
||||||
|
cryptography
|
||||||
|
rsa
|
||||||
|
stem
|
||||||
|
|
||||||
|
[options.entry_points]
|
||||||
|
console_scripts =
|
||||||
|
phantompy = phantompy.__main__:iMain
|
||||||
|
|
||||||
|
[easy_install]
|
||||||
|
zip_ok = false
|
||||||
|
|
||||||
|
[flake8]
|
||||||
|
jobs = 1
|
||||||
|
max-line-length = 88
|
||||||
|
ignore =
|
||||||
|
E111
|
||||||
|
E114
|
||||||
|
E128
|
||||||
|
E225
|
||||||
|
E225
|
||||||
|
E261
|
||||||
|
E302
|
||||||
|
E305
|
||||||
|
E402
|
||||||
|
E501
|
||||||
|
E502
|
||||||
|
E541
|
||||||
|
E701
|
||||||
|
E704
|
||||||
|
E722
|
||||||
|
E741
|
||||||
|
F508
|
||||||
|
F541
|
||||||
|
W503
|
||||||
|
|
44
setup.py
Normal file
44
setup.py
Normal file
@ -0,0 +1,44 @@
|
|||||||
|
# -*-mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*
|
||||||
|
|
||||||
|
import re
|
||||||
|
from setuptools import setup
|
||||||
|
|
||||||
|
with open("qasync/__init__.py") as f:
|
||||||
|
version = re.search(r'__version__\s+=\s+"(.*)"', f.read()).group(1)
|
||||||
|
|
||||||
|
long_description = "\n\n".join([
|
||||||
|
open("README.md").read(),
|
||||||
|
])
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
setup(
|
||||||
|
name="phantompy",
|
||||||
|
version=__version__,
|
||||||
|
description="""A simple replacement for phantomjs using PyQt""",
|
||||||
|
long_description=long_description,
|
||||||
|
author="Michael Franzl (originally)",
|
||||||
|
author_email='',
|
||||||
|
license="1clause BSD",
|
||||||
|
packages=['phantompy'],
|
||||||
|
# url="",
|
||||||
|
# download_url="https://",
|
||||||
|
keywords=['JavaScript', 'phantomjs', 'asyncio'],
|
||||||
|
# maybe less - nothing fancy
|
||||||
|
python_requires="~=3.6",
|
||||||
|
# probably works on PyQt6 and PySide2 but untested
|
||||||
|
# https://github.com/CabbageDevelopment/qasync/
|
||||||
|
install_requires=['qasync',
|
||||||
|
'PyQt5'],
|
||||||
|
entry_points={
|
||||||
|
'console_scripts': ['phantompy = phantompy.__main__:iMain', ]},
|
||||||
|
classifiers=[
|
||||||
|
'Development Status :: 4 - Beta',
|
||||||
|
'Environment :: Console',
|
||||||
|
'Intended Audience :: Developers',
|
||||||
|
'Intended Audience :: Web Developers',
|
||||||
|
'Natural Language :: English',
|
||||||
|
'Operating System :: OS Independent',
|
||||||
|
'Programming Language :: Python :: 3',
|
||||||
|
'Topic :: Software Development :: Documentation',
|
||||||
|
],
|
||||||
|
)
|
22
tests/conftest.py
Normal file
22
tests/conftest.py
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*-
|
||||||
|
|
||||||
|
# (c) 2018 Gerard Marull-Paretas <gerard@teslabs.com>
|
||||||
|
# (c) 2014 Mark Harviston <mark.harviston@gmail.com>
|
||||||
|
# (c) 2014 Arve Knudsen <arve.knudsen@gmail.com>
|
||||||
|
# BSD License
|
||||||
|
|
||||||
|
# phantompy test - just test qasync for now
|
||||||
|
|
||||||
|
import os
|
||||||
|
import logging
|
||||||
|
from pytest import fixture
|
||||||
|
|
||||||
|
logging.basicConfig(
|
||||||
|
level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"
|
||||||
|
)
|
||||||
|
|
||||||
|
@fixture(scope="session")
|
||||||
|
def application():
|
||||||
|
from phantompy.qasync_phantompy import QApplication
|
||||||
|
|
||||||
|
return QApplication([])
|
Loading…
x
Reference in New Issue
Block a user