diff --git a/README.rst b/README.rst index b9151c9..94f3262 100644 --- a/README.rst +++ b/README.rst @@ -70,6 +70,26 @@ must be specified, everything else is optional. scope: surt: http://(org,example, +Brozzler Web Console +-------------------- + +Brozzler comes with a rudimentary web application for viewing crawl job status. +To install the brozzler with dependencies required to run this app, run + +:: + + pip install brozzler[webconsole] + + +To start the app, run + +:: + + brozzler-webconsole + + +XXX configuration stuff + Fonts (for decent screenshots) ------------------------------ diff --git a/webconsole/brozzler-webconsole/__init__.py b/brozzler/webconsole/__init__.py similarity index 80% rename from webconsole/brozzler-webconsole/__init__.py rename to brozzler/webconsole/__init__.py index cc6d90b..93c5e5c 100644 --- a/webconsole/brozzler-webconsole/__init__.py +++ b/brozzler/webconsole/__init__.py @@ -17,14 +17,22 @@ See the License for the specific language governing permissions and limitations under the License. ''' -import flask +import logging +import sys +try: + import flask +except ImportError as e: + logging.critical( + '%s: %s\n\nYou might need to run "pip install ' + 'brozzler[webconsole]".\nSee README.rst for more information.', + type(e).__name__, e) + sys.exit(1) + import rethinkstuff import json -import sys import os import importlib import rethinkdb -import logging import yaml # flask does its own logging config @@ -157,6 +165,39 @@ def api404(path): def root(path): return flask.render_template("index.html") -if __name__ == "__main__": - app.run(host="0.0.0.0", port=8081, debug=True) +import pdb; pdb.set_trace() +try: + import gunicorn.app.base + from gunicorn.six import iteritems + + class GunicornBrozzlerWebConsole(gunicorn.app.base.BaseApplication): + def __init__(self, app, options=None): + self.options = options or {} + self.application = app + super(GunicornBrozzlerWebConsole, self).__init__() + + def load_config(self): + config = dict( + [(key, value) for key, value in iteritems(self.options) + if key in self.cfg.settings and value is not None]) + for key, value in iteritems(config): + self.cfg.set(key.lower(), value) + + def load(self): + return self.application + + def run(**options): + import pdb; pdb.set_trace() + logging.info('running brozzler-webconsole using gunicorn') + GunicornBrozzlerWebConsole(app, options).run() + +except ImportError: + def run(): + import pdb; pdb.set_trace() + logging.info('running brozzler-webconsole using simple flask app.run') + app.run() + +if __name__ == "__main__": + # arguments? + run() diff --git a/webconsole/brozzler-webconsole/static/brozzler.svg b/brozzler/webconsole/static/brozzler.svg similarity index 100% rename from webconsole/brozzler-webconsole/static/brozzler.svg rename to brozzler/webconsole/static/brozzler.svg diff --git a/webconsole/brozzler-webconsole/static/js/app.js b/brozzler/webconsole/static/js/app.js similarity index 100% rename from webconsole/brozzler-webconsole/static/js/app.js rename to brozzler/webconsole/static/js/app.js diff --git a/webconsole/brozzler-webconsole/static/noVNC b/brozzler/webconsole/static/noVNC similarity index 100% rename from webconsole/brozzler-webconsole/static/noVNC rename to brozzler/webconsole/static/noVNC diff --git a/webconsole/brozzler-webconsole/static/partials/home.html b/brozzler/webconsole/static/partials/home.html similarity index 100% rename from webconsole/brozzler-webconsole/static/partials/home.html rename to brozzler/webconsole/static/partials/home.html diff --git a/webconsole/brozzler-webconsole/static/partials/job.html b/brozzler/webconsole/static/partials/job.html similarity index 100% rename from webconsole/brozzler-webconsole/static/partials/job.html rename to brozzler/webconsole/static/partials/job.html diff --git a/webconsole/brozzler-webconsole/static/partials/site.html b/brozzler/webconsole/static/partials/site.html similarity index 100% rename from webconsole/brozzler-webconsole/static/partials/site.html rename to brozzler/webconsole/static/partials/site.html diff --git a/webconsole/brozzler-webconsole/static/partials/workers.html b/brozzler/webconsole/static/partials/workers.html similarity index 100% rename from webconsole/brozzler-webconsole/static/partials/workers.html rename to brozzler/webconsole/static/partials/workers.html diff --git a/webconsole/brozzler-webconsole/templates/index.html b/brozzler/webconsole/templates/index.html similarity index 100% rename from webconsole/brozzler-webconsole/templates/index.html rename to brozzler/webconsole/templates/index.html diff --git a/setup.py b/setup.py index 7f6255c..3f0a09e 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ import glob setuptools.setup( name='brozzler', - version='1.1.dev20', + version='1.1.dev21', description='Distributed web crawling with browsers', url='https://github.com/internetarchive/brozzler', author='Noah Levitt', @@ -31,6 +31,11 @@ setuptools.setup( packages=['brozzler'], package_data={'brozzler': ['behaviors.d/*.js*', 'behaviors.yaml']}, scripts=glob.glob('bin/*'), + entry_points={ + 'console_scripts': [ + 'brozzler-webconsole = brozzler.webconsole:run', + ], + }, install_requires=[ 'PyYAML', 'youtube-dl', @@ -38,11 +43,15 @@ setuptools.setup( 'requests', 'websocket-client', 'pillow', - 'surt>=0.3b2', - 'rethinkstuff', + 'surt>=0.3.0', + 'rethinkstuff>=0.1.5', 'rethinkdb>=2.3,<2.4', 'psutil', ], + extras_require={ + 'webconsole': ['flask>=0.11', 'gunicorn'], + # 'brozzler-easy': ['warcprox', 'pywb'], + }, zip_safe=False, classifiers=[ 'Development Status :: 4 - Beta', diff --git a/webconsole/README.rst b/webconsole/README.rst deleted file mode 100644 index 659fbfd..0000000 --- a/webconsole/README.rst +++ /dev/null @@ -1 +0,0 @@ -gunicorn --bind=0.0.0.0:8081 brozzler-webconsole:app diff --git a/webconsole/requirements.txt b/webconsole/requirements.txt deleted file mode 100644 index 1255527..0000000 --- a/webconsole/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -rethinkstuff>=0.1.5 -flask>=0.11 -gunicorn -PyYAML