make brozzler-webconsole a part of the main brozzler package, using optional "extras_require" dependencies

README.rst

@@ -70,6 +70,26 @@ must be specified, everything else is optional.
         scope:
           surt: http://(org,example,
 
+Brozzler Web Console
+--------------------
+
+Brozzler comes with a rudimentary web application for viewing crawl job status.
+To install the brozzler with dependencies required to run this app, run
+
+::
+
+    pip install brozzler[webconsole]
+
+
+To start the app, run
+
+::
+
+    brozzler-webconsole
+
+
+XXX configuration stuff
+
 Fonts (for decent screenshots)
 ------------------------------
 

brozzler/webconsole/__init__.py

@@ -17,14 +17,22 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 
-import flask
+import logging
+import sys
+try:
+    import flask
+except ImportError as e:
+    logging.critical(
+            '%s: %s\n\nYou might need to run "pip install '
+            'brozzler[webconsole]".\nSee README.rst for more information.',
+            type(e).__name__, e)
+    sys.exit(1)
+
 import rethinkstuff
 import json
-import sys
 import os
 import importlib
 import rethinkdb
-import logging
 import yaml
 
 # flask does its own logging config
@@ -157,6 +165,39 @@ def api404(path):
 def root(path):
     return flask.render_template("index.html")
 
-if __name__ == "__main__":
+import pdb; pdb.set_trace()
-    app.run(host="0.0.0.0", port=8081, debug=True)
+try:
+    import gunicorn.app.base
+    from gunicorn.six import iteritems
+
+    class GunicornBrozzlerWebConsole(gunicorn.app.base.BaseApplication):
+        def __init__(self, app, options=None):
+            self.options = options or {}
+            self.application = app
+            super(GunicornBrozzlerWebConsole, self).__init__()
+
+        def load_config(self):
+            config = dict(
+                    [(key, value) for key, value in iteritems(self.options)
+                        if key in self.cfg.settings and value is not None])
+            for key, value in iteritems(config):
+                self.cfg.set(key.lower(), value)
+
+        def load(self):
+            return self.application
+
+    def run(**options):
+        import pdb; pdb.set_trace()
+        logging.info('running brozzler-webconsole using gunicorn')
+        GunicornBrozzlerWebConsole(app, options).run()
+
+except ImportError:
+    def run():
+        import pdb; pdb.set_trace()
+        logging.info('running brozzler-webconsole using simple flask app.run')
+        app.run()
+
+if __name__ == "__main__":
+    # arguments?
+    run()
 

setup.py

@@ -21,7 +21,7 @@ import glob
 
 setuptools.setup(
         name='brozzler',
-        version='1.1.dev20',
+        version='1.1.dev21',
         description='Distributed web crawling with browsers',
         url='https://github.com/internetarchive/brozzler',
         author='Noah Levitt',
@@ -31,6 +31,11 @@ setuptools.setup(
         packages=['brozzler'],
         package_data={'brozzler': ['behaviors.d/*.js*', 'behaviors.yaml']},
         scripts=glob.glob('bin/*'),
+        entry_points={
+            'console_scripts': [
+                'brozzler-webconsole = brozzler.webconsole:run',
+            ],
+        },
         install_requires=[
             'PyYAML',
             'youtube-dl',
@@ -38,11 +43,15 @@ setuptools.setup(
             'requests',
             'websocket-client',
             'pillow',
-            'surt>=0.3b2',
+            'surt>=0.3.0',
-            'rethinkstuff',
+            'rethinkstuff>=0.1.5',
             'rethinkdb>=2.3,<2.4',
             'psutil',
         ],
+        extras_require={
+            'webconsole': ['flask>=0.11', 'gunicorn'],
+            # 'brozzler-easy': ['warcprox', 'pywb'],
+        },
         zip_safe=False,
         classifiers=[
             'Development Status :: 4 - Beta',

webconsole/README.rst

@@ -1 +0,0 @@
-gunicorn --bind=0.0.0.0:8081 brozzler-webconsole:app

webconsole/requirements.txt

@@ -1,4 +0,0 @@
-rethinkstuff>=0.1.5
-flask>=0.11
-gunicorn
-PyYAML