diff --git a/.gitmodules b/.gitmodules index fb40a46..88c87e5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "webconsole/static/noVNC"] - path = webconsole/static/noVNC +[submodule "webconsole/brozzler-webconsole/static/noVNC"] + path = webconsole/brozzler-webconsole/static/noVNC url = https://github.com/kanaka/noVNC.git diff --git a/brozzler/job.py b/brozzler/job.py index 4f67232..dd329e0 100644 --- a/brozzler/job.py +++ b/brozzler/job.py @@ -88,3 +88,6 @@ class Job(brozzler.BaseDictable): self.finished = finished self.stop_requested = stop_requested + def __str__(self): + return 'Job(id=%s)' % self.id + diff --git a/webconsole/brozzler-webconsole.py b/webconsole/brozzler-webconsole/__init__.py similarity index 67% rename from webconsole/brozzler-webconsole.py rename to webconsole/brozzler-webconsole/__init__.py index 89b4ede..b5d2f0c 100644 --- a/webconsole/brozzler-webconsole.py +++ b/webconsole/brozzler-webconsole/__init__.py @@ -3,19 +3,30 @@ import rethinkstuff import json import logging import sys +import os logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s") app = flask.Flask(__name__) -r = rethinkstuff.Rethinker(["wbgrp-svc020", "wbgrp-svc035", "wbgrp-svc036"], - db="archiveit_brozzler") +# configure with environment variables +SETTINGS= { + 'RETHINKDB_SERVERS': os.environ.get( + 'RETHINKDB_SERVERS', 'localhost').split(','), + 'RETHINKDB_DB': os.environ.get('RETHINKDB_DB', 'brozzler'), + 'WAYBACK_BASEURL': os.environ.get( + 'WAYBACK_BASEURL', 'http://wbgrp-svc107.us.archive.org:8091'), +} +r = rethinkstuff.Rethinker( + SETTINGS['RETHINKDB_SERVERS'], db=SETTINGS['RETHINKDB_DB']) @app.route("/api/sites//queued_count") @app.route("/api/site//queued_count") def queued_count(site_id): - count = r.table("pages").between([site_id, 0, False, r.minval], [site_id, 0, False, r.maxval], index="priority_by_site").count().run() + count = r.table("pages").between( + [site_id, 0, False, r.minval], [site_id, 0, False, r.maxval], + index="priority_by_site").count().run() return flask.jsonify(count=count) @app.route("/api/sites//queue") @@ -24,7 +35,9 @@ def queue(site_id): logging.info("flask.request.args=%s", flask.request.args) start = flask.request.args.get("start", 0) end = flask.request.args.get("end", start + 90) - queue_ = r.table("pages").between([site_id, 0, False, r.minval], [site_id, 0, False, r.maxval], index="priority_by_site")[start:end].run() + queue_ = r.table("pages").between( + [site_id, 0, False, r.minval], [site_id, 0, False, r.maxval], + index="priority_by_site")[start:end].run() return flask.jsonify(queue_=list(queue_)) @app.route("/api/sites//pages_count") @@ -32,7 +45,10 @@ def queue(site_id): @app.route("/api/sites//page_count") @app.route("/api/site//page_count") def page_count(site_id): - count = r.table("pages").between([site_id, 1, False, r.minval], [site_id, r.maxval, False, r.maxval], index="priority_by_site").count().run() + count = r.table("pages").between( + [site_id, 1, False, r.minval], + [site_id, r.maxval, False, r.maxval], + index="priority_by_site").count().run() return flask.jsonify(count=count) @app.route("/api/sites//pages") @@ -42,7 +58,10 @@ def pages(site_id): logging.info("flask.request.args=%s", flask.request.args) start = int(flask.request.args.get("start", 0)) end = int(flask.request.args.get("end", start + 90)) - pages_ = r.table("pages").between([site_id, 1, False, r.minval], [site_id, r.maxval, False, r.maxval], index="priority_by_site")[start:end].run() + pages_ = r.table("pages").between( + [site_id, 1, False, r.minval], + [site_id, r.maxval, False, r.maxval], + index="priority_by_site")[start:end].run() return flask.jsonify(pages=list(pages_)) @app.route("/api/sites/") @@ -81,8 +100,13 @@ def services(): @app.route("/api/jobs") def jobs(): jobs_ = list(r.table("jobs").run()) + jobs_ = sorted(jobs_, key=lambda j: j['id'], reverse=True) return flask.jsonify(jobs=jobs_) +@app.route("/api/config") +def config(): + return flask.jsonify(config=SETTINGS) + @app.route("/api/") @app.route("/api", defaults={"path":""}) def api404(path): @@ -91,7 +115,7 @@ def api404(path): @app.route("/", defaults={"path": ""}) @app.route("/") def root(path): - return app.send_static_file("index.html") + return flask.render_template("index.html") if __name__ == "__main__": app.run(host="0.0.0.0", port=8081, debug=True) diff --git a/webconsole/static/brozzler.svg b/webconsole/brozzler-webconsole/static/brozzler.svg similarity index 100% rename from webconsole/static/brozzler.svg rename to webconsole/brozzler-webconsole/static/brozzler.svg diff --git a/webconsole/static/js/app.js b/webconsole/brozzler-webconsole/static/js/app.js similarity index 87% rename from webconsole/static/js/app.js rename to webconsole/brozzler-webconsole/static/js/app.js index 74add56..d81370f 100644 --- a/webconsole/static/js/app.js +++ b/webconsole/brozzler-webconsole/static/js/app.js @@ -52,16 +52,22 @@ var brozzlerControllers = angular.module("brozzlerControllers", []); brozzlerControllers.controller("HomeController", ["$scope", "$http", function($scope, $http) { - $http.get("/api/jobs").success(function(data) { - $scope.jobs = data.jobs; - }); - $http.get("/api/services").success(function(data) { - $scope.services = data.services; - }); + $http.get("/api/config").success(function(data) { + $scope.config = data.config; + }); + $http.get("/api/jobs").success(function(data) { + $scope.jobs = data.jobs; + }); + $http.get("/api/services").success(function(data) { + $scope.services = data.services; + }); }]); brozzlerControllers.controller("WorkersListController", ["$scope", "$http", function($scope, $http) { + $http.get("/api/config").success(function(data) { + $scope.config = data.config; + }); $http.get("/api/workers").success(function(data) { $scope.workers = data.workers; }); @@ -80,9 +86,9 @@ function pageCountSuccessCallback(site, job) { // console.log("site = ", site); // console.log("/api/sites/" + site.id + "/page_count = ", data); site.page_count = data.count; - if (job) { - job.page_count += data.count; - } + if (job) { + job.page_count += data.count; + } } } @@ -91,9 +97,9 @@ function queuedCountSuccessCallback(site, job) { // console.log("site = ", site); // console.log("/api/sites/" + site.id + "/queued_count = ", data); site.queued_count = data.count; - if (job) { - job.queued_count += data.count; - } + if (job) { + job.queued_count += data.count; + } } } @@ -114,6 +120,10 @@ function loadSiteStats($http, site, job) { brozzlerControllers.controller("JobController", ["$scope", "$routeParams", "$http", function($scope, $routeParams, $http) { + console.log('JobController'); + $http.get("/api/config").success(function(data) { + $scope.config = data.config; + }); $http.get("/api/jobs/" + $routeParams.id).success(function(data) { $scope.job = data; $scope.job.page_count = $scope.job.queued_count = 0; @@ -127,11 +137,10 @@ brozzlerControllers.controller("JobController", ["$scope", "$routeParams", "$htt $scope.sites = data.sites; // console.log("sites=", $scope.sites); for (var i = 0; i < $scope.sites.length; i++) { - loadSiteStats($http, $scope.sites[i], $scope.job); + loadSiteStats($http, $scope.sites[i], $scope.job); } }); }); - }]); brozzlerControllers.controller("SiteController", ["$scope", "$routeParams", "$http", "$window", @@ -163,6 +172,9 @@ brozzlerControllers.controller("SiteController", ["$scope", "$routeParams", "$ht }; + $http.get("/api/config").success(function(data) { + $scope.config = data.config; + }); $http.get("/api/site/" + $routeParams.id).success(function(data) { $scope.site = data; loadSiteStats($http, $scope.site); diff --git a/webconsole/static/noVNC b/webconsole/brozzler-webconsole/static/noVNC similarity index 100% rename from webconsole/static/noVNC rename to webconsole/brozzler-webconsole/static/noVNC diff --git a/webconsole/static/partials/home.html b/webconsole/brozzler-webconsole/static/partials/home.html similarity index 100% rename from webconsole/static/partials/home.html rename to webconsole/brozzler-webconsole/static/partials/home.html diff --git a/webconsole/static/partials/job.html b/webconsole/brozzler-webconsole/static/partials/job.html similarity index 95% rename from webconsole/static/partials/job.html rename to webconsole/brozzler-webconsole/static/partials/job.html index f97ed57..7919653 100644 --- a/webconsole/static/partials/job.html +++ b/webconsole/brozzler-webconsole/static/partials/job.html @@ -40,7 +40,7 @@

Sites

- thumb + thumb
{{site.seed}}