From d1158ab2242815b28fe7bb066042b5b5982e4627 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Thu, 8 Oct 2015 00:33:49 +0000 Subject: [PATCH] incremental progress on web console --- webconsole/brozzler-webconsole.py | 33 ++++++++++ .../static/brozzler.svg | 0 webconsole/static/index.html | 10 ++- webconsole/static/js/app.js | 3 + webconsole/static/js/controllers.js | 60 ++++++++++++----- webconsole/static/partials/job.html | 66 +++++++++---------- webconsole/static/partials/jobs.html | 28 +++++--- webconsole/static/partials/site.html | 32 ++++++++- 8 files changed, 169 insertions(+), 63 deletions(-) rename brozzler.svg => webconsole/static/brozzler.svg (100%) diff --git a/webconsole/brozzler-webconsole.py b/webconsole/brozzler-webconsole.py index 1053ea0..80d9d3d 100644 --- a/webconsole/brozzler-webconsole.py +++ b/webconsole/brozzler-webconsole.py @@ -12,6 +12,39 @@ app = flask.Flask(__name__) r = rethinkstuff.Rethinker(["wbgrp-svc020", "wbgrp-svc035", "wbgrp-svc036"], db="archiveit_brozzler") +@app.route("/api/sites//queued_count") +@app.route("/api/site//queued_count") +def queued_count(site_id): + count = r.table("pages").between([site_id, 0, False, r.minval], [site_id, 0, False, r.maxval], index="priority_by_site").count().run() + return flask.jsonify(count=count) + +@app.route("/api/sites//queue") +@app.route("/api/site//queue") +def queue(site_id): + logging.info("flask.request.args=%s", flask.request.args) + start = flask.request.args.get("start", 0) + end = flask.request.args.get("end", start + 90) + queue_ = r.table("pages").between([site_id, 0, False, r.minval], [site_id, 0, False, r.maxval], index="priority_by_site")[start:end].run() + return flask.jsonify(queue_=list(queue_)) + +@app.route("/api/sites//pages_count") +@app.route("/api/site//pages_count") +@app.route("/api/sites//page_count") +@app.route("/api/site//page_count") +def page_count(site_id): + count = r.table("pages").between([site_id, 1, False, r.minval], [site_id, r.maxval, False, r.maxval], index="priority_by_site").count().run() + return flask.jsonify(count=count) + +@app.route("/api/sites//pages") +@app.route("/api/site//pages") +def pages(site_id): + """Pages already crawled.""" + logging.info("flask.request.args=%s", flask.request.args) + start = flask.request.args.get("start", 0) + end = flask.request.args.get("end", start + 90) + pages_ = r.table("pages").between([site_id, 1, False, r.minval], [site_id, r.maxval, False, r.maxval], index="priority_by_site")[start:end].run() + return flask.jsonify(pages=list(pages_)) + @app.route("/api/sites/") @app.route("/api/site/") def site(site_id): diff --git a/brozzler.svg b/webconsole/static/brozzler.svg similarity index 100% rename from brozzler.svg rename to webconsole/static/brozzler.svg diff --git a/webconsole/static/index.html b/webconsole/static/index.html index 721a287..a5505eb 100644 --- a/webconsole/static/index.html +++ b/webconsole/static/index.html @@ -7,15 +7,21 @@ Brozzler Console: Jobs + + -
-
+
diff --git a/webconsole/static/js/app.js b/webconsole/static/js/app.js index d9574e0..cb7cdc4 100644 --- a/webconsole/static/js/app.js +++ b/webconsole/static/js/app.js @@ -20,6 +20,9 @@ brozzlerConsoleApp.config(["$routeProvider", "$locationProvider", templateUrl: "/static/partials/site.html", controller: "SiteController" }). + when("/", { + redirectTo: "/jobs" + }). otherwise({ template: '

Not Found

How the heck did you get here?
', }); diff --git a/webconsole/static/js/controllers.js b/webconsole/static/js/controllers.js index 4029071..8c118c5 100644 --- a/webconsole/static/js/controllers.js +++ b/webconsole/static/js/controllers.js @@ -14,23 +14,47 @@ brozzlerControllers.controller("JobController", ["$scope", "$routeParams", "$htt $scope.phoneId = $routeParams.phoneId; $http.get("/api/jobs/" + $routeParams.id).success(function(data) { $scope.job = data; - console.log("job=", $scope.job); + // console.log("job=", $scope.job); }); + function statsSuccessCallback(site, bucket) { + return function(data) { + // console.log("site = ", site); + // console.log("/api/stats/" + bucket + " = ", data); + site.stats = data; + } + } + + function pageCountSuccessCallback(site, bucket) { + return function(data) { + // console.log("site = ", site); + // console.log("/api/sites/" + site.id + "/page_count = ", data); + site.page_count = data.count; + } + } + + function queuedCountSuccessCallback(site, bucket) { + return function(data) { + console.log("site = ", site); + console.log("/api/sites/" + site.id + "/queued_count = ", data); + site.queued_count = data.count; + } + } + $http.get("/api/jobs/" + $routeParams.id + "/sites").success(function(data) { $scope.sites = data.sites; - console.log("sites=", $scope.sites); + // console.log("sites=", $scope.sites); for (var i = 0; i < $scope.sites.length; i++) { var site = $scope.sites[i]; // parse Warcprox-Meta to find stats bucket + $http.get("/api/sites/" + site.id + "/page_count").success(pageCountSuccessCallback(site, bucket)); + $http.get("/api/sites/" + site.id + "/queued_count").success(queuedCountSuccessCallback(site, bucket)); + var warcprox_meta = angular.fromJson(site.extra_headers["Warcprox-Meta"]); for (var j = 0; j < warcprox_meta.stats.buckets.length; j++) { if (warcprox_meta.stats.buckets[j].indexOf("seed") >= 0) { - console.log("warcprox_meta.stats.buckets[" + j + "]=" + warcprox_meta.stats.buckets[j]); var bucket = warcprox_meta.stats.buckets[j]; - $http.get("/api/stats/" + warcprox_meta.stats.buckets[j]).success(function(data) { - console.log("/api/stats/" + bucket + "=", data); - site.stats = data; - }); + // console.log("warcprox_meta.stats.buckets[" + j + "]=" + bucket); + $http.get("/api/stats/" + bucket).success(statsSuccessCallback(site, bucket)); } } } @@ -41,16 +65,22 @@ brozzlerControllers.controller("SiteController", ["$scope", "$routeParams", "$ht function($scope, $routeParams, $http) { $http.get("/api/site/" + $routeParams.id).success(function(data) { $scope.site = data; + // console.log("site = ", $scope.site); + }); + + $http.get("/api/site/" + $routeParams.id + "/pages?start=0&end=99").success(function(data) { + $scope.pages = data.pages; + // console.log("pages = ", $scope.pages); }); }]); /* -$http.get(...) - .then(function(response){ - // successHandler - // do some stuff - return $http.get('/somethingelse') // get more data - }) - .then(anotherSuccessHandler) - .catch(errorHandler) +$http.get("/api/site/" + $routeParams.id).then(function(response) { + console.log("/api/site/" + $routeParams.id + " returned", response); + $scope.site = response.data; + return $http.get("/api/site/" + $routeParams.id + "/pages"); +}).then(function(response) { + console.log("/api/site/" + $routeParams.id + "/pages returned", response); + $scope.site.pages = response.data.pages; +}); */ diff --git a/webconsole/static/partials/job.html b/webconsole/static/partials/job.html index 4a05af9..9783f82 100644 --- a/webconsole/static/partials/job.html +++ b/webconsole/static/partials/job.html @@ -1,41 +1,41 @@ +
    +
  1. Jobs
    2. +
  2. {{job.id}}
    3. +
+ +
+

Brozzler + +

+
+ diff --git a/webconsole/static/partials/jobs.html b/webconsole/static/partials/jobs.html index 719c75b..a410761 100644 --- a/webconsole/static/partials/jobs.html +++ b/webconsole/static/partials/jobs.html @@ -1,7 +1,15 @@ +
    +
  1. Jobs
    2. +
+ +
+

Brozzler + +

+
+
-
-

Jobs

-
+

Jobs

@@ -16,13 +24,13 @@ - - {{job.id}} - {{job.status}} - {{job.started}} - {{job.finished}} - {{job.conf.seeds.length}} - + + {{job.id}} + {{job.status}} + {{job.started}} + {{job.finished}} + {{job.conf.seeds.length}} +
diff --git a/webconsole/static/partials/site.html b/webconsole/static/partials/site.html index 986b17b..4523196 100644 --- a/webconsole/static/partials/site.html +++ b/webconsole/static/partials/site.html @@ -1,11 +1,37 @@ +
    +
  1. Jobs
    2. +
  2. {{site.job_id}}
    3. +
  3. {{site.seed}}
    4. +
+ +
+

Brozzler + +

+
+
-
-

Site {{site.seed}} (Job {{site.job_id}})

-
+

Site {{site.seed}} (Job {{site.job_id}})

site={{site}}
+ +
+
+

Pages

+
+
+ thumb +
+

Thumbnail label

+ {{page}} +
+
+
+
+
+