more incremental progress on web console

webconsole/brozzler-webconsole.py

@@ -73,8 +73,13 @@ def jobs():
     jobs_ = list(r.table("jobs").run())
     return flask.jsonify(jobs=jobs_)
 
+@app.route("/api/<path:path>")
+@app.route("/api", defaults={"path":""})
+def api404(path):
+    flask.abort(404)
+
 @app.route("/", defaults={"path": ""})
-@app.route('/<path:path>')
+@app.route("/<path:path>")
 def root(path):
     return app.send_static_file("index.html")
 

webconsole/static/index.html

@@ -17,6 +17,9 @@
   .thumbnail:focus, .thumbnail:hover { text-decoration: none; }
   .thumbnail { word-wrap: break-word; }
   /* .glyphicon { color: #563d7c; } */
+  .bigstats { font-size:125% }
+  .bigstats .fa { font-size:150% }
+  .bigstats .stat { padding:2rem }
 </style>
 </head>
 

webconsole/static/js/controllers.js

@@ -11,10 +11,14 @@ brozzlerControllers.controller("JobsListController", ["$scope", "$http",
 
 brozzlerControllers.controller("JobController", ["$scope", "$routeParams", "$http",
   function($scope, $routeParams, $http) {
-    $scope.phoneId = $routeParams.phoneId;
     $http.get("/api/jobs/" + $routeParams.id).success(function(data) {
       $scope.job = data;
-      // console.log("job=", $scope.job);
+      $scope.job.page_count = $scope.job.queued_count = 0;
+      console.log("job=", $scope.job);
+      $http.get("/api/stats/" + $scope.job.conf.warcprox_meta.stats.buckets[0]).success(function(data) {
+        $scope.job.stats = data;
+        // console.log("job stats=", $scope.job.stats);
+      });
     });
 
     function statsSuccessCallback(site, bucket) {
@@ -30,14 +34,16 @@ brozzlerControllers.controller("JobController", ["$scope", "$routeParams", "$htt
         // console.log("site = ", site);
         // console.log("/api/sites/" + site.id + "/page_count = ", data);
         site.page_count = data.count;
+        $scope.job.page_count += data.count;
       }
     }
 
     function queuedCountSuccessCallback(site, bucket) {
       return function(data) {
-        console.log("site = ", site);
+        // console.log("site = ", site);
-        console.log("/api/sites/" + site.id + "/queued_count = ", data);
+        // console.log("/api/sites/" + site.id + "/queued_count = ", data);
         site.queued_count = data.count;
+        $scope.job.queued_count += data.count;
       }
     }
 
@@ -45,10 +51,11 @@ brozzlerControllers.controller("JobController", ["$scope", "$routeParams", "$htt
       $scope.sites = data.sites;
       // console.log("sites=", $scope.sites);
       for (var i = 0; i < $scope.sites.length; i++) {
-        var site = $scope.sites[i]; // parse Warcprox-Meta to find stats bucket
+        var site = $scope.sites[i]; 
         $http.get("/api/sites/" + site.id + "/page_count").success(pageCountSuccessCallback(site, bucket));
         $http.get("/api/sites/" + site.id + "/queued_count").success(queuedCountSuccessCallback(site, bucket));
 
+        // parse Warcprox-Meta to find stats bucket
         var warcprox_meta = angular.fromJson(site.extra_headers["Warcprox-Meta"]);
         for (var j = 0; j < warcprox_meta.stats.buckets.length; j++) {
           if (warcprox_meta.stats.buckets[j].indexOf("seed") >= 0) {

webconsole/static/partials/job.html

@@ -10,7 +10,30 @@
 </div>
 
 <div>
-  <h2>Job {{job.id}}</h2>
+    <h2>Job {{job.id}} <small>{{job.started}}-{{job.finished}} {{job.status}}</small></h2>
+
+    <div class="row bigstats">
+        <div class="col-sm-6 col-md-3">
+            <div class="stat">
+                <span class="fa fa-file-text"></span> <strong>{{job.page_count}}</strong> pages crawled
+            </div>
+        </div>
+        <div class="col-sm-6 col-md-3">
+            <div class="stat">
+                <span class="fa fa-clone"></span> <strong>{{job.stats.total.urls}}</strong> urls crawled
+            </div>
+        </div>
+        <div class="col-sm-6 col-md-3">
+            <div class="stat">
+                <span class="fa fa-archive"></span> <strong>{{job.stats.total.wire_bytes | byteformat}}</strong> crawled
+            </div>
+        </div>
+        <div class="col-sm-6 col-md-3">
+            <div class="stat">
+                <span class="fa fa-ellipsis-h"></span> <strong>{{job.queued_count}}</strong> pages queued
+            </div>
+        </div>
+    </div>
 
   <div class="row">
     <div class="col-sm-12">

webconsole/static/partials/site.html

@@ -13,9 +13,26 @@
 <div>
   <h2>Site {{site.seed}} (Job <a href="/jobs/{{site.job_id}}">{{site.job_id}}</a>)</h2>
 
-  <div class="row">
+    <div class="row bigstats">
-    <div class="col-sm-12">
+        <div class="col-sm-6 col-md-3">
-      site={{site}}
+            <div class="stat">
+                <span class="fa fa-file-text"></span> <strong>{{site.page_count}}</strong> pages crawled
+            </div>
+        </div>
+        <div class="col-sm-6 col-md-3">
+            <div class="stat">
+                <span class="fa fa-clone"></span> <strong>{{site.stats.total.urls}}</strong> urls crawled
+            </div>
+        </div>
+        <div class="col-sm-6 col-md-3">
+            <div class="stat">
+                <span class="fa fa-archive"></span> <strong>{{site.stats.total.wire_bytes | byteformat}}</strong> crawled
+            </div>
+        </div>
+        <div class="col-sm-6 col-md-3">
+            <div class="stat">
+                <span class="fa fa-ellipsis-h"></span> <strong>{{site.queued_count}}</strong> pages queued
+            </div>
         </div>
     </div>