Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 08:39:59 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

more incremental progress on web console

Browse Source
This commit is contained in:
Noah Levitt 2015-10-09 20:12:40 +00:00
parent 2ddda68392
commit 0591548861
5 changed files with 66 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
webconsole/brozzler-webconsole.py
View File

@ -73,8 +73,13 @@ def jobs():
jobs_ = list(r.table("jobs").run())
return flask.jsonify(jobs=jobs_)
@app.route("/api/<path:path>")
@app.route("/api", defaults={"path":""})
def api404(path):
flask.abort(404)
@app.route("/", defaults={"path": ""})
@app.route('/<path:path>')
@app.route("/<path:path>")
def root(path):
return app.send_static_file("index.html")

3
webconsole/static/index.html
View File

@ -17,6 +17,9 @@
.thumbnail:focus, .thumbnail:hover { text-decoration: none; }
.thumbnail { word-wrap: break-word; }
/* .glyphicon { color: #563d7c; } */
.bigstats { font-size:125% }
.bigstats .fa { font-size:150% }
.bigstats .stat { padding:2rem }
</style>
</head>

17
webconsole/static/js/controllers.js
View File

@ -11,10 +11,14 @@ brozzlerControllers.controller("JobsListController", ["$scope", "$http",
brozzlerControllers.controller("JobController", ["$scope", "$routeParams", "$http",
function($scope, $routeParams, $http) {
$scope.phoneId = $routeParams.phoneId;
$http.get("/api/jobs/" + $routeParams.id).success(function(data) {
$scope.job = data;
// console.log("job=", $scope.job);
$scope.job.page_count = $scope.job.queued_count = 0;
console.log("job=", $scope.job);
$http.get("/api/stats/" + $scope.job.conf.warcprox_meta.stats.buckets[0]).success(function(data) {
$scope.job.stats = data;
// console.log("job stats=", $scope.job.stats);
});
});
function statsSuccessCallback(site, bucket) {
@ -30,14 +34,16 @@ brozzlerControllers.controller("JobController", ["$scope", "$routeParams", "$htt
// console.log("site = ", site);
// console.log("/api/sites/" + site.id + "/page_count = ", data);
site.page_count = data.count;
$scope.job.page_count += data.count;
}
}
function queuedCountSuccessCallback(site, bucket) {
return function(data) {
console.log("site = ", site);
console.log("/api/sites/" + site.id + "/queued_count = ", data);
// console.log("site = ", site);
// console.log("/api/sites/" + site.id + "/queued_count = ", data);
site.queued_count = data.count;
$scope.job.queued_count += data.count;
}
}
@ -45,10 +51,11 @@ brozzlerControllers.controller("JobController", ["$scope", "$routeParams", "$htt
$scope.sites = data.sites;
// console.log("sites=", $scope.sites);
for (var i = 0; i < $scope.sites.length; i++) {
var site = $scope.sites[i]; // parse Warcprox-Meta to find stats bucket
var site = $scope.sites[i];
$http.get("/api/sites/" + site.id + "/page_count").success(pageCountSuccessCallback(site, bucket));
$http.get("/api/sites/" + site.id + "/queued_count").success(queuedCountSuccessCallback(site, bucket));
// parse Warcprox-Meta to find stats bucket
var warcprox_meta = angular.fromJson(site.extra_headers["Warcprox-Meta"]);
for (var j = 0; j < warcprox_meta.stats.buckets.length; j++) {
if (warcprox_meta.stats.buckets[j].indexOf("seed") >= 0) {

25
webconsole/static/partials/job.html
View File

@ -10,7 +10,30 @@
</div>
<div>
<h2>Job {{job.id}}</h2>
<h2>Job {{job.id}} <small>{{job.started}}-{{job.finished}} {{job.status}}</small></h2>
<div class="row bigstats">
<div class="col-sm-6 col-md-3">
<div class="stat">
<span class="fa fa-file-text"></span> <strong>{{job.page_count}}</strong> pages crawled
</div>
</div>
<div class="col-sm-6 col-md-3">
<div class="stat">
<span class="fa fa-clone"></span> <strong>{{job.stats.total.urls}}</strong> urls crawled
</div>
</div>
<div class="col-sm-6 col-md-3">
<div class="stat">
<span class="fa fa-archive"></span> <strong>{{job.stats.total.wire_bytes | byteformat}}</strong> crawled
</div>
</div>
<div class="col-sm-6 col-md-3">
<div class="stat">
<span class="fa fa-ellipsis-h"></span> <strong>{{job.queued_count}}</strong> pages queued
</div>
</div>
</div>
<div class="row">
<div class="col-sm-12">

25
webconsole/static/partials/site.html
View File

@ -13,11 +13,28 @@
<div>
<h2>Site {{site.seed}} (Job <a href="/jobs/{{site.job_id}}">{{site.job_id}}</a>)</h2>
<div class="row">
<div class="col-sm-12">
site={{site}}
<div class="row bigstats">
<div class="col-sm-6 col-md-3">
<div class="stat">
<span class="fa fa-file-text"></span> <strong>{{site.page_count}}</strong> pages crawled
</div>
</div>
<div class="col-sm-6 col-md-3">
<div class="stat">
<span class="fa fa-clone"></span> <strong>{{site.stats.total.urls}}</strong> urls crawled
</div>
</div>
<div class="col-sm-6 col-md-3">
<div class="stat">
<span class="fa fa-archive"></span> <strong>{{site.stats.total.wire_bytes | byteformat}}</strong> crawled
</div>
</div>
<div class="col-sm-6 col-md-3">
<div class="stat">
<span class="fa fa-ellipsis-h"></span> <strong>{{site.queued_count}}</strong> pages queued
</div>
</div>
</div>
</div>
<div class="row">
<div class="col-sm-12">