Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-09-18 19:54:39 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

un-hardcode some stuff in webconsole, load from environment variables instead

Browse source
This commit is contained in:
Noah Levitt 2016-04-19 18:51:14 +00:00
parent 35b713a2e7
commit 72a94ed816
11 changed files with 67 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

122
webconsole/brozzler-webconsole/__init__.py Normal file
View file

@ -0,0 +1,122 @@
import flask
import rethinkstuff
import json
import logging
import sys
import os
logging.basicConfig(stream=sys.stdout, level=logging.INFO,
format="%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s")
app = flask.Flask(__name__)
# configure with environment variables
SETTINGS= {
'RETHINKDB_SERVERS': os.environ.get(
'RETHINKDB_SERVERS', 'localhost').split(','),
'RETHINKDB_DB': os.environ.get('RETHINKDB_DB', 'brozzler'),
'WAYBACK_BASEURL': os.environ.get(
'WAYBACK_BASEURL', 'http://wbgrp-svc107.us.archive.org:8091'),
}
r = rethinkstuff.Rethinker(
SETTINGS['RETHINKDB_SERVERS'], db=SETTINGS['RETHINKDB_DB'])
@app.route("/api/sites/<site_id>/queued_count")
@app.route("/api/site/<site_id>/queued_count")
def queued_count(site_id):
count = r.table("pages").between(
[site_id, 0, False, r.minval], [site_id, 0, False, r.maxval],
index="priority_by_site").count().run()
return flask.jsonify(count=count)
@app.route("/api/sites/<site_id>/queue")
@app.route("/api/site/<site_id>/queue")
def queue(site_id):
logging.info("flask.request.args=%s", flask.request.args)
start = flask.request.args.get("start", 0)
end = flask.request.args.get("end", start + 90)
queue_ = r.table("pages").between(
[site_id, 0, False, r.minval], [site_id, 0, False, r.maxval],
index="priority_by_site")[start:end].run()
return flask.jsonify(queue_=list(queue_))
@app.route("/api/sites/<site_id>/pages_count")
@app.route("/api/site/<site_id>/pages_count")
@app.route("/api/sites/<site_id>/page_count")
@app.route("/api/site/<site_id>/page_count")
def page_count(site_id):
count = r.table("pages").between(
[site_id, 1, False, r.minval],
[site_id, r.maxval, False, r.maxval],
index="priority_by_site").count().run()
return flask.jsonify(count=count)
@app.route("/api/sites/<site_id>/pages")
@app.route("/api/site/<site_id>/pages")
def pages(site_id):
"""Pages already crawled."""
logging.info("flask.request.args=%s", flask.request.args)
start = int(flask.request.args.get("start", 0))
end = int(flask.request.args.get("end", start + 90))
pages_ = r.table("pages").between(
[site_id, 1, False, r.minval],
[site_id, r.maxval, False, r.maxval],
index="priority_by_site")[start:end].run()
return flask.jsonify(pages=list(pages_))
@app.route("/api/sites/<site_id>")
@app.route("/api/site/<site_id>")
def site(site_id):
site_ = r.table("sites").get(site_id).run()
return flask.jsonify(site_)
@app.route("/api/stats/<bucket>")
def stats(bucket):
stats_ = r.table("stats").get(bucket).run()
return flask.jsonify(stats_)
@app.route("/api/jobs/<int:job_id>/sites")
@app.route("/api/job/<int:job_id>/sites")
def sites(job_id):
sites_ = r.table("sites").get_all(job_id, index="job_id").run()
return flask.jsonify(sites=list(sites_))
@app.route("/api/jobs/<int:job_id>")
@app.route("/api/job/<int:job_id>")
def job(job_id):
job_ = r.table("jobs").get(job_id).run()
return flask.jsonify(job_)
@app.route("/api/workers")
def workers():
workers_ = r.table("services").filter({"role":"brozzler-worker"}).run()
return flask.jsonify(workers=list(workers_))
@app.route("/api/services")
def services():
services_ = r.table("services").run()
return flask.jsonify(services=list(services_))
@app.route("/api/jobs")
def jobs():
jobs_ = list(r.table("jobs").run())
jobs_ = sorted(jobs_, key=lambda j: j['id'], reverse=True)
return flask.jsonify(jobs=jobs_)
@app.route("/api/config")
def config():
return flask.jsonify(config=SETTINGS)
@app.route("/api/<path:path>")
@app.route("/api", defaults={"path":""})
def api404(path):
flask.abort(404)
@app.route("/", defaults={"path": ""})
@app.route("/<path:path>")
def root(path):
return flask.render_template("index.html")
if __name__ == "__main__":
app.run(host="0.0.0.0", port=8081, debug=True)

262
webconsole/brozzler-webconsole/static/brozzler.svg Normal file
View file

@ -0,0 +1,262 @@
<svg width='780' height='580' xmlns='http://www.w3.org/2000/svg' version='1.1'
xmlns:svg="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
>
<!-- make sure glyph is visible within svg window -->
<g fill-rule='nonzero' transform='translate(30 0)'>
<!-- draw actual outline using lines and Bezier curves-->
<path fill='#666' stroke='black' stroke-width='0' transform='scale(1.35) translate(-46 517)' d='
M 582,-127
L 582,-229
Q 582,-240 571,-240
L 485,-240
L 410,-323
Q 364,-374 297,-374
Q 237,-374 193,-332
L 42,-193
Q 39,-187 39,-186
Q 39,-181 42,-178
L 110,-110
Q 112,-107 117,-107
Q 123,-107 125,-110
L 230,-214
L 279,-156
Q 315,-117 367,-117
L 571,-117
Q 582,-117 582,-127
Z
'/>
<defs
id="defs3043">
<linearGradient
id="linearGradient3803">
<stop
style="stop-color:#d7def0;stop-opacity:1;"
offset="0"
id="stop3805" />
<stop
id="stop3811"
offset="0.5"
style="stop-color:#ffffff;stop-opacity:1" />
<stop
style="stop-color:#d5def0;stop-opacity:1"
offset="1"
id="stop3807" />
</linearGradient>
<linearGradient
id="linearGradient3776"
inkscape:collect="always">
<stop
id="stop3778"
offset="0"
style="stop-color:#b2cde9;stop-opacity:1" />
<stop
id="stop3780"
offset="1"
style="stop-color:#c4dbee;stop-opacity:1" />
</linearGradient>
<linearGradient
id="linearGradient3750">
<stop
id="stop3752"
offset="0"
style="stop-color:#d0e2f1;stop-opacity:1" />
<stop
style="stop-color:#cadef0;stop-opacity:1"
offset="0.85580856"
id="stop3756" />
<stop
id="stop3754"
offset="1"
style="stop-color:#95bee3;stop-opacity:1" />
</linearGradient>
<linearGradient
id="linearGradient3708">
<stop
style="stop-color:#658db6;stop-opacity:1"
offset="0"
id="stop3710" />
<stop
id="stop3716"
offset="0.76777935"
style="stop-color:#527fab;stop-opacity:1;" />
<stop
style="stop-color:#4071a0;stop-opacity:1"
offset="1"
id="stop3712" />
</linearGradient>
<linearGradient
id="linearGradient3698">
<stop
style="stop-color:#96d0e1;stop-opacity:1"
offset="0"
id="stop3700" />
<stop
id="stop3706"
offset="0.67819428"
style="stop-color:#89b7e1;stop-opacity:1" />
<stop
style="stop-color:#699dd3;stop-opacity:1"
offset="1"
id="stop3702" />
</linearGradient>
<linearGradient
inkscape:collect="always"
id="linearGradient3647">
<stop
style="stop-color:#3b79bc;stop-opacity:1;"
offset="0"
id="stop3649" />
<stop
style="stop-color:#94b8e0;stop-opacity:1"
offset="1"
id="stop3651" />
</linearGradient>
<linearGradient
inkscape:collect="always"
id="linearGradient3588">
<stop
style="stop-color:#ffffff;stop-opacity:1"
offset="0"
id="stop3590" />
<stop
style="stop-color:#000000;stop-opacity:0;"
offset="1"
id="stop3592" />
</linearGradient>
<radialGradient
inkscape:collect="always"
xlink:href="#linearGradient3588"
id="radialGradient3594"
cx="-118.77966"
cy="121.49152"
fx="-118.77966"
fy="121.49152"
r="25.491526"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.02177942,-0.95743591,0.97872327,0.02221687,-235.0993,5.0684454)" />
<linearGradient
inkscape:collect="always"
xlink:href="#linearGradient3647"
id="linearGradient3653"
x1="-397.81323"
y1="149.18764"
x2="-397.55933"
y2="51.355946"
gradientUnits="userSpaceOnUse" />
<radialGradient
inkscape:collect="always"
xlink:href="#linearGradient3698"
id="radialGradient3704"
cx="-383.2746"
cy="217.91029"
fx="-383.2746"
fy="217.91029"
r="59.401995"
gradientTransform="matrix(-1.2861568,-0.08596317,0.11453678,-1.7136762,-425.01982,469.50099)"
gradientUnits="userSpaceOnUse" />
<radialGradient
inkscape:collect="always"
xlink:href="#linearGradient3708"
id="radialGradient3714"
cx="-123.5"
cy="-11.570732"
fx="-123.5"
fy="-11.570732"
r="95.627118"
gradientTransform="matrix(-0.00756512,0.55751399,-1.0314585,-0.01398286,113.23967,103.212)"
gradientUnits="userSpaceOnUse" />
<radialGradient
inkscape:collect="always"
xlink:href="#linearGradient3750"
id="radialGradient3748"
cx="-94.87291"
cy="165.27281"
fx="-94.87291"
fy="165.27281"
r="60.481357"
gradientTransform="matrix(0.81293878,1.6998003,-2.1519091,1.0291615,564.39485,118.47915)"
gradientUnits="userSpaceOnUse" />
<linearGradient
inkscape:collect="always"
xlink:href="#linearGradient3776"
id="linearGradient3774"
x1="162.07127"
y1="85.239708"
x2="220.76114"
y2="78.875748"
gradientUnits="userSpaceOnUse"
gradientTransform="translate(3.3917128,7.418629)" />
<linearGradient
inkscape:collect="always"
xlink:href="#linearGradient3803"
id="linearGradient3809"
x1="-382.04123"
y1="37.280548"
x2="-381.39438"
y2="165.56691"
gradientUnits="userSpaceOnUse" />
</defs>
<path
sodipodi:type="arc"
style="fill:url(#radialGradient3594);fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path2814"
sodipodi:cx="-118.23729"
sodipodi:cy="122.57627"
sodipodi:rx="25.491526"
sodipodi:ry="25.491526"
d="m -92.745764,122.57627 a 25.491526,25.491526 0 1 1 -50.983056,0 25.491526,25.491526 0 1 1 50.983056,0 z"
transform="matrix(4.680851,0,0,4.7978723,685.10478,-449.69946)" />
<path
style="fill:url(#linearGradient3774);fill-opacity:1;fill-rule:nonzero;stroke:none"
d="m 232.17258,88.120422 c 0,15.673918 -19.79135,34.931518 -45.84395,34.931518 -26.0526,0 -59.92241,-16.08123 -59.92241,-31.755152 0,-15.673924 21.11981,-28.38015 47.17241,-28.38015 19.90254,0 46.36122,18.293224 56.45971,20.3521 0.79179,1.710571 1.36862,2.925087 2.13424,4.851684 z"
id="path3655"
sodipodi:nodetypes="cssscc" />
<path
style="fill:#2e5c91;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="m 38.822019,65.971523 c 12.38148,-9.610993 35.314514,-1.245318 51.289554,19.334679 15.975027,20.579998 17.694937,51.065068 5.31349,60.676058 -12.38147,9.61099 -34.17571,-5.29155 -50.15074,-25.87156 -12.20392,-15.72181 -4.05062,-41.19089 -8.61646,-50.430553 0.61589,-1.122052 1.381696,-2.456607 2.164156,-3.708624 z"
id="path3655-4-8"
sodipodi:nodetypes="cssscc" />
<path
style="fill:url(#radialGradient3714);fill-opacity:1;fill-rule:nonzero;stroke:none"
d="m 230.04347,83.261765 c -7.0081,-0.03265 -61.07025,0.289575 -107.66568,0.0654 -17.371,5.108098 -31.704627,13.258827 -39.181777,29.154945 -5.33639,-4.54237 -40.74576,-42.215609 -44.40678,-46.440684 31.38983,-41.648805 74.528017,-45.559321 82.915257,-45.559321 8.38724,0 70.64407,-8.631855 108.33898,62.77966 z"
id="path3596"
sodipodi:nodetypes="ccccsc" />
<path
style="fill:#699dd3;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="m 117.12454,243.96815 c -12.49835,-9.45851 -14.5752,-36.93927 1.14635,-57.71356 15.72155,-20.77428 41.03582,-34.94753 53.53417,-25.48904 12.49834,9.4585 7.44792,38.96701 -8.27364,59.74129 -12.01027,15.87024 -35.4911,16.88498 -43.22681,23.69505 -1.23894,-0.0455 -1.95523,-0.0605 -3.18007,-0.23374 z"
id="path3655-4"
sodipodi:nodetypes="cssscc" />
<path
style="fill:url(#radialGradient3748);fill-opacity:1;fill-rule:nonzero;stroke:none"
d="m 120.3032,244.20103 c 3.58354,-6.02268 28.85859,-52.8991 52.69131,-92.9389 4.41104,-17.56095 5.34663,-33.64185 -4.5584,-48.14993 6.62173,-2.29412 58.23852,-13.976353 63.73684,-14.987686 19.9656,48.180076 1.44992,87.338276 -2.80522,94.565966 -4.25515,7.22768 -28.40179,65.25666 -109.06453,61.51055 z"
id="path3596-1"
sodipodi:nodetypes="ccccsc" />
<path
style="fill:url(#radialGradient3704);fill-opacity:1;fill-rule:nonzero;stroke:none"
d="m 36.696853,69.642524 c 3.46858,6.089612 30.72312,52.780196 53.77852,93.272576 13.094367,12.50527 27.684997,19.48512 45.191737,18.03328 -1.2738,6.89113 -16.62898,57.75037 -18.4638,63.03126 -51.756237,-6.42158 -76.669777,-41.85476 -80.854757,-49.1233 -4.18497,-7.26855 -42.7297502,-56.91452 0.3483,-125.213816 z"
id="path3596-1-7"
sodipodi:nodetypes="ccccsc" />
<path
transform="matrix(0.77294737,0,0,0.77619098,435.90647,53.275706)"
style="fill:url(#linearGradient3653);fill-opacity:1;stroke:url(#linearGradient3809);stroke-width:10.07013607;stroke-miterlimit:4;stroke-opacity:1"
d="m -338.44068,101.42373 c 0,32.65032 -26.46832,59.11864 -59.11865,59.11864 -32.65032,0 -59.11864,-26.46832 -59.11864,-59.11864 0,-32.650327 26.46832,-59.118646 59.11864,-59.118646 32.65033,0 59.11865,26.468319 59.11865,59.118646 z"
id="path3645" />
<path
style="fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="m 163.54619,108.89582 c 18.52979,17.09836 16.03302,29.55794 10.0625,44 -3.10892,-22.25001 -2.34478,-32.42697 -10.0625,-44 z"
id="rect3782"
sodipodi:nodetypes="ccc" />
<path
style="fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="m 101.42092,173.63924 c -22.645593,-14.47335 -29.809884,-45.71983 -8.813354,-62.99032 -10.847561,19.77514 -6.225429,32.39863 8.813354,62.99032 z"
id="rect3782-4"
sodipodi:nodetypes="ccc" />
<!-- <text x="400" y="100" fill="black" font-size="70">brozzler</text> -->
</g>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 9.1 KiB

186
webconsole/brozzler-webconsole/static/js/app.js Normal file
View file

@ -0,0 +1,186 @@
"use strict";
var brozzlerConsoleApp = angular.module("brozzlerConsoleApp", [
"ngRoute",
"brozzlerControllers",
]);
brozzlerConsoleApp.config(["$routeProvider", "$locationProvider",
function($routeProvider, $locationProvider) {
$routeProvider.
when("/workers", {
templateUrl: "/static/partials/workers.html",
controller: "WorkersListController"
}).
when("/jobs/:id", {
templateUrl: "/static/partials/job.html",
controller: "JobController"
}).
when("/sites/:id", {
templateUrl: "/static/partials/site.html",
controller: "SiteController"
}).
when("/", {
templateUrl: "/static/partials/home.html",
controller: "HomeController"
}).
otherwise({
template: '<div> <div class="page-header"> <h1>Not Found</h1> </div> <div class="row"> <div class="col-sm-12"> How the heck did you get here? </div> </div> </div> ',
});
$locationProvider.html5Mode({
enabled: true,
requireBase: false,
});
}]);
// copied from https://bitbucket.org/webarchive/ait5/src/master/archiveit/static/app/js/filters/ByteFormat.js
brozzlerConsoleApp.filter("byteformat", function() {
return function(bytes, precision) {
var bytes_f = parseFloat(bytes);
if (bytes_f == 0 || isNaN(bytes_f) || !isFinite(bytes_f)) return "0";
if (bytes_f < 1024) return bytes_f.toFixed(0) + " bytes";
if (typeof precision === "undefined") precision = 1;
var units = ["bytes", "kB", "MB", "GB", "TB", "PB"];
var number = Math.floor(Math.log(bytes_f) / Math.log(1024));
var result = (bytes_f / Math.pow(1024, Math.floor(number))).toFixed(precision) + " " + units[number];
return result;
}
});
var brozzlerControllers = angular.module("brozzlerControllers", []);
brozzlerControllers.controller("HomeController", ["$scope", "$http",
function($scope, $http) {
$http.get("/api/config").success(function(data) {
$scope.config = data.config;
});
$http.get("/api/jobs").success(function(data) {
$scope.jobs = data.jobs;
});
$http.get("/api/services").success(function(data) {
$scope.services = data.services;
});
}]);
brozzlerControllers.controller("WorkersListController", ["$scope", "$http",
function($scope, $http) {
$http.get("/api/config").success(function(data) {
$scope.config = data.config;
});
$http.get("/api/workers").success(function(data) {
$scope.workers = data.workers;
});
}]);
function statsSuccessCallback(site, bucket) {
return function(data) {
// console.log("site = ", site);
// console.log("/api/stats/" + bucket + " = ", data);
site.stats = data;
}
}
function pageCountSuccessCallback(site, job) {
return function(data) {
// console.log("site = ", site);
// console.log("/api/sites/" + site.id + "/page_count = ", data);
site.page_count = data.count;
if (job) {
job.page_count += data.count;
}
}
}
function queuedCountSuccessCallback(site, job) {
return function(data) {
// console.log("site = ", site);
// console.log("/api/sites/" + site.id + "/queued_count = ", data);
site.queued_count = data.count;
if (job) {
job.queued_count += data.count;
}
}
}
function loadSiteStats($http, site, job) {
$http.get("/api/sites/" + site.id + "/page_count").success(pageCountSuccessCallback(site, job));
$http.get("/api/sites/" + site.id + "/queued_count").success(queuedCountSuccessCallback(site, job));
// parse Warcprox-Meta to find stats bucket
var warcprox_meta = angular.fromJson(site.extra_headers["Warcprox-Meta"]);
for (var j = 0; j < warcprox_meta.stats.buckets.length; j++) {
if (warcprox_meta.stats.buckets[j].indexOf("seed") >= 0) {
var bucket = warcprox_meta.stats.buckets[j];
// console.log("warcprox_meta.stats.buckets[" + j + "]=" + bucket);
$http.get("/api/stats/" + bucket).success(statsSuccessCallback(site, bucket));
}
}
}
brozzlerControllers.controller("JobController", ["$scope", "$routeParams", "$http",
function($scope, $routeParams, $http) {
console.log('JobController');
$http.get("/api/config").success(function(data) {
$scope.config = data.config;
});
$http.get("/api/jobs/" + $routeParams.id).success(function(data) {
$scope.job = data;
$scope.job.page_count = $scope.job.queued_count = 0;
// console.log("job=", $scope.job);
$http.get("/api/stats/" + $scope.job.conf.warcprox_meta.stats.buckets[0]).success(function(data) {
$scope.job.stats = data;
// console.log("job stats=", $scope.job.stats);
});
$http.get("/api/jobs/" + $routeParams.id + "/sites").success(function(data) {
$scope.sites = data.sites;
// console.log("sites=", $scope.sites);
for (var i = 0; i < $scope.sites.length; i++) {
loadSiteStats($http, $scope.sites[i], $scope.job);
}
});
});
}]);
brozzlerControllers.controller("SiteController", ["$scope", "$routeParams", "$http", "$window",
function($scope, $routeParams, $http, $window) {
var start = 0;
$scope.loading = false;
$scope.pages = [];
$window.addEventListener("scroll", function() {
// console.log("window.scrollTop=" + window.scrollTop + " window.offsetHeight=" + window.offsetHeight + " window.scrollHeight=" + window.scrollHeight);
if ($window.innerHeight + $window.scrollY + 50 >= window.document.documentElement.scrollHeight) {
loadMorePages();
}
});
var loadMorePages = function() {
if ($scope.loading)
return;
$scope.loading = true;
// console.log("load more! start=" + start);
$http.get("/api/site/" + $routeParams.id + "/pages?start=" + start + "&end=" + (start+90)).then(function(response) {
$scope.pages = $scope.pages.concat(response.data.pages);
// console.log("pages = ", $scope.pages);
start += response.data.pages.length;
$scope.loading = false;
}, function(reason) {
$scope.loading = false;
});
};
$http.get("/api/config").success(function(data) {
$scope.config = data.config;
});
$http.get("/api/site/" + $routeParams.id).success(function(data) {
$scope.site = data;
loadSiteStats($http, $scope.site);
// console.log("site = ", $scope.site);
});
loadMorePages();
}]);

1
webconsole/brozzler-webconsole/static/noVNC Submodule

@ -0,0 +1 @@
Subproject commit 6a90803feb124791960e3962e328aa3cfb729aeb

69
webconsole/brozzler-webconsole/static/partials/home.html Normal file
View file

@ -0,0 +1,69 @@
<ol class="breadcrumb">
<li class="active">Home</li>
</ol>
<div class="page-header">
<h1>Brozzler
<a href="/"><img src="/static/brozzler.svg" style="height:1.5em;float:right"></a>
</h1>
</div>
<div>
<h2>Services</h2>
<p><a href="/workers">Brozzler Workers</a></p>
<div class="row">
<div class="col-sm-12">
<table class="table table-striped">
<thead>
<tr>
<th>role</th>
<th>host</th>
<th>pid</th>
<th>load</th>
<th>first heartbeat</th>
<th>last heartbeat</th>
</tr>
</thead>
<tbody>
<tr ng-repeat="service in services">
<td>{{service.role}}</td>
<td>{{service.host}}</td>
<td>{{service.pid}}</td>
<td>{{service.load}}</td>
<td>{{service.first_heartbeat}}</td>
<td>{{service.last_heartbeat}}</td>
</tr>
</tbody>
</table>
</div>
</div>
<h2>Jobs</h2>
<div class="row">
<div class="col-sm-12">
<table class="table table-striped">
<thead>
<tr>
<th>id</th>
<th>status</th>
<th>started</th>
<th>finished</th>
<th># of seeds</th>
</tr>
</thead>
<tbody>
<tr ng-repeat="job in jobs">
<td><a href="/jobs/{{job.id}}">{{job.id}}</a></td>
<td>{{job.status}}</td>
<td>{{job.started}}</td>
<td>{{job.finished}}</td>
<td>{{job.conf.seeds.length}}</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>

65
webconsole/brozzler-webconsole/static/partials/job.html Normal file
View file

@ -0,0 +1,65 @@
<ol class="breadcrumb">
<li><a href="/">Home</a></li>
<li class="active">Job {{job.id}}</li>
</ol>
<div class="page-header">
<h1>Brozzler
<a href="/"><img src="/static/brozzler.svg" style="height:1.5em;float:right"></a>
</h1>
</div>
<div>
<h2>Job {{job.id}} <small>{{job.started}}-{{job.finished}} {{job.status}}</small></h2>
<div class="row bigstats">
<div class="col-sm-6 col-md-3">
<div class="stat">
<span class="fa fa-file-text"></span> <strong>{{job.page_count}}</strong> pages crawled
</div>
</div>
<div class="col-sm-6 col-md-3">
<div class="stat">
<span class="fa fa-clone"></span> <strong>{{job.stats.total.urls}}</strong> urls crawled
</div>
</div>
<div class="col-sm-6 col-md-3">
<div class="stat">
<span class="fa fa-archive"></span> <strong>{{job.stats.total.wire_bytes | byteformat}}</strong> crawled
</div>
</div>
<div class="col-sm-6 col-md-3">
<div class="stat">
<span class="fa fa-ellipsis-h"></span> <strong>{{job.queued_count}}</strong> pages queued
</div>
</div>
</div>
<div class="row">
<div class="col-sm-12">
<h3>Sites</h3>
<div class="col-sm-6 col-md-4" ng-repeat="site in sites">
<a class="thumbnail" href="/sites/{{site.id}}">
<img style="width:300px;height:190px" src="{{config.WAYBACK_BASEURL}}/3/thumbnail:{{site.seed}}" alt="thumb">
<div class="caption">
<h5>{{site.seed}}</h5>
<!--
<div><span class="glyphicon glyphicon-file"></span> <strong>{{site.page_count}}</strong> pages crawled</div>
<div><span class="glyphicon glyphicon-duplicate"></span> <strong>{{site.stats.total.urls}}</strong> urls crawled</div>
<div><span class="glyphicon glyphicon-oil"></span> <strong>{{site.stats.total.wire_bytes | byteformat}}</strong> crawled</div>
<div><span class="glyphicon glyphicon-menu-hamburger"></span> <strong>{{site.queued_count}}</strong> pages queued</div>
-->
<ul class="fa-ul">
<li><span class="fa fa-li fa-file-text"></span> <strong>{{site.page_count}}</strong> pages crawled</li>
<li><span class="fa fa-li fa-clone"></span> <strong>{{site.stats.total.urls}}</strong> urls crawled</li>
<li><span class="fa fa-li fa-archive"></span> <strong>{{site.stats.total.wire_bytes | byteformat}}</strong> crawled</li>
<li><span class="fa fa-li fa-ellipsis-h"></span> <strong>{{site.queued_count}}</strong> pages queued</li>
</ul>
</div>
</a>
</div>
</div>
</div>
</div>

58
webconsole/brozzler-webconsole/static/partials/site.html Normal file
View file

@ -0,0 +1,58 @@
<ol class="breadcrumb">
<li><a href="/">Home</a></li>
<li><a href="/jobs/{{site.job_id}}">Job {{site.job_id}}</a></li>
<li class="active">{{site.seed}}</li>
</ol>
<div class="page-header">
<h1>Brozzler
<a href="/"><img src="/static/brozzler.svg" style="height:1.5em;float:right"></a>
</h1>
</div>
<div>
<h2>Site {{site.seed}} (Job <a href="/jobs/{{site.job_id}}">{{site.job_id}}</a>)</h2>
<div class="row bigstats">
<div class="col-sm-6 col-md-3">
<div class="stat">
<span class="fa fa-file-text"></span> <strong>{{site.page_count}}</strong> pages crawled
</div>
</div>
<div class="col-sm-6 col-md-3">
<div class="stat">
<span class="fa fa-clone"></span> <strong>{{site.stats.total.urls}}</strong> urls crawled
</div>
</div>
<div class="col-sm-6 col-md-3">
<div class="stat">
<span class="fa fa-archive"></span> <strong>{{site.stats.total.wire_bytes | byteformat}}</strong> crawled
</div>
</div>
<div class="col-sm-6 col-md-3">
<div class="stat">
<span class="fa fa-ellipsis-h"></span> <strong>{{site.queued_count}}</strong> pages queued
</div>
</div>
</div>
<div class="row">
<div class="col-sm-12">
<h2>Pages</h2>
<div class="col-sm-6 col-md-4" ng-repeat="page in pages">
<a class="thumbnail" href="{{config.WAYBACK_BASEURL}}/3/{{page.url}}">
<img style="width:300px;height:190px" src="{{config.WAYBACK_BASEURL}}/3/thumbnail:{{page.url}}" alt="thumb">
<div class="caption">
<h5>{{page.url}}</h5>
</div>
</a>
</div>
</div>
<div class="col-sm-12" ng-show="loading">
<div style="text-align:center;font-size:300%">
<span class="fa fa-spinner fa-pulse"></span>
</div>
</div>
</div>
</div>

24
webconsole/brozzler-webconsole/static/partials/workers.html Normal file
View file

@ -0,0 +1,24 @@
<ol class="breadcrumb">
<li><a href="/">Home</a></li>
<li class="active">Workers</li>
</ol>
<div class="page-header">
<h1>Brozzler
<a href="/"><img src="/static/brozzler.svg" style="height:1.5em;float:right"></a>
</h1>
</div>
<div>
<h2>Workers</h2>
<div class="row">
<div class="col-xs-12 col-sm-12 col-md-6 col-lg-6" ng-repeat="worker in workers">
<div>{{worker.host}}</div>
<iframe style="width:45rem;height:32rem;"
ng-src="{{'/static/noVNC/vnc.html?host=' + worker.host + '&port=8901&autoconnect=1&resize=downscale'}}">
</iframe>
</div>
</div>
</div>

31
webconsole/brozzler-webconsole/templates/index.html Normal file
View file

@ -0,0 +1,31 @@
<!doctype html>
<html lang="en" ng-app="brozzlerConsoleApp">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
<meta name="apple-mobile-web-app-capable" content="yes" />
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<title>Brozzler Console</title>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.5/css/bootstrap.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.5/css/bootstrap-theme.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.4.0/css/font-awesome.css">
<script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.4.6/angular.js"></script>
<script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.4.6/angular-route.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/ngInfiniteScroll/1.2.1/ng-infinite-scroll.js"></script>
<script src="/static/js/app.js"></script>
<style>
body { padding-top: 1rem; }
.thumbnail:focus, .thumbnail:hover { text-decoration: none; }
.thumbnail { word-wrap: break-word; }
.bigstats { font-size:125% }
.bigstats .fa { font-size:150% }
.bigstats .stat { padding:2rem }
</style>
</head>
<body role="document">
<div ng-view class="container" role="main">
</div>
</body>
</html>