mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-04-19 23:35:54 -04:00
un-hardcode some stuff in webconsole, load from environment variables instead
This commit is contained in:
Noah Levitt
parent
35b713a2e7
commit
72a94ed816
4
.gitmodules
vendored
4
.gitmodules
vendored
@ -1,3 +1,3 @@
|
||||
[submodule "webconsole/static/noVNC"]
|
||||
path = webconsole/static/noVNC
|
||||
[submodule "webconsole/brozzler-webconsole/static/noVNC"]
|
||||
path = webconsole/brozzler-webconsole/static/noVNC
|
||||
url = https://github.com/kanaka/noVNC.git
|
||||
|
||||
@ -88,3 +88,6 @@ class Job(brozzler.BaseDictable):
|
||||
self.finished = finished
|
||||
self.stop_requested = stop_requested
|
||||
|
||||
def __str__(self):
|
||||
return 'Job(id=%s)' % self.id
|
||||
|
||||
|
||||
@ -3,19 +3,30 @@ import rethinkstuff
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import os
|
||||
|
||||
logging.basicConfig(stream=sys.stdout, level=logging.INFO,
|
||||
format="%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s")
|
||||
|
||||
app = flask.Flask(__name__)
|
||||
|
||||
r = rethinkstuff.Rethinker(["wbgrp-svc020", "wbgrp-svc035", "wbgrp-svc036"],
|
||||
db="archiveit_brozzler")
|
||||
# configure with environment variables
|
||||
SETTINGS= {
|
||||
'RETHINKDB_SERVERS': os.environ.get(
|
||||
'RETHINKDB_SERVERS', 'localhost').split(','),
|
||||
'RETHINKDB_DB': os.environ.get('RETHINKDB_DB', 'brozzler'),
|
||||
'WAYBACK_BASEURL': os.environ.get(
|
||||
'WAYBACK_BASEURL', 'http://wbgrp-svc107.us.archive.org:8091'),
|
||||
}
|
||||
r = rethinkstuff.Rethinker(
|
||||
SETTINGS['RETHINKDB_SERVERS'], db=SETTINGS['RETHINKDB_DB'])
|
||||
|
||||
@app.route("/api/sites/<site_id>/queued_count")
|
||||
@app.route("/api/site/<site_id>/queued_count")
|
||||
def queued_count(site_id):
|
||||
count = r.table("pages").between([site_id, 0, False, r.minval], [site_id, 0, False, r.maxval], index="priority_by_site").count().run()
|
||||
count = r.table("pages").between(
|
||||
[site_id, 0, False, r.minval], [site_id, 0, False, r.maxval],
|
||||
index="priority_by_site").count().run()
|
||||
return flask.jsonify(count=count)
|
||||
|
||||
@app.route("/api/sites/<site_id>/queue")
|
||||
@ -24,7 +35,9 @@ def queue(site_id):
|
||||
logging.info("flask.request.args=%s", flask.request.args)
|
||||
start = flask.request.args.get("start", 0)
|
||||
end = flask.request.args.get("end", start + 90)
|
||||
queue_ = r.table("pages").between([site_id, 0, False, r.minval], [site_id, 0, False, r.maxval], index="priority_by_site")[start:end].run()
|
||||
queue_ = r.table("pages").between(
|
||||
[site_id, 0, False, r.minval], [site_id, 0, False, r.maxval],
|
||||
index="priority_by_site")[start:end].run()
|
||||
return flask.jsonify(queue_=list(queue_))
|
||||
|
||||
@app.route("/api/sites/<site_id>/pages_count")
|
||||
@ -32,7 +45,10 @@ def queue(site_id):
|
||||
@app.route("/api/sites/<site_id>/page_count")
|
||||
@app.route("/api/site/<site_id>/page_count")
|
||||
def page_count(site_id):
|
||||
count = r.table("pages").between([site_id, 1, False, r.minval], [site_id, r.maxval, False, r.maxval], index="priority_by_site").count().run()
|
||||
count = r.table("pages").between(
|
||||
[site_id, 1, False, r.minval],
|
||||
[site_id, r.maxval, False, r.maxval],
|
||||
index="priority_by_site").count().run()
|
||||
return flask.jsonify(count=count)
|
||||
|
||||
@app.route("/api/sites/<site_id>/pages")
|
||||
@ -42,7 +58,10 @@ def pages(site_id):
|
||||
logging.info("flask.request.args=%s", flask.request.args)
|
||||
start = int(flask.request.args.get("start", 0))
|
||||
end = int(flask.request.args.get("end", start + 90))
|
||||
pages_ = r.table("pages").between([site_id, 1, False, r.minval], [site_id, r.maxval, False, r.maxval], index="priority_by_site")[start:end].run()
|
||||
pages_ = r.table("pages").between(
|
||||
[site_id, 1, False, r.minval],
|
||||
[site_id, r.maxval, False, r.maxval],
|
||||
index="priority_by_site")[start:end].run()
|
||||
return flask.jsonify(pages=list(pages_))
|
||||
|
||||
@app.route("/api/sites/<site_id>")
|
||||
@ -81,8 +100,13 @@ def services():
|
||||
@app.route("/api/jobs")
|
||||
def jobs():
|
||||
jobs_ = list(r.table("jobs").run())
|
||||
jobs_ = sorted(jobs_, key=lambda j: j['id'], reverse=True)
|
||||
return flask.jsonify(jobs=jobs_)
|
||||
|
||||
@app.route("/api/config")
|
||||
def config():
|
||||
return flask.jsonify(config=SETTINGS)
|
||||
|
||||
@app.route("/api/<path:path>")
|
||||
@app.route("/api", defaults={"path":""})
|
||||
def api404(path):
|
||||
@ -91,7 +115,7 @@ def api404(path):
|
||||
@app.route("/", defaults={"path": ""})
|
||||
@app.route("/<path:path>")
|
||||
def root(path):
|
||||
return app.send_static_file("index.html")
|
||||
return flask.render_template("index.html")
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(host="0.0.0.0", port=8081, debug=True)
|
||||
|
Before Width: | Height: | Size: 9.1 KiB After Width: | Height: | Size: 9.1 KiB |
@ -52,16 +52,22 @@ var brozzlerControllers = angular.module("brozzlerControllers", []);
|
||||
|
||||
brozzlerControllers.controller("HomeController", ["$scope", "$http",
|
||||
function($scope, $http) {
|
||||
$http.get("/api/jobs").success(function(data) {
|
||||
$scope.jobs = data.jobs;
|
||||
});
|
||||
$http.get("/api/services").success(function(data) {
|
||||
$scope.services = data.services;
|
||||
});
|
||||
$http.get("/api/config").success(function(data) {
|
||||
$scope.config = data.config;
|
||||
});
|
||||
$http.get("/api/jobs").success(function(data) {
|
||||
$scope.jobs = data.jobs;
|
||||
});
|
||||
$http.get("/api/services").success(function(data) {
|
||||
$scope.services = data.services;
|
||||
});
|
||||
}]);
|
||||
|
||||
brozzlerControllers.controller("WorkersListController", ["$scope", "$http",
|
||||
function($scope, $http) {
|
||||
$http.get("/api/config").success(function(data) {
|
||||
$scope.config = data.config;
|
||||
});
|
||||
$http.get("/api/workers").success(function(data) {
|
||||
$scope.workers = data.workers;
|
||||
});
|
||||
@ -80,9 +86,9 @@ function pageCountSuccessCallback(site, job) {
|
||||
// console.log("site = ", site);
|
||||
// console.log("/api/sites/" + site.id + "/page_count = ", data);
|
||||
site.page_count = data.count;
|
||||
if (job) {
|
||||
job.page_count += data.count;
|
||||
}
|
||||
if (job) {
|
||||
job.page_count += data.count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -91,9 +97,9 @@ function queuedCountSuccessCallback(site, job) {
|
||||
// console.log("site = ", site);
|
||||
// console.log("/api/sites/" + site.id + "/queued_count = ", data);
|
||||
site.queued_count = data.count;
|
||||
if (job) {
|
||||
job.queued_count += data.count;
|
||||
}
|
||||
if (job) {
|
||||
job.queued_count += data.count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -114,6 +120,10 @@ function loadSiteStats($http, site, job) {
|
||||
|
||||
brozzlerControllers.controller("JobController", ["$scope", "$routeParams", "$http",
|
||||
function($scope, $routeParams, $http) {
|
||||
console.log('JobController');
|
||||
$http.get("/api/config").success(function(data) {
|
||||
$scope.config = data.config;
|
||||
});
|
||||
$http.get("/api/jobs/" + $routeParams.id).success(function(data) {
|
||||
$scope.job = data;
|
||||
$scope.job.page_count = $scope.job.queued_count = 0;
|
||||
@ -127,11 +137,10 @@ brozzlerControllers.controller("JobController", ["$scope", "$routeParams", "$htt
|
||||
$scope.sites = data.sites;
|
||||
// console.log("sites=", $scope.sites);
|
||||
for (var i = 0; i < $scope.sites.length; i++) {
|
||||
loadSiteStats($http, $scope.sites[i], $scope.job);
|
||||
loadSiteStats($http, $scope.sites[i], $scope.job);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
}]);
|
||||
|
||||
brozzlerControllers.controller("SiteController", ["$scope", "$routeParams", "$http", "$window",
|
||||
@ -163,6 +172,9 @@ brozzlerControllers.controller("SiteController", ["$scope", "$routeParams", "$ht
|
||||
|
||||
};
|
||||
|
||||
$http.get("/api/config").success(function(data) {
|
||||
$scope.config = data.config;
|
||||
});
|
||||
$http.get("/api/site/" + $routeParams.id).success(function(data) {
|
||||
$scope.site = data;
|
||||
loadSiteStats($http, $scope.site);
|
||||
@ -40,7 +40,7 @@
|
||||
<h3>Sites</h3>
|
||||
<div class="col-sm-6 col-md-4" ng-repeat="site in sites">
|
||||
<a class="thumbnail" href="/sites/{{site.id}}">
|
||||
<img style="width:300px;height:190px" src="http://wbgrp-svc107.us.archive.org:8091/web/3/thumbnail:{{site.seed}}" alt="thumb">
|
||||
<img style="width:300px;height:190px" src="{{config.WAYBACK_BASEURL}}/3/thumbnail:{{site.seed}}" alt="thumb">
|
||||
<div class="caption">
|
||||
<h5>{{site.seed}}</h5>
|
||||
<!--
|
||||
@ -40,8 +40,8 @@
|
||||
<div class="col-sm-12">
|
||||
<h2>Pages</h2>
|
||||
<div class="col-sm-6 col-md-4" ng-repeat="page in pages">
|
||||
<a class="thumbnail" href="http://wbgrp-svc107.us.archive.org:8091/web/3/{{page.url}}">
|
||||
<img style="width:300px;height:190px" src="http://wbgrp-svc107.us.archive.org:8091/web/3/thumbnail:{{page.url}}" alt="thumb">
|
||||
<a class="thumbnail" href="{{config.WAYBACK_BASEURL}}/3/{{page.url}}">
|
||||
<img style="width:300px;height:190px" src="{{config.WAYBACK_BASEURL}}/3/thumbnail:{{page.url}}" alt="thumb">
|
||||
<div class="caption">
|
||||
<h5>{{page.url}}</h5>
|
||||
</div>
|
||||
@ -10,8 +10,8 @@
|
||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.5/css/bootstrap.css">
|
||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.5/css/bootstrap-theme.css">
|
||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.4.0/css/font-awesome.css">
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/angular.js/1.4.6/angular.js"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/angular.js/1.4.6/angular-route.js"></script>
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.4.6/angular.js"></script>
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.4.6/angular-route.js"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/ngInfiniteScroll/1.2.1/ng-infinite-scroll.js"></script>
|
||||
<script src="/static/js/app.js"></script>
|
||||
<style>
|
||||
Loading…
x
Reference in New Issue
Block a user