Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 00:29:53 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

handle new bucket format in brozzler-webconsole

Browse Source
This commit is contained in:
Noah Levitt 2016-06-28 00:13:01 +00:00
parent 98915b3d86
commit cb4a16e58c
3 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
brozzler/webconsole/__init__.py
View File

@ -165,7 +165,6 @@ def api404(path):
def root(path): def root(path):
return flask.render_template("index.html") return flask.render_template("index.html")
import pdb; pdb.set_trace()
try: try:
import gunicorn.app.base import gunicorn.app.base
from gunicorn.six import iteritems from gunicorn.six import iteritems
@ -187,13 +186,11 @@ try:
return self.application return self.application
def run(**options): def run(**options):
import pdb; pdb.set_trace()
logging.info('running brozzler-webconsole using gunicorn') logging.info('running brozzler-webconsole using gunicorn')
GunicornBrozzlerWebConsole(app, options).run() GunicornBrozzlerWebConsole(app, options).run()
except ImportError: except ImportError:
def run(): def run():
import pdb; pdb.set_trace()
logging.info('running brozzler-webconsole using simple flask app.run') logging.info('running brozzler-webconsole using simple flask app.run')
app.run() app.run()

13
brozzler/webconsole/static/js/app.js
View File

@ -127,8 +127,11 @@ function loadSiteStats($http, site, job) {
// look at Warcprox-Meta to find stats bucket // look at Warcprox-Meta to find stats bucket
for (var j = 0; j < site.warcprox_meta.stats.buckets.length; j++) { for (var j = 0; j < site.warcprox_meta.stats.buckets.length; j++) {
if (site.warcprox_meta.stats.buckets[j].indexOf("seed") >= 0) { var bucket = site.warcprox_meta.stats.buckets[j];
var bucket = site.warcprox_meta.stats.buckets[j]; if (typeof(bucket) == "object") {
bucket = bucket["bucket"];
}
if (bucket.indexOf("seed") >= 0) {
// console.log("warcprox_meta.stats.buckets[" + j + "]=" + bucket); // console.log("warcprox_meta.stats.buckets[" + j + "]=" + bucket);
$http.get("/api/stats/" + bucket).success(statsSuccessCallback(site, bucket)); $http.get("/api/stats/" + bucket).success(statsSuccessCallback(site, bucket));
} }
@ -146,7 +149,11 @@ brozzlerControllers.controller("JobController", ["$scope", "$routeParams", "$htt
$scope.job = data; $scope.job = data;
$scope.job.page_count = $scope.job.queued_count = 0; $scope.job.page_count = $scope.job.queued_count = 0;
// console.log("job=", $scope.job); // console.log("job=", $scope.job);
$http.get("/api/stats/" + $scope.job.conf.warcprox_meta.stats.buckets[0]).success(function(data) { var bucket = $scope.job.conf.warcprox_meta.stats.buckets[0];
if (typeof(bucket) == "object") {
bucket = bucket["bucket"];
}
$http.get("/api/stats/" + bucket).success(function(data) {
$scope.job.stats = data; $scope.job.stats = data;
// console.log("job stats=", $scope.job.stats); // console.log("job stats=", $scope.job.stats);
}); });

2
setup.py
View File

@ -21,7 +21,7 @@ import setuptools
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1.dev22', version='1.1.dev23',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',