Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-20 04:44:12 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

avoid js errors in case site or job is not configured to keep stats

Browse source
This commit is contained in:
Noah Levitt 2017-01-20 23:36:23 -08:00
parent 65f818e901
commit 095456aa27
2 changed files with 24 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

51
brozzler/dashboard/static/js/app.js
View file

@ -1,7 +1,7 @@
/* /*
* brozzler/dashboard/static/js/app.js - brozzler dashboard angularjs code * brozzler/dashboard/static/js/app.js - brozzler dashboard angularjs code
* *
* Copyright (C) 2014-2016 Internet Archive * Copyright (C) 2014-2017 Internet Archive
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -96,16 +96,12 @@ brozzlerControllers.controller("WorkersListController", ["$scope", "$http",
function statsSuccessCallback(site, bucket) { function statsSuccessCallback(site, bucket) {
return function(data) { return function(data) {
// console.log("site = ", site);
// console.log("/api/stats/" + bucket + " = ", data);
site.stats = data; site.stats = data;
} }
} }
function pageCountSuccessCallback(site, job) { function pageCountSuccessCallback(site, job) {
return function(data) { return function(data) {
// console.log("site = ", site);
// console.log("/api/sites/" + site.id + "/page_count = ", data);
site.page_count = data.count; site.page_count = data.count;
if (job) { if (job) {
job.page_count += data.count; job.page_count += data.count;
@ -115,8 +111,6 @@ function pageCountSuccessCallback(site, job) {
function queuedCountSuccessCallback(site, job) { function queuedCountSuccessCallback(site, job) {
return function(data) { return function(data) {
// console.log("site = ", site);
// console.log("/api/sites/" + site.id + "/queued_count = ", data);
site.queued_count = data.count; site.queued_count = data.count;
if (job) { if (job) {
job.queued_count += data.count; job.queued_count += data.count;
@ -129,41 +123,44 @@ function loadSiteStats($http, site, job) {
$http.get("/api/sites/" + site.id + "/queued_count").success(queuedCountSuccessCallback(site, job)); $http.get("/api/sites/" + site.id + "/queued_count").success(queuedCountSuccessCallback(site, job));
// look at Warcprox-Meta to find stats bucket // look at Warcprox-Meta to find stats bucket
for (var j = 0; j < site.warcprox_meta.stats.buckets.length; j++) { try {
var bucket = site.warcprox_meta.stats.buckets[j]; for (var j = 0; j < site.warcprox_meta.stats.buckets.length; j++) {
if (typeof(bucket) == "object") { var bucket = site.warcprox_meta.stats.buckets[j];
bucket = bucket["bucket"]; if (typeof(bucket) == "object") {
} bucket = bucket["bucket"];
if (bucket.indexOf("seed") >= 0) { }
// console.log("warcprox_meta.stats.buckets[" + j + "]=" + bucket); if (bucket.indexOf("seed") >= 0) {
$http.get("/api/stats/" + bucket).success(statsSuccessCallback(site, bucket)); $http.get("/api/stats/" + bucket).success(statsSuccessCallback(site, bucket));
}
} }
} catch (e) {
// no stats bucket for this site
} }
} }
brozzlerControllers.controller("JobController", ["$scope", "$routeParams", "$http", brozzlerControllers.controller("JobController", ["$scope", "$routeParams", "$http",
function($scope, $routeParams, $http) { function($scope, $routeParams, $http) {
$scope.show_yaml = false; $scope.show_yaml = false;
// console.log('JobController');
$http.get("/api/config").success(function(data) { $http.get("/api/config").success(function(data) {
$scope.config = data.config; $scope.config = data.config;
}); });
$http.get("/api/jobs/" + $routeParams.id).success(function(data) { $http.get("/api/jobs/" + $routeParams.id).success(function(data) {
$scope.job = data; $scope.job = data;
$scope.job.page_count = $scope.job.queued_count = 0; $scope.job.page_count = $scope.job.queued_count = 0;
// console.log("job=", $scope.job); try {
var bucket = $scope.job.conf.warcprox_meta.stats.buckets[0]; var bucket = $scope.job.conf.warcprox_meta.stats.buckets[0];
if (typeof(bucket) == "object") { if (typeof(bucket) == "object") {
bucket = bucket["bucket"]; bucket = bucket["bucket"];
}
$http.get("/api/stats/" + bucket).success(function(data) {
$scope.job.stats = data;
});
} catch (e) {
// no stats bucket for this job
} }
$http.get("/api/stats/" + bucket).success(function(data) {
$scope.job.stats = data;
// console.log("job stats=", $scope.job.stats);
});
$http.get("/api/jobs/" + $routeParams.id + "/sites").success(function(data) { $http.get("/api/jobs/" + $routeParams.id + "/sites").success(function(data) {
$scope.sites = data.sites; $scope.sites = data.sites;
// console.log("sites=", $scope.sites);
for (var i = 0; i < $scope.sites.length; i++) { for (var i = 0; i < $scope.sites.length; i++) {
loadSiteStats($http, $scope.sites[i], $scope.job); loadSiteStats($http, $scope.sites[i], $scope.job);
} }
@ -180,7 +177,6 @@ brozzlerControllers.controller("SiteController", ["$scope", "$routeParams", "$ht
$scope.loading = false; $scope.loading = false;
$scope.pages = []; $scope.pages = [];
$window.addEventListener("scroll", function() { $window.addEventListener("scroll", function() {
// console.log("window.scrollTop=" + window.scrollTop + " window.offsetHeight=" + window.offsetHeight + " window.scrollHeight=" + window.scrollHeight);
if ($window.innerHeight + $window.scrollY + 50 >= window.document.documentElement.scrollHeight) { if ($window.innerHeight + $window.scrollY + 50 >= window.document.documentElement.scrollHeight) {
loadMorePages(); loadMorePages();
} }
@ -191,10 +187,8 @@ brozzlerControllers.controller("SiteController", ["$scope", "$routeParams", "$ht
return; return;
$scope.loading = true; $scope.loading = true;
// console.log("load more! start=" + start);
$http.get("/api/site/" + $routeParams.id + "/pages?start=" + start + "&end=" + (start+90)).then(function(response) { $http.get("/api/site/" + $routeParams.id + "/pages?start=" + start + "&end=" + (start+90)).then(function(response) {
$scope.pages = $scope.pages.concat(response.data.pages); $scope.pages = $scope.pages.concat(response.data.pages);
// console.log("pages = ", $scope.pages);
start += response.data.pages.length; start += response.data.pages.length;
$scope.loading = false; $scope.loading = false;
}, function(reason) { }, function(reason) {
@ -209,7 +203,6 @@ brozzlerControllers.controller("SiteController", ["$scope", "$routeParams", "$ht
$http.get("/api/site/" + $routeParams.id).success(function(data) { $http.get("/api/site/" + $routeParams.id).success(function(data) {
$scope.site = data; $scope.site = data;
loadSiteStats($http, $scope.site); loadSiteStats($http, $scope.site);
// console.log("site = ", $scope.site);
}); });
$http.get("/api/site/" + $routeParams.id + "/yaml").success(function(data) { $http.get("/api/site/" + $routeParams.id + "/yaml").success(function(data) {
$scope.site_yaml = data; $scope.site_yaml = data;

4
setup.py
View file

@ -2,7 +2,7 @@
''' '''
setup.py - brozzler setup script setup.py - brozzler setup script
Copyright (C) 2014-2016 Internet Archive Copyright (C) 2014-2017 Internet Archive
Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.
@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1b9.dev170', version='1.1b9.dev171',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',