mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-02-23 16:19:49 -05:00
proof of concept presenting workers in web console with novnc
This commit is contained in:
Noah Levitt
parent
a0f4fd449c
commit
7b39ba021b
3
.gitmodules
vendored
Normal file
3
.gitmodules
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
[submodule "webconsole/static/noVNC"]
|
||||
path = webconsole/static/noVNC
|
||||
url = git@github.com:kanaka/noVNC.git
|
||||
@ -34,18 +34,21 @@ _status() {
|
||||
|
||||
_stop() {
|
||||
if _status ; then
|
||||
set -x
|
||||
pkill -f /home/nlevitt/workspace/pygwb/pygwb-ve27/bin/gunicorn
|
||||
pkill -f /home/nlevitt/workspace/ait5/scripts/ait-brozzler-boss.py
|
||||
pkill -f 0.0.0.0:8888
|
||||
# pkill -f /home/nlevitt/workspace/brozzler/brozzler-ve34/bin/brozzler-worker
|
||||
for node in aidata{400,400-bu,401-bu} ; do
|
||||
pkill -f app=.*brozzler-webconsole.py
|
||||
set +x
|
||||
for node in aidata{400,401}{,-bu} ; do
|
||||
container_id=$(ssh $node docker ps --filter=image=internetarchive/brozzler-worker --filter=status=running --format='{{.ID}}')
|
||||
[ -n "$container_id" ] && (set -x ; ssh $node docker stop --time=60 $container_id )
|
||||
done
|
||||
pkill -f app=.*brozzler-webconsole.py
|
||||
fi
|
||||
|
||||
set -x
|
||||
ssh wbgrp-svc111 pkill -f /home/nlevitt/workspace/warcprox/warcprox-ve34/bin/warcprox
|
||||
set +x
|
||||
|
||||
if _status > /dev/null ; then
|
||||
while _status > /dev/null ; do sleep 0.5 ; done
|
||||
@ -72,7 +75,8 @@ EOF
|
||||
set -e
|
||||
sudo umount /1/brzl/warcs
|
||||
mv -v /1/brzl /tmp/brzl.$tstamp
|
||||
mkdir -vp /1/brzl/{warcs,logs}
|
||||
mkdir -vp /1/brzl/{logs,warcs}
|
||||
sudo chown -v archiveit /1/brzl/warcs
|
||||
# chgrp -v archiveit /1/brzl/warcs/ && chmod g+w /1/brzl/warcs
|
||||
ssh wbgrp-svc111 mv -v "/1/brzl/warcs /tmp/brzl-warcs.$tstamp && mkdir -vp /1/brzl/warcs"
|
||||
sudo -H -u archiveit sshfs wbgrp-svc111:/1/brzl/warcs /1/brzl/warcs -o nonempty,ro,allow_other
|
||||
@ -91,12 +95,12 @@ start_brozzler_boss() {
|
||||
|
||||
start_brozzler_workers() {
|
||||
echo $0: starting brozzler-workers
|
||||
for node in aidata{400,400-bu,401,401-bu} ; do
|
||||
for node in aidata{400,401}{,-bu} ; do
|
||||
(
|
||||
set -x
|
||||
ssh $node "docker --version || curl -sSL https://get.docker.com/ | sh && sudo usermod -aG docker $USER"
|
||||
ssh $node 'docker build -t internetarchive/brozzler-worker /home/nlevitt/workspace/brozzler/docker'
|
||||
ssh -fn $node 'docker run --rm internetarchive/brozzler-worker /sbin/my_init -- setuser brozzler bash -c "DISPLAY=:1 brozzler-worker --rethinkdb-servers=wbgrp-svc036,wbgrp-svc020,wbgrp-svc035 --rethinkdb-db=archiveit_brozzler --max-browsers=10"' &>> /1/brzl/logs/brozzler-worker-$node.out
|
||||
ssh -fn $node 'docker run -t --rm -p 8901:8901 -p 5901:5901 internetarchive/brozzler-worker /sbin/my_init -- setuser brozzler bash -c "DISPLAY=:1 brozzler-worker --rethinkdb-servers=wbgrp-svc036,wbgrp-svc020,wbgrp-svc035 --rethinkdb-db=archiveit_brozzler --max-browsers=10"' &>> /1/brzl/logs/brozzler-worker-$node.out
|
||||
sleep 5
|
||||
)
|
||||
done
|
||||
|
||||
@ -9,25 +9,18 @@ RUN apt-get -y install chromium-browser
|
||||
RUN apt-get -y install xfonts-base fonts-arphic-bkai00mp fonts-arphic-bsmi00lp fonts-arphic-gbsn00lp fonts-arphic-gkai00mp fonts-arphic-ukai fonts-farsiweb fonts-nafees fonts-sil-abyssinica fonts-sil-ezra fonts-sil-padauk fonts-unfonts-extra fonts-unfonts-core ttf-indic-fonts fonts-thai-tlwg fonts-lklug-sinhala
|
||||
RUN apt-get -y install python3-pip git vlc
|
||||
RUN apt-get -y install libjpeg-turbo8-dev zlib1g-dev
|
||||
RUN pip3 install websockify
|
||||
|
||||
RUN adduser --disabled-password --gecos="Charlie Brozzler" brozzler
|
||||
|
||||
RUN mkdir -vp /etc/service/vncserver
|
||||
ADD vncserver.sh /etc/service/vncserver/run
|
||||
|
||||
RUN mkdir /etc/service/vlc-screencast
|
||||
ADD vlc-screencast.sh /etc/service/vlc-screencast/run
|
||||
RUN mkdir -vp /etc/service/vnc-websock
|
||||
ADD vnc-websock.sh /etc/service/vnc-websock/run
|
||||
|
||||
RUN adduser --disabled-password --gecos="Charlie Brozzler" brozzler
|
||||
|
||||
EXPOSE 5901
|
||||
EXPOSE 5901 8901
|
||||
EXPOSE 8080
|
||||
|
||||
RUN pip3 install -i http://crawl342.us.archive.org:9000/nlevitt/dev/+simple/ git+https://github.com/nlevitt/brozzler.git
|
||||
|
||||
#
|
||||
# ENTRYPOINT ["/sbin/my_init", "--", "setuser", "brozzler", "brozzler-worker"]
|
||||
#
|
||||
|
||||
#
|
||||
# docker run --rm --publish=8080:8080 internetarchive/brozzler-worker /sbin/my_init -- setuser brozzler brozzler-worker --rethinkdb-servers=foo,bar --max-browsers=5
|
||||
#
|
||||
|
||||
|
||||
@ -1,2 +0,0 @@
|
||||
#!/bin/sh
|
||||
DISPLAY=:1 exec setuser brozzler cvlc screen:// :screen-fps=3 :screen-caching=100 ':sout=#transcode{vcodec=theo,vb=800,scale=0.5,acodec=none}:http{mux=ogg,dst=:8080/screen}' :sout-keep >> /tmp/vlc-screencast.out 2>&1
|
||||
@ -1,6 +1,11 @@
|
||||
#!/bin/sh
|
||||
#!/bin/bash
|
||||
|
||||
# https://github.com/phusion/baseimage-docker#adding-additional-daemons
|
||||
# /usr/bin/vncserver backgrounds the Xvnc4 process, so we run Xvnc4 directly
|
||||
|
||||
exec setuser brozzler Xvnc4 :1 -desktop brozzler@`hostname`:1 -auth /tmp/Xauthority.brozzler -geometry 1600x1000 -depth 24 -rfbwait 0 -nolisten tcp -rfbport 5901 -pn -fp /usr/X11R6/lib/X11/fonts/Type1/,/usr/X11R6/lib/X11/fonts/Speedo/,/usr/X11R6/lib/X11/fonts/misc/,/usr/X11R6/lib/X11/fonts/75dpi/,/usr/X11R6/lib/X11/fonts/100dpi/,/usr/share/fonts/X11/misc/,/usr/share/fonts/X11/Type1/,/usr/share/fonts/X11/75dpi/,/usr/share/fonts/X11/100dpi/ -co /etc/X11/rgb >> /tmp/`hostname`:1.log 2>&1
|
||||
# password_file=/tmp/vnc-passwd
|
||||
# /bin/echo -ne '\x95\x3f\x23\x7a\x76\x2a\x05\x89' > $password_file
|
||||
# exec setuser brozzler Xvnc4 :1 -desktop brozzler@`hostname`:1 -auth /tmp/Xauthority.brozzler -geometry 1600x1000 -depth 24 -rfbwait 0 -nolisten tcp -rfbport 5901 -rfbauth $password_file -pn -fp /usr/share/fonts/X11/misc/ -co /etc/X11/rgb >> /tmp/`hostname`:1.log 2>&1
|
||||
|
||||
exec setuser brozzler Xvnc4 :1 -desktop brozzler@`hostname`:1 -auth /tmp/Xauthority.brozzler -geometry 1600x1000 -depth 24 -rfbwait 0 -nolisten tcp -rfbport 5901 -SecurityTypes None -pn -fp /usr/share/fonts/X11/misc/ -co /etc/X11/rgb AcceptCutText=0 AcceptPointerEvents=0 AcceptKeyEvents=0 >> /tmp/`hostname`:1.log 2>&1
|
||||
|
||||
|
||||
@ -68,6 +68,11 @@ def job(job_id):
|
||||
job_ = r.table("jobs").get(job_id).run()
|
||||
return flask.jsonify(job_)
|
||||
|
||||
@app.route("/api/workers")
|
||||
def workers():
|
||||
workers_ = [{"host":host,"vnc_websocket_port":8901} for host in ["aidata400", "aidata401", "aidata400-bu", "aidata401-bu"]]
|
||||
return flask.jsonify(workers=workers_)
|
||||
|
||||
@app.route("/api/jobs")
|
||||
def jobs():
|
||||
jobs_ = list(r.table("jobs").run())
|
||||
|
||||
@ -12,7 +12,6 @@
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/angular.js/1.4.6/angular-route.js"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/ngInfiniteScroll/1.2.1/ng-infinite-scroll.js"></script>
|
||||
<script src="/static/js/app.js"></script>
|
||||
<script src="/static/js/controllers.js"></script>
|
||||
<style>
|
||||
body { padding-top: 1rem; }
|
||||
.thumbnail:focus, .thumbnail:hover { text-decoration: none; }
|
||||
|
||||
@ -8,6 +8,10 @@ var brozzlerConsoleApp = angular.module("brozzlerConsoleApp", [
|
||||
brozzlerConsoleApp.config(["$routeProvider", "$locationProvider",
|
||||
function($routeProvider, $locationProvider) {
|
||||
$routeProvider.
|
||||
when("/workers", {
|
||||
templateUrl: "/static/partials/workers.html",
|
||||
controller: "WorkersListController"
|
||||
}).
|
||||
when("/jobs", {
|
||||
templateUrl: "/static/partials/jobs.html",
|
||||
controller: "JobsListController"
|
||||
@ -46,3 +50,125 @@ brozzlerConsoleApp.filter("byteformat", function() {
|
||||
return result;
|
||||
}
|
||||
});
|
||||
|
||||
var brozzlerControllers = angular.module("brozzlerControllers", []);
|
||||
|
||||
brozzlerControllers.controller("JobsListController", ["$scope", "$http",
|
||||
function($scope, $http) {
|
||||
$http.get("/api/jobs").success(function(data) {
|
||||
$scope.jobs = data.jobs;
|
||||
});
|
||||
}]);
|
||||
|
||||
brozzlerControllers.controller("WorkersListController", ["$scope", "$http",
|
||||
function($scope, $http) {
|
||||
$http.get("/api/workers").success(function(data) {
|
||||
$scope.workers = data.workers;
|
||||
});
|
||||
}]);
|
||||
|
||||
function statsSuccessCallback(site, bucket) {
|
||||
return function(data) {
|
||||
// console.log("site = ", site);
|
||||
// console.log("/api/stats/" + bucket + " = ", data);
|
||||
site.stats = data;
|
||||
}
|
||||
}
|
||||
|
||||
function pageCountSuccessCallback(site, job) {
|
||||
return function(data) {
|
||||
// console.log("site = ", site);
|
||||
// console.log("/api/sites/" + site.id + "/page_count = ", data);
|
||||
site.page_count = data.count;
|
||||
if (job) {
|
||||
job.page_count += data.count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function queuedCountSuccessCallback(site, job) {
|
||||
return function(data) {
|
||||
// console.log("site = ", site);
|
||||
// console.log("/api/sites/" + site.id + "/queued_count = ", data);
|
||||
site.queued_count = data.count;
|
||||
if (job) {
|
||||
job.queued_count += data.count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function loadSiteStats($http, site, job) {
|
||||
$http.get("/api/sites/" + site.id + "/page_count").success(pageCountSuccessCallback(site, job));
|
||||
$http.get("/api/sites/" + site.id + "/queued_count").success(queuedCountSuccessCallback(site, job));
|
||||
|
||||
// parse Warcprox-Meta to find stats bucket
|
||||
var warcprox_meta = angular.fromJson(site.extra_headers["Warcprox-Meta"]);
|
||||
for (var j = 0; j < warcprox_meta.stats.buckets.length; j++) {
|
||||
if (warcprox_meta.stats.buckets[j].indexOf("seed") >= 0) {
|
||||
var bucket = warcprox_meta.stats.buckets[j];
|
||||
// console.log("warcprox_meta.stats.buckets[" + j + "]=" + bucket);
|
||||
$http.get("/api/stats/" + bucket).success(statsSuccessCallback(site, bucket));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
brozzlerControllers.controller("JobController", ["$scope", "$routeParams", "$http",
|
||||
function($scope, $routeParams, $http) {
|
||||
$http.get("/api/jobs/" + $routeParams.id).success(function(data) {
|
||||
$scope.job = data;
|
||||
$scope.job.page_count = $scope.job.queued_count = 0;
|
||||
// console.log("job=", $scope.job);
|
||||
$http.get("/api/stats/" + $scope.job.conf.warcprox_meta.stats.buckets[0]).success(function(data) {
|
||||
$scope.job.stats = data;
|
||||
// console.log("job stats=", $scope.job.stats);
|
||||
});
|
||||
|
||||
$http.get("/api/jobs/" + $routeParams.id + "/sites").success(function(data) {
|
||||
$scope.sites = data.sites;
|
||||
// console.log("sites=", $scope.sites);
|
||||
for (var i = 0; i < $scope.sites.length; i++) {
|
||||
loadSiteStats($http, $scope.sites[i], $scope.job);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
}]);
|
||||
|
||||
brozzlerControllers.controller("SiteController", ["$scope", "$routeParams", "$http", "$window",
|
||||
function($scope, $routeParams, $http, $window) {
|
||||
var start = 0;
|
||||
$scope.loading = false;
|
||||
$scope.pages = [];
|
||||
$window.addEventListener("scroll", function() {
|
||||
// console.log("window.scrollTop=" + window.scrollTop + " window.offsetHeight=" + window.offsetHeight + " window.scrollHeight=" + window.scrollHeight);
|
||||
if ($window.innerHeight + $window.scrollY + 50 >= window.document.documentElement.scrollHeight) {
|
||||
loadMorePages();
|
||||
}
|
||||
});
|
||||
|
||||
var loadMorePages = function() {
|
||||
if ($scope.loading)
|
||||
return;
|
||||
$scope.loading = true;
|
||||
|
||||
// console.log("load more! start=" + start);
|
||||
$http.get("/api/site/" + $routeParams.id + "/pages?start=" + start + "&end=" + (start+90)).then(function(response) {
|
||||
$scope.pages = $scope.pages.concat(response.data.pages);
|
||||
// console.log("pages = ", $scope.pages);
|
||||
start += response.data.pages.length;
|
||||
$scope.loading = false;
|
||||
}, function(reason) {
|
||||
$scope.loading = false;
|
||||
});
|
||||
|
||||
};
|
||||
|
||||
$http.get("/api/site/" + $routeParams.id).success(function(data) {
|
||||
$scope.site = data;
|
||||
loadSiteStats($http, $scope.site);
|
||||
// console.log("site = ", $scope.site);
|
||||
});
|
||||
|
||||
loadMorePages();
|
||||
}]);
|
||||
|
||||
|
||||
@ -1,116 +0,0 @@
|
||||
"use strict";
|
||||
|
||||
var brozzlerControllers = angular.module("brozzlerControllers", []);
|
||||
|
||||
brozzlerControllers.controller("JobsListController", ["$scope", "$http",
|
||||
function($scope, $http) {
|
||||
$http.get("/api/jobs").success(function(data) {
|
||||
$scope.jobs = data.jobs;
|
||||
});
|
||||
}]);
|
||||
|
||||
function statsSuccessCallback(site, bucket) {
|
||||
return function(data) {
|
||||
// console.log("site = ", site);
|
||||
// console.log("/api/stats/" + bucket + " = ", data);
|
||||
site.stats = data;
|
||||
}
|
||||
}
|
||||
|
||||
function pageCountSuccessCallback(site, job) {
|
||||
return function(data) {
|
||||
// console.log("site = ", site);
|
||||
// console.log("/api/sites/" + site.id + "/page_count = ", data);
|
||||
site.page_count = data.count;
|
||||
if (job) {
|
||||
job.page_count += data.count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function queuedCountSuccessCallback(site, job) {
|
||||
return function(data) {
|
||||
// console.log("site = ", site);
|
||||
// console.log("/api/sites/" + site.id + "/queued_count = ", data);
|
||||
site.queued_count = data.count;
|
||||
if (job) {
|
||||
job.queued_count += data.count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function loadSiteStats($http, site, job) {
|
||||
$http.get("/api/sites/" + site.id + "/page_count").success(pageCountSuccessCallback(site, job));
|
||||
$http.get("/api/sites/" + site.id + "/queued_count").success(queuedCountSuccessCallback(site, job));
|
||||
|
||||
// parse Warcprox-Meta to find stats bucket
|
||||
var warcprox_meta = angular.fromJson(site.extra_headers["Warcprox-Meta"]);
|
||||
for (var j = 0; j < warcprox_meta.stats.buckets.length; j++) {
|
||||
if (warcprox_meta.stats.buckets[j].indexOf("seed") >= 0) {
|
||||
var bucket = warcprox_meta.stats.buckets[j];
|
||||
// console.log("warcprox_meta.stats.buckets[" + j + "]=" + bucket);
|
||||
$http.get("/api/stats/" + bucket).success(statsSuccessCallback(site, bucket));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
brozzlerControllers.controller("JobController", ["$scope", "$routeParams", "$http",
|
||||
function($scope, $routeParams, $http) {
|
||||
$http.get("/api/jobs/" + $routeParams.id).success(function(data) {
|
||||
$scope.job = data;
|
||||
$scope.job.page_count = $scope.job.queued_count = 0;
|
||||
// console.log("job=", $scope.job);
|
||||
$http.get("/api/stats/" + $scope.job.conf.warcprox_meta.stats.buckets[0]).success(function(data) {
|
||||
$scope.job.stats = data;
|
||||
// console.log("job stats=", $scope.job.stats);
|
||||
});
|
||||
|
||||
$http.get("/api/jobs/" + $routeParams.id + "/sites").success(function(data) {
|
||||
$scope.sites = data.sites;
|
||||
// console.log("sites=", $scope.sites);
|
||||
for (var i = 0; i < $scope.sites.length; i++) {
|
||||
loadSiteStats($http, $scope.sites[i], $scope.job);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
}]);
|
||||
|
||||
brozzlerControllers.controller("SiteController", ["$scope", "$routeParams", "$http", "$window",
|
||||
function($scope, $routeParams, $http, $window) {
|
||||
var start = 0;
|
||||
$scope.loading = false;
|
||||
$scope.pages = [];
|
||||
$window.addEventListener("scroll", function() {
|
||||
// console.log("window.scrollTop=" + window.scrollTop + " window.offsetHeight=" + window.offsetHeight + " window.scrollHeight=" + window.scrollHeight);
|
||||
if ($window.innerHeight + $window.scrollY + 50 >= window.document.documentElement.scrollHeight) {
|
||||
loadMorePages();
|
||||
}
|
||||
});
|
||||
|
||||
var loadMorePages = function() {
|
||||
if ($scope.loading)
|
||||
return;
|
||||
$scope.loading = true;
|
||||
|
||||
// console.log("load more! start=" + start);
|
||||
$http.get("/api/site/" + $routeParams.id + "/pages?start=" + start + "&end=" + (start+90)).then(function(response) {
|
||||
$scope.pages = $scope.pages.concat(response.data.pages);
|
||||
// console.log("pages = ", $scope.pages);
|
||||
start += response.data.pages.length;
|
||||
$scope.loading = false;
|
||||
}, function(reason) {
|
||||
$scope.loading = false;
|
||||
});
|
||||
|
||||
};
|
||||
|
||||
$http.get("/api/site/" + $routeParams.id).success(function(data) {
|
||||
$scope.site = data;
|
||||
loadSiteStats($http, $scope.site);
|
||||
// console.log("site = ", $scope.site);
|
||||
});
|
||||
|
||||
loadMorePages();
|
||||
}]);
|
||||
|
||||
1
webconsole/static/noVNC
Submodule
1
webconsole/static/noVNC
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit 6a90803feb124791960e3962e328aa3cfb729aeb
|
||||
24
webconsole/static/partials/workers.html
Normal file
24
webconsole/static/partials/workers.html
Normal file
@ -0,0 +1,24 @@
|
||||
<ol class="breadcrumb">
|
||||
<li class="active">Workers</li>
|
||||
</ol>
|
||||
|
||||
<div class="page-header">
|
||||
<h1>Brozzler
|
||||
<a href="/"><img src="/static/brozzler.svg" style="height:1.5em;float:right"></a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<h2>Workers</h2>
|
||||
|
||||
<div class="row">
|
||||
<div class="col-sm-12" ng-repeat="worker in workers">
|
||||
<!--
|
||||
<iframe style="width:800px;height:550px"
|
||||
ng-src="{{'/static/noVNC/vnc.html?host=' + worker.host + '&port=' + worker.vnc_websocket_port + '&autoconnect=1&resize=downscale'}}">
|
||||
{{worker}}
|
||||
</iframe>
|
||||
-->
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
Loading…
x
Reference in New Issue
Block a user