Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-20 12:54:23 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

add info to display of jobless sites in brozzler-webconsole; fix creation of "least_hops" index on the rethinkdb table "pages"

Browse source
This commit is contained in:
Noah Levitt 2016-08-09 11:24:58 -07:00
parent 74b229cfb0
commit f671cf4f11
4 changed files with 13 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

4
brozzler/easy.py
View file

@ -1,7 +1,7 @@
#!/usr/bin/env python #!/usr/bin/env python
''' '''
brozzler-easy - brozzler-worker, warcprox, and pywb all working together in a brozzler-easy - brozzler-worker, warcprox, pywb, and brozzler-webconsole all
single process working together in a single process
Copyright (C) 2016 Internet Archive Copyright (C) 2016 Internet Archive

2
brozzler/frontier.py
View file

@ -69,7 +69,7 @@ class RethinkDbFrontier:
self.r.table("pages").index_create( self.r.table("pages").index_create(
"least_hops", [ "least_hops", [
self.r.row["site_id"], self.r.row["brozzle_count"], self.r.row["site_id"], self.r.row["brozzle_count"],
self.r.row["hops_from_seed"]]) self.r.row["hops_from_seed"]]).run()
if not "jobs" in tables: if not "jobs" in tables:
self.logger.info( self.logger.info(
"creating rethinkdb table 'jobs' in database %s", "creating rethinkdb table 'jobs' in database %s",

12
brozzler/webconsole/static/partials/home.html
View file

@ -73,12 +73,18 @@
<thead> <thead>
<tr> <tr>
<th>id</th> <th>id</th>
<th>status</th>
<th>started</th>
<th>seed url</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
<tr ng-repeat="site in joblessSites"> <tr ng-repeat="site in joblessSites">
<td><a href="/sites/{{site.id}}">{{site.id}}</a></td> <td><a href="/sites/{{site.id}}">{{site.id}}</a></td>
</tr> <td>{{site.status}}</td>
<td>{{site.start_time}}</td>
<td>{{site.seed}}</td>
</tr>
</tbody> </tbody>
</table> </table>
</div> </div>

2
setup.py
View file

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1b6.dev74', version='1.1b6.dev75',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',