Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-08-08 06:22:23 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'ARI-5617' into qa

Browse source
This commit is contained in:
Barbara Miller 2018-03-13 16:34:21 -07:00
commit d371db8166
5 changed files with 12 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

2
brozzler/__init__.py
View file

@ -264,7 +264,7 @@ def jinja2_environment(behaviors_dir=None):
'js-templates')) 'js-templates'))
else: else:
_loader=jinja2.PackageLoader('brozzler', 'js-templates') _loader=jinja2.PackageLoader('brozzler', 'js-templates')
_jinja2_env = jinja2.Environment(loader=_loader) _jinja2_env = jinja2.Environment(loader=_loader, auto_reload=False)
_jinja2_env.filters['json'] = json.dumps _jinja2_env.filters['json'] = json.dumps
return _jinja2_env return _jinja2_env

7
brozzler/behaviors.yaml
View file

@ -21,6 +21,13 @@
url_regex: '^https?://(?:www\.)?facebook\.com/.*$' url_regex: '^https?://(?:www\.)?facebook\.com/.*$'
behavior_js_template: facebook.js behavior_js_template: facebook.js
request_idle_timeout_sec: 30 request_idle_timeout_sec: 30
-
url_regex: '^https?://americaspresidents\.si\.edu/gallery.*$'
behavior_js_template: umbraBehavior.js.j2
default_parameters:
actions:
- selector: div.see-more, li.next
idcheck: false
- -
url_regex: '^https?://(?:www\.)?marquette\.edu/.*$' url_regex: '^https?://(?:www\.)?marquette\.edu/.*$'
behavior_js_template: marquette_edu.js behavior_js_template: marquette_edu.js

2
brozzler/chrome.py
View file

@ -158,7 +158,7 @@ class Chrome:
'--disable-background-networking', '--disable-background-networking',
'--disable-renderer-backgrounding', '--disable-hang-monitor', '--disable-renderer-backgrounding', '--disable-hang-monitor',
'--disable-background-timer-throttling', '--mute-audio', '--disable-background-timer-throttling', '--mute-audio',
'--disable-web-sockets', '--disable-cache', '--disable-web-sockets', '--disable-cache', '--single-process',
'--window-size=1100,900', '--no-default-browser-check', '--window-size=1100,900', '--no-default-browser-check',
'--disable-first-run-ui', '--no-first-run', '--disable-first-run-ui', '--no-first-run',
'--homepage=about:blank', '--disable-direct-npapi-requests', '--homepage=about:blank', '--disable-direct-npapi-requests',

3
brozzler/js-templates/umbraBehavior.js.j2
View file

@ -33,6 +33,7 @@ class UmbraBehavior {
// should match older default and simpleclicks behavior, and more // should match older default and simpleclicks behavior, and more
var k = this.index; var k = this.index;
var selector = this.actions[k].selector; var selector = this.actions[k].selector;
var idCheck = this.actions[k].idcheck ? this.actions[k].idcheck : true;
var action = this.actions[k].do ? this.actions[k].do : 'click'; var action = this.actions[k].do ? this.actions[k].do : 'click';
var closeSelector = this.actions[k].closeSelector ? this.actions[k].closeSelector : null; var closeSelector = this.actions[k].closeSelector ? this.actions[k].closeSelector : null;
var didSomething = false; var didSomething = false;
@ -62,7 +63,7 @@ class UmbraBehavior {
continue; continue;
} }
for ( var i = 0; i < doTargetsLength; i++) { for ( var i = 0; i < doTargetsLength; i++) {
if (this.alreadyDone.indexOf(doTargets[i]) > -1) { if idCheck and (this.alreadyDone.indexOf(doTargets[i]) > -1) {
continue; continue;
} }
if (!this.isVisible(doTargets[i])) { if (!this.isVisible(doTargets[i])) {

2
setup.py
View file

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1b13.dev286', version='1.1b13.dev288',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',