diff --git a/brozzler/behaviors.yaml b/brozzler/behaviors.yaml index f6eea9b..2000477 100644 --- a/brozzler/behaviors.yaml +++ b/brozzler/behaviors.yaml @@ -22,9 +22,10 @@ behavior_js_template: umbraBehavior.js.j2 request_idle_timeout_sec: 30 default_parameters: - interval: 500 + interval: 1000 actions: - - selector: 'a[data-testid="snapshot_footer_link"]' + - selector: a[data-testid="snapshot_footer_link"] + childSelector: i[class="_271o img sp_-vbjDsgypf1 sx_1de63f"] closeSelector: 'div._7lq1 > button' - url_regex: '^https?://(?:www\.)?facebook\.com/.*$' diff --git a/brozzler/js-templates/umbraBehavior.js.j2 b/brozzler/js-templates/umbraBehavior.js.j2 index cca01df..7ed736a 100644 --- a/brozzler/js-templates/umbraBehavior.js.j2 +++ b/brozzler/js-templates/umbraBehavior.js.j2 @@ -33,6 +33,7 @@ class UmbraBehavior { // should match older default and simpleclicks behavior, and more var k = this.index; var selector = this.actions[k].selector; + var childSelector = this.actions[k].childSelector; var repeatSameElement = this.actions[k].repeatSameElement ? this.actions[k].repeatSameElement : false; var limit = this.actions[k].limit ? this.actions[k].limit : false; var action = this.actions[k].do ? this.actions[k].do : 'click'; @@ -101,6 +102,17 @@ class UmbraBehavior { var where = this.aboveBelowOrOnScreen(doTargets[i]); if (where == 0) { this.doTarget(doTargets[i], action); + if (childSelector) { + var childSelectors = documents[j].querySelectorAll(childSelector); + while (childSelectors.length > 0) { + for (var i = 0; i < childSelectors.length; i++) { + if (this.isVisible(childSelectors[i])) { + childSelectors[i].click(); + } + } + childSelectors = documents[j].querySelectorAll(childSelector); + } + } didSomething = true; break; } else if (where > 0) { @@ -116,12 +128,12 @@ class UmbraBehavior { } if (!didSomething) { - if (somethingLeftAbove) { - window.scrollBy(0, -500); - this.idleSince = null; - } else if (somethingLeftBelow || ( (window.scrollY + window.innerHeight) < document.documentElement.scrollHeight)) { + if (somethingLeftBelow || ( (window.scrollY + window.innerHeight) < document.documentElement.scrollHeight)) { window.scrollBy(0, 200); this.idleSince = null; + } else if (somethingLeftAbove) { + window.scrollBy(0, -500); + this.idleSince = null; } else if (this.idleSince == null) { this.idleSince = Date.now(); } diff --git a/setup.py b/setup.py index d486e54..82a6739 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def find_package_data(package): setuptools.setup( name='brozzler', - version='1.5.11', + version='1.5.13', description='Distributed web crawling with browsers', url='https://github.com/internetarchive/brozzler', author='Noah Levitt',