Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-20 23:56:34 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge branch 'senate-videos' into qa

Browse Source
This commit is contained in:
Barbara Miller 2019-08-21 15:11:04 -07:00
commit 5304ba4491
3 changed files with 12 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
brozzler/browser.py
View File

@ -366,11 +366,12 @@ class Browser:
self.send_to_chrome(method='ServiceWorker.enable')
self.send_to_chrome(method='ServiceWorker.setForceUpdateOnPageLoad')
# disable google analytics
# disable google analytics and amp analytics
self.send_to_chrome(
method='Network.setBlockedURLs',
params={'urls': ['*google-analytics.com/analytics.js',
'*google-analytics.com/ga.js']})
'*google-analytics.com/ga.js',
'*cdn.ampproject.org/*/amp-analytics*.js']})
def stop(self):
'''

8
brozzler/js-templates/extract-outlinks.js
View File

@ -8,6 +8,14 @@ var __brzl_compileOutlinks = function(frame) {
if (frame && frame.document) {
outlinks = Array.prototype.slice.call(
frame.document.querySelectorAll('a[href], area[href]'));
popups = Array.prototype.slice.call(
frame.document.querySelectorAll('a[onclick]'));
if (popups && popups.length > 0) {
for (var p=0; p < popups.length; p++) {
popups[p] = popups[p].onclick.toString().split("'")[1];
}
outlinks = outlinks.concat(popups);
}
for (var i = 0; i < frame.frames.length; i++) {
if (frame.frames[i] && !__brzl_framesDone.has(frame.frames[i])) {
outlinks = outlinks.concat(

2
setup.py
View File

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup(
name='brozzler',
version='1.5.6',
version='1.5.7',
description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler',
author='Noah Levitt',