Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-19 12:24:20 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

avoid "Uncaught RangeError: Maximum call stack size exceeded" compiling outlinks

Browse source
This commit is contained in:
Noah Levitt 2016-08-04 17:33:06 -07:00
parent 7734399a22
commit 20f9934dd9
2 changed files with 7 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

10
brozzler/browser.py
View file

@ -301,17 +301,19 @@ class Browser:
self._behavior = None self._behavior = None
OUTLINKS_JS = """ OUTLINKS_JS = """
var compileOutlinks = function(frame) { var __brzl_framesDone = new Set();
var __brzl_compileOutlinks = function(frame) {
__brzl_framesDone.add(frame);
var outlinks = Array.prototype.slice.call( var outlinks = Array.prototype.slice.call(
frame.document.querySelectorAll('a[href]')); frame.document.querySelectorAll('a[href]'));
for (var i = 0; i < frame.frames.length; i++) { for (var i = 0; i < frame.frames.length; i++) {
if (frame.frames[i]) { // sometimes undefined (why?) if (frame.frames[i] && !__brzl_framesDone.has(frame.frames[i])) {
outlinks = outlinks.concat(compileOutlinks(frame.frames[i])); outlinks = outlinks.concat(__brzl_compileOutlinks(frame.frames[i]));
} }
} }
return outlinks; return outlinks;
} }
compileOutlinks(window).join(' '); __brzl_compileOutlinks(window).join(' ');
""" """
def _chain_chrome_messages(self, chain): def _chain_chrome_messages(self, chain):

2
setup.py
View file

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1b5.dev66', version='1.1b5.dev67',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',