Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 16:49:56 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

pass behavior_parameters from job configuration into Site objects

Browse Source
This commit is contained in:
Noah Levitt 2016-11-09 13:43:10 -08:00
parent 8e115b44fa
commit 02bf23059e
2 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
brozzler/job.py
View File

@ -88,7 +88,8 @@ def new_job(frontier, job_conf):
warcprox_meta=merged_conf.get("warcprox_meta"), warcprox_meta=merged_conf.get("warcprox_meta"),
metadata=merged_conf.get("metadata"), metadata=merged_conf.get("metadata"),
remember_outlinks=merged_conf.get("remember_outlinks"), remember_outlinks=merged_conf.get("remember_outlinks"),
user_agent=merged_conf.get("user_agent")) user_agent=merged_conf.get("user_agent"),
behavior_parameters=merged_conf.get("behavior_parameters"))
sites.append(site) sites.append(site)
# insert all the sites into database before the job # insert all the sites into database before the job

2
setup.py
View File

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1b7.dev120', version='1.1b7.dev121',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',