mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-06-20 04:44:12 -04:00
fix oversight including username/password in site config when starting a new job
This commit is contained in:
parent
c2704b18be
commit
76b658747e
2 changed files with 4 additions and 4 deletions
|
@ -75,8 +75,6 @@ def new_job(frontier, job_conf):
|
|||
sites = []
|
||||
for seed_conf in job_conf["seeds"]:
|
||||
merged_conf = merge(seed_conf, job_conf)
|
||||
if "login" in merged_conf and "metadata" in merged_conf:
|
||||
merged_conf["metadata"]["login"] = merged_conf["login"]
|
||||
site = brozzler.Site(
|
||||
job_id=job.id, seed=merged_conf["url"],
|
||||
scope=merged_conf.get("scope"),
|
||||
|
@ -89,7 +87,9 @@ def new_job(frontier, job_conf):
|
|||
metadata=merged_conf.get("metadata"),
|
||||
remember_outlinks=merged_conf.get("remember_outlinks"),
|
||||
user_agent=merged_conf.get("user_agent"),
|
||||
behavior_parameters=merged_conf.get("behavior_parameters"))
|
||||
behavior_parameters=merged_conf.get("behavior_parameters"),
|
||||
username=merged_conf.get("username"),
|
||||
password=merged_conf.get("password"))
|
||||
sites.append(site)
|
||||
|
||||
# insert all the sites into database before the job
|
||||
|
|
2
setup.py
2
setup.py
|
@ -32,7 +32,7 @@ def find_package_data(package):
|
|||
|
||||
setuptools.setup(
|
||||
name='brozzler',
|
||||
version='1.1b9.dev157',
|
||||
version='1.1b9.dev158',
|
||||
description='Distributed web crawling with browsers',
|
||||
url='https://github.com/internetarchive/brozzler',
|
||||
author='Noah Levitt',
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue