mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-02-23 16:19:49 -05:00
use yaml.safe_load()
getting new warnings see https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation
This commit is contained in:
parent
6f5f090c33
commit
d729c8d0d5
@ -106,7 +106,7 @@ def behaviors(behaviors_dir=None):
|
||||
d = behaviors_dir or os.path.dirname(__file__)
|
||||
behaviors_yaml = os.path.join(d, 'behaviors.yaml')
|
||||
with open(behaviors_yaml) as fin:
|
||||
_behaviors = yaml.load(fin)
|
||||
_behaviors = yaml.safe_load(fin)
|
||||
return _behaviors
|
||||
|
||||
def behavior_script(url, template_parameters=None, behaviors_dir=None):
|
||||
|
@ -35,7 +35,7 @@ import yaml
|
||||
def load_schema():
|
||||
schema_file = os.path.join(os.path.dirname(__file__), 'job_schema.yaml')
|
||||
with open(schema_file) as f:
|
||||
return yaml.load(f)
|
||||
return yaml.safe_load(f)
|
||||
|
||||
class JobValidator(cerberus.Validator):
|
||||
def _validate_type_url(self, value):
|
||||
@ -68,7 +68,7 @@ def new_job_file(frontier, job_conf_file):
|
||||
'''Returns new Job.'''
|
||||
logging.info("loading %s", job_conf_file)
|
||||
with open(job_conf_file) as f:
|
||||
job_conf = yaml.load(f)
|
||||
job_conf = yaml.safe_load(f)
|
||||
return new_job(frontier, job_conf)
|
||||
|
||||
def new_job(frontier, job_conf):
|
||||
|
5
setup.py
5
setup.py
@ -2,7 +2,7 @@
|
||||
'''
|
||||
setup.py - brozzler setup script
|
||||
|
||||
Copyright (C) 2014-2018 Internet Archive
|
||||
Copyright (C) 2014-2019 Internet Archive
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
@ -32,7 +32,7 @@ def find_package_data(package):
|
||||
|
||||
setuptools.setup(
|
||||
name='brozzler',
|
||||
version='1.5.1',
|
||||
version='1.5.2',
|
||||
description='Distributed web crawling with browsers',
|
||||
url='https://github.com/internetarchive/brozzler',
|
||||
author='Noah Levitt',
|
||||
@ -98,6 +98,7 @@ setuptools.setup(
|
||||
'Programming Language :: Python :: 3.4',
|
||||
'Programming Language :: Python :: 3.5',
|
||||
'Programming Language :: Python :: 3.6',
|
||||
'Programming Language :: Python :: 3.7',
|
||||
'Topic :: Internet :: WWW/HTTP',
|
||||
'Topic :: System :: Archiving',
|
||||
])
|
||||
|
@ -151,7 +151,7 @@ def test_robots_connection_failure():
|
||||
assert brozzler.is_permitted_by_robots(site, url)
|
||||
|
||||
def test_scoping():
|
||||
test_scope = yaml.load('''
|
||||
test_scope = yaml.safe_load('''
|
||||
max_hops: 100
|
||||
accepts:
|
||||
- url_match: REGEX_MATCH
|
||||
|
Loading…
x
Reference in New Issue
Block a user