Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-18 11:59:26 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

use yaml.safe_load()

Browse source 
getting new warnings
see https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation
This commit is contained in:
Noah Levitt 2019-03-18 15:49:44 -07:00
parent 6f5f090c33
commit d729c8d0d5
4 changed files with 7 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

2
brozzler/__init__.py
View file

@ -106,7 +106,7 @@ def behaviors(behaviors_dir=None):
d = behaviors_dir or os.path.dirname(__file__) d = behaviors_dir or os.path.dirname(__file__)
behaviors_yaml = os.path.join(d, 'behaviors.yaml') behaviors_yaml = os.path.join(d, 'behaviors.yaml')
with open(behaviors_yaml) as fin: with open(behaviors_yaml) as fin:
_behaviors = yaml.load(fin) _behaviors = yaml.safe_load(fin)
return _behaviors return _behaviors
def behavior_script(url, template_parameters=None, behaviors_dir=None): def behavior_script(url, template_parameters=None, behaviors_dir=None):

4
brozzler/model.py
View file

@ -35,7 +35,7 @@ import yaml
def load_schema(): def load_schema():
schema_file = os.path.join(os.path.dirname(__file__), 'job_schema.yaml') schema_file = os.path.join(os.path.dirname(__file__), 'job_schema.yaml')
with open(schema_file) as f: with open(schema_file) as f:
return yaml.load(f) return yaml.safe_load(f)
class JobValidator(cerberus.Validator): class JobValidator(cerberus.Validator):
def _validate_type_url(self, value): def _validate_type_url(self, value):
@ -68,7 +68,7 @@ def new_job_file(frontier, job_conf_file):
'''Returns new Job.''' '''Returns new Job.'''
logging.info("loading %s", job_conf_file) logging.info("loading %s", job_conf_file)
with open(job_conf_file) as f: with open(job_conf_file) as f:
job_conf = yaml.load(f) job_conf = yaml.safe_load(f)
return new_job(frontier, job_conf) return new_job(frontier, job_conf)
def new_job(frontier, job_conf): def new_job(frontier, job_conf):

5
setup.py
View file

@ -2,7 +2,7 @@
''' '''
setup.py - brozzler setup script setup.py - brozzler setup script
Copyright (C) 2014-2018 Internet Archive Copyright (C) 2014-2019 Internet Archive
Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.
@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.5.1', version='1.5.2',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',
@ -98,6 +98,7 @@ setuptools.setup(
'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Topic :: Internet :: WWW/HTTP', 'Topic :: Internet :: WWW/HTTP',
'Topic :: System :: Archiving', 'Topic :: System :: Archiving',
]) ])

2
tests/test_units.py
View file

@ -151,7 +151,7 @@ def test_robots_connection_failure():
assert brozzler.is_permitted_by_robots(site, url) assert brozzler.is_permitted_by_robots(site, url)
def test_scoping(): def test_scoping():
test_scope = yaml.load(''' test_scope = yaml.safe_load('''
max_hops: 100 max_hops: 100
accepts: accepts:
- url_match: REGEX_MATCH - url_match: REGEX_MATCH