From 77c800f6a2deb8465151700035df725f0c5f3949 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 28 Jun 2016 15:13:48 -0500 Subject: [PATCH] renaming scope rule "host" to "domain" to make it a less confusing, since rules apply to subdomains as well --- brozzler/site.py | 42 +++++++++++++++++++++--------------------- setup.py | 2 +- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/brozzler/site.py b/brozzler/site.py index 74927bd..53593de 100644 --- a/brozzler/site.py +++ b/brozzler/site.py @@ -1,20 +1,20 @@ -# -# brozzler/site.py - classes representing sites and pages -# -# Copyright (C) 2014-2016 Internet Archive -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +''' +brozzler/site.py - classes representing sites and pages + +Copyright (C) 2014-2016 Internet Archive + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' import surt import json @@ -170,7 +170,7 @@ class Site(brozzler.BaseDictable): Examples of valid rules: [ { - "host": "monkey.org", + "domain": "monkey.org", "url_match": "STRING_MATCH", "value": "bar", }, @@ -179,7 +179,7 @@ class Site(brozzler.BaseDictable): "value": "http://(com,woop,)/fuh/", }, { - "host": "badhost.com", + "domain": "bad.domain.com", }, ] """ @@ -188,7 +188,7 @@ class Site(brozzler.BaseDictable): else: u = url - if "host" in rule and not u.matches_ip_or_domain(rule["host"]): + if "domain" in rule and not u.matches_ip_or_domain(rule["domain"]): return False if "url_match" in rule: if rule["url_match"] == "STRING_MATCH": @@ -207,7 +207,7 @@ class Site(brozzler.BaseDictable): self.logger.warn("invalid rule.url_match=%s", rule.url_match) return False else: - if "host" in rule: + if "domain" in rule: # we already know that it matches from earlier check return True else: diff --git a/setup.py b/setup.py index 371d04f..6f9e009 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ import setuptools setuptools.setup( name='brozzler', - version='1.1.dev27', + version='1.1.dev28', description='Distributed web crawling with browsers', url='https://github.com/internetarchive/brozzler', author='Noah Levitt',