Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-19 20:34:09 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

add only one site at a time, specify settings with command line switches

Browse source
This commit is contained in:
Noah Levitt 2015-07-21 06:32:00 +00:00
parent 38ddfe498d
commit 1e56bc8686
1 changed files with 26 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

35
bin/brozzler-add-site
View file

@ -7,18 +7,22 @@ import sys
import logging
import brozzler
import kombu
import json
import re
arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__),
description="brozzler-add-site - register site to crawl with brozzler-hq",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# epilog="""Examples
# brozzler-add-site http://example.com/
# brozzler-add-site '{"seed":"http://example.com/","proxy":"localhost:8000","enable_warcprox_features":true,"ignore_robots":true,"time_limit":600}'
# """
arg_parser.add_argument('sites', metavar='SITE', nargs='+', help='sites to crawl, either a url or a json object')
arg_parser.add_argument('-u', '--url', dest='amqp_url', default='amqp://guest:guest@localhost:5672/%2f',
arg_parser.add_argument('seed', metavar='SEED', help='seed url')
arg_parser.add_argument('-u', '--amqp-url', dest='amqp_url', default='amqp://guest:guest@localhost:5672/%2f',
help='URL identifying the amqp server to talk to')
arg_parser.add_argument("--proxy", dest="proxy", default=None, help="http proxy for this site")
arg_parser.add_argument("--time-limit", dest="time_limit", default=None, help="time limit in seconds for this site")
arg_parser.add_argument("-H", "--extra-header", action="append",
dest="extra_headers", default=None, help="extra http header to send with every request for this site (may be used multiple times)")
arg_parser.add_argument("--ignore-robots", dest="ignore_robots",
action="store_true", help="ignore robots.txt for this site")
arg_parser.add_argument('--enable-warcprox-features', dest='enable_warcprox_features',
action='store_true', help='enable special features for this site that assume the configured proxy is warcprox')
arg_parser.add_argument("-v", "--verbose", dest="log_level",
action="store_const", default=logging.INFO, const=logging.DEBUG)
arg_parser.add_argument("--version", action="version",
@ -28,14 +32,19 @@ args = arg_parser.parse_args(args=sys.argv[1:])
logging.basicConfig(stream=sys.stdout, level=args.log_level,
format="%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s")
extra_headers = {}
for hh in args.extra_headers:
[k,v] = re.split(r":\s*", hh, 1)
extra_headers[k] = v
site = brozzler.Site(seed=args.seed, proxy=args.proxy,
time_limit=int(args.time_limit) if args.time_limit else None,
ignore_robots=args.ignore_robots,
enable_warcprox_features=args.enable_warcprox_features,
extra_headers=extra_headers)
with kombu.Connection(args.amqp_url) as conn:
q = conn.SimpleQueue("brozzler.sites.new")
for s in args.sites:
site = None
try:
site = brozzler.Site(**json.loads(s))
except ValueError:
site = brozzler.Site(s)
# XXX check url syntax?
d = site.to_dict()
logging.info("""feeding amqp queue "{}" with {}""".format(q.queue.name, d))