diff --git a/bin/brozzler-add-site b/bin/brozzler-add-site index 361913b..ca4c713 100755 --- a/bin/brozzler-add-site +++ b/bin/brozzler-add-site @@ -7,18 +7,22 @@ import sys import logging import brozzler import kombu -import json +import re arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__), description="brozzler-add-site - register site to crawl with brozzler-hq", formatter_class=argparse.ArgumentDefaultsHelpFormatter) -# epilog="""Examples -# brozzler-add-site http://example.com/ -# brozzler-add-site '{"seed":"http://example.com/","proxy":"localhost:8000","enable_warcprox_features":true,"ignore_robots":true,"time_limit":600}' -# """ -arg_parser.add_argument('sites', metavar='SITE', nargs='+', help='sites to crawl, either a url or a json object') -arg_parser.add_argument('-u', '--url', dest='amqp_url', default='amqp://guest:guest@localhost:5672/%2f', +arg_parser.add_argument('seed', metavar='SEED', help='seed url') +arg_parser.add_argument('-u', '--amqp-url', dest='amqp_url', default='amqp://guest:guest@localhost:5672/%2f', help='URL identifying the amqp server to talk to') +arg_parser.add_argument("--proxy", dest="proxy", default=None, help="http proxy for this site") +arg_parser.add_argument("--time-limit", dest="time_limit", default=None, help="time limit in seconds for this site") +arg_parser.add_argument("-H", "--extra-header", action="append", + dest="extra_headers", default=None, help="extra http header to send with every request for this site (may be used multiple times)") +arg_parser.add_argument("--ignore-robots", dest="ignore_robots", + action="store_true", help="ignore robots.txt for this site") +arg_parser.add_argument('--enable-warcprox-features', dest='enable_warcprox_features', + action='store_true', help='enable special features for this site that assume the configured proxy is warcprox') arg_parser.add_argument("-v", "--verbose", dest="log_level", action="store_const", default=logging.INFO, const=logging.DEBUG) arg_parser.add_argument("--version", action="version", @@ -28,17 +32,22 @@ args = arg_parser.parse_args(args=sys.argv[1:]) logging.basicConfig(stream=sys.stdout, level=args.log_level, format="%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s") +extra_headers = {} +for hh in args.extra_headers: + [k,v] = re.split(r":\s*", hh, 1) + extra_headers[k] = v + +site = brozzler.Site(seed=args.seed, proxy=args.proxy, + time_limit=int(args.time_limit) if args.time_limit else None, + ignore_robots=args.ignore_robots, + enable_warcprox_features=args.enable_warcprox_features, + extra_headers=extra_headers) + with kombu.Connection(args.amqp_url) as conn: q = conn.SimpleQueue("brozzler.sites.new") - for s in args.sites: - site = None - try: - site = brozzler.Site(**json.loads(s)) - except ValueError: - site = brozzler.Site(s) - # XXX check url syntax? - d = site.to_dict() - logging.info("""feeding amqp queue "{}" with {}""".format(q.queue.name, d)) - q.put(d) + # XXX check url syntax? + d = site.to_dict() + logging.info("""feeding amqp queue "{}" with {}""".format(q.queue.name, d)) + q.put(d)