brozzler/bin/brozzler-add-site

#!/usr/bin/env python
# vim: set sw=4 et:

import argparse
import os
import sys
import logging
import brozzler
import kombu
import re

arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__),
        description="brozzler-add-site - register site to crawl with brozzler-hq",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
arg_parser.add_argument('seed', metavar='SEED', help='seed url')
arg_parser.add_argument('-u', '--amqp-url', dest='amqp_url', default='amqp://guest:guest@localhost:5672/%2f',
        help='URL identifying the amqp server to talk to')
arg_parser.add_argument("--proxy", dest="proxy", default=None, help="http proxy for this site")
arg_parser.add_argument("--time-limit", dest="time_limit", default=None, help="time limit in seconds for this site")
arg_parser.add_argument("-H", "--extra-header", action="append",
        dest="extra_headers", default=None, help="extra http header to send with every request for this site (may be used multiple times)")
arg_parser.add_argument("--ignore-robots", dest="ignore_robots",
        action="store_true", help="ignore robots.txt for this site")
arg_parser.add_argument('--enable-warcprox-features', dest='enable_warcprox_features',
                action='store_true', help='enable special features for this site that assume the configured proxy is warcprox')
arg_parser.add_argument("-v", "--verbose", dest="log_level",
        action="store_const", default=logging.INFO, const=logging.DEBUG)
arg_parser.add_argument("--version", action="version",
        version="brozzler {} - {}".format(brozzler.version, os.path.basename(__file__)))
args = arg_parser.parse_args(args=sys.argv[1:])

logging.basicConfig(stream=sys.stdout, level=args.log_level,
        format="%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s")

extra_headers = {}
if args.extra_headers:
    for hh in args.extra_headers:
        [k,v] = re.split(r":\s*", hh, 1)
        extra_headers[k] = v

site = brozzler.Site(seed=args.seed, proxy=args.proxy,
        time_limit=int(args.time_limit) if args.time_limit else None,
        ignore_robots=args.ignore_robots,
        enable_warcprox_features=args.enable_warcprox_features,
        extra_headers=extra_headers)

with kombu.Connection(args.amqp_url) as conn:
    q = conn.SimpleQueue("brozzler.sites.new")
    # XXX check url syntax?
    d = site.to_dict()
    logging.info("""feeding amqp queue "{}" with {}""".format(q.queue.name, d))
    q.put(d)