#!/usr/bin/env python # vim: set sw=4 et: import argparse import os import sys import logging import umbra import kombu arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__), description="brozzler-add-site - register site to crawl with brozzler-hq", formatter_class=argparse.ArgumentDefaultsHelpFormatter) arg_parser.add_argument('urls', metavar='URL', nargs='+', help='seed URL(s) of sites to crawl') arg_parser.add_argument('-u', '--url', dest='amqp_url', default='amqp://guest:guest@localhost:5672/%2f', help='URL identifying the amqp server to talk to') arg_parser.add_argument("-v", "--verbose", dest="log_level", action="store_const", default=logging.INFO, const=logging.DEBUG) arg_parser.add_argument("--version", action="version", version="umbra {} - {}".format(umbra.version, os.path.basename(__file__))) args = arg_parser.parse_args(args=sys.argv[1:]) logging.basicConfig(stream=sys.stdout, level=args.log_level, format="%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s") with kombu.Connection(args.amqp_url) as conn: q = conn.SimpleQueue("brozzler.sites.new") for url in args.urls: q.put({"seed":url})