mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-02-24 08:39:59 -05:00
37 lines
1.9 KiB
Python
Executable File
37 lines
1.9 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# vim: set sw=4 et:
|
|
|
|
import os
|
|
import sys
|
|
import argparse
|
|
import logging
|
|
from kombu import Connection, Exchange, Queue
|
|
|
|
arg_parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),
|
|
description='queue-url - send url to umbra via AMQP',
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
|
arg_parser.add_argument('-u', '--url', dest='amqp_url', default='amqp://guest:guest@localhost:5672/%2f',
|
|
help='URL identifying the AMQP server to talk to')
|
|
arg_parser.add_argument('--exchange', dest='amqp_exchange', default='umbra',
|
|
help='AMQP exchange name')
|
|
arg_parser.add_argument('--routing-key', dest='amqp_routing_key', default='url',
|
|
help='AMQP routing key')
|
|
arg_parser.add_argument('-i', '--client-id', dest='client_id', default='load_url.0',
|
|
help='client id - included in the json payload with each url; umbra uses this value as the routing key to send requests back to')
|
|
arg_parser.add_argument('-v', '--verbose', dest='log_level',
|
|
action="store_const", default=logging.INFO, const=logging.DEBUG)
|
|
arg_parser.add_argument('urls', metavar='URL', nargs='+', help='URLs to send to umbra')
|
|
args = arg_parser.parse_args(args=sys.argv[1:])
|
|
|
|
logging.basicConfig(stream=sys.stdout, level=args.log_level,
|
|
format='%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s')
|
|
|
|
exchange = Exchange(args.amqp_exchange, 'direct', durable=True)
|
|
with Connection(args.amqp_url) as conn:
|
|
producer = conn.Producer(serializer='json')
|
|
for url in args.urls:
|
|
payload = {'url': url, 'metadata': {}, 'clientId': args.client_id}
|
|
logging.info("sending to amqp url={} exchange={} routing_key={} -- {}".format(args.amqp_url, args.amqp_exchange, args.amqp_routing_key, payload))
|
|
producer.publish(payload, routing_key=args.amqp_routing_key, exchange=exchange)
|
|
|