Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-05-04 15:45:00 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

make load_url handle arguments similarly to umbra

Browse source
This commit is contained in:
Noah Levitt 2014-01-27 19:34:54 -08:00
parent e9f6408881
commit 8eb92b28e6
1 changed files with 15 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

18
bin/load_url.py
View file

@ -1,4 +1,5 @@
#!/usr/bin/env python #!/usr/bin/env python
# vim: set sw=4 et:
from json import dumps, loads from json import dumps, loads
import os,sys,argparse, urllib.request, urllib.error, urllib.parse import os,sys,argparse, urllib.request, urllib.error, urllib.parse
import websocket import websocket
@ -7,10 +8,21 @@ import uuid
import logging import logging
import threading import threading
from kombu import Connection, Exchange, Queue from kombu import Connection, Exchange, Queue
logging.basicConfig(level=logging.INFO)
logging.basicConfig(stream=sys.stdout, level=logging.INFO,
format='%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s')
arg_parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),
description='load_url.py - send url to umbra via amqp',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
arg_parser.add_argument('-u', '--url', dest='amqp_url', default='amqp://guest:guest@localhost:5672//',
help='URL identifying the amqp server to talk to')
arg_parser.add_argument('urls', metavar='URL', nargs='+', help='URLs to send to umbra')
args = arg_parser.parse_args(args=sys.argv[1:])
umbra_exchange = Exchange('umbra', 'direct', durable=True) umbra_exchange = Exchange('umbra', 'direct', durable=True)
with Connection(sys.argv[1]) as conn: with Connection(args.amqp_url) as conn:
producer = conn.Producer(serializer='json') producer = conn.Producer(serializer='json')
for url in sys.argv[2:]: for url in args.urls:
producer.publish({'url': url}, 'url', exchange=umbra_exchange) producer.publish({'url': url}, 'url', exchange=umbra_exchange)