mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-02-24 00:29:53 -05:00
56 lines
2.6 KiB
Python
Executable File
56 lines
2.6 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# vim: set sw=4 et:
|
|
import os
|
|
import sys
|
|
import argparse
|
|
import logging
|
|
import socket
|
|
import umbra
|
|
from kombu import Connection, Exchange, Queue
|
|
|
|
arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__),
|
|
description='drain-queue - consume messages from AMQP queue',
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
|
arg_parser.add_argument('-u', '--url', dest='amqp_url', default='amqp://guest:guest@localhost:5672/%2f',
|
|
help='URL identifying the AMQP server to talk to')
|
|
arg_parser.add_argument('--exchange', dest='amqp_exchange', default='umbra',
|
|
help='AMQP exchange name')
|
|
arg_parser.add_argument('--queue', dest='amqp_queue', default='urls',
|
|
help='AMQP queue name')
|
|
arg_parser.add_argument('-n', '--no-ack', dest='no_ack', action="store_const",
|
|
default=False, const=True, help="leave messages on the queue (default: remove them from the queue)")
|
|
arg_parser.add_argument('-r', '--run-forever', dest='run_forever', action="store_const",
|
|
default=False, const=True, help="run forever, waiting for new messages to appear on the queue (default: exit when all messages in the queue have been consumed)")
|
|
arg_parser.add_argument('-v', '--verbose', dest='log_level',
|
|
action="store_const", default=logging.INFO, const=logging.DEBUG)
|
|
arg_parser.add_argument('--version', action='version',
|
|
version="umbra {} - {}".format(umbra.version, os.path.basename(__file__)))
|
|
args = arg_parser.parse_args(args=sys.argv[1:])
|
|
|
|
logging.basicConfig(stream=sys.stderr, level=args.log_level,
|
|
format='%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s')
|
|
|
|
def print_and_maybe_ack(body, message):
|
|
# do this instead of print(body) so that output syntax is json, not python
|
|
# dict (they are similar but not identical)
|
|
print(message.body.decode('utf-8'))
|
|
|
|
if not args.no_ack:
|
|
message.ack()
|
|
|
|
exchange = Exchange(args.amqp_exchange, 'direct', durable=True)
|
|
queue = Queue(args.amqp_queue, exchange=exchange)
|
|
try:
|
|
with Connection(args.amqp_url) as conn:
|
|
with conn.Consumer(queue, callbacks=[print_and_maybe_ack]) as consumer:
|
|
consumer.qos(prefetch_count=1)
|
|
while True:
|
|
try:
|
|
conn.drain_events(timeout=0.5)
|
|
except socket.timeout:
|
|
if not args.run_forever:
|
|
logging.debug("exiting, no messages left on the queue")
|
|
break
|
|
except KeyboardInterrupt:
|
|
logging.debug("exiting, stopped by user")
|