Merge remote-tracking branch 'internetarchive/master'

bin/drain-queue

@@ -31,7 +31,10 @@ logging.basicConfig(stream=sys.stderr, level=args.log_level,
         format='%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s')
 
 def print_and_maybe_ack(body, message):
-    print(body)
+    # do this instead of print(body) so that output syntax is json, not python
+    # dict (they are similar but not identical)
+    print(message.body.decode('utf-8'))
+
     if not args.no_ack:
         message.ack()
 

bin/queue-json

@@ -0,0 +1,38 @@
+#!/usr/bin/python3.4
+# vim: set sw=4 et:
+
+import os
+import sys
+import argparse
+import logging
+import umbra
+import json
+from kombu import Connection, Exchange, Queue
+
+arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__),
+        description='queue-json - send json message to umbra via AMQP',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+arg_parser.add_argument('-u', '--url', dest='amqp_url', default='amqp://guest:guest@localhost:5672/%2f',
+        help='URL identifying the AMQP server to talk to')
+arg_parser.add_argument('--exchange', dest='amqp_exchange', default='umbra',
+        help='AMQP exchange name')
+arg_parser.add_argument('--routing-key', dest='amqp_routing_key', default='url',
+        help='AMQP routing key')
+arg_parser.add_argument('-v', '--verbose', dest='log_level',
+        action="store_const", default=logging.INFO, const=logging.DEBUG)
+arg_parser.add_argument('--version', action='version',
+        version="umbra {} - {}".format(umbra.version, os.path.basename(__file__)))
+arg_parser.add_argument('payload_json', metavar='JSON_PAYLOAD', help='json payload to send to umbra')
+args = arg_parser.parse_args(args=sys.argv[1:])
+
+logging.basicConfig(stream=sys.stdout, level=args.log_level,
+        format='%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s')
+
+payload = json.loads(args.payload_json)
+
+exchange = Exchange(args.amqp_exchange, 'direct', durable=True)
+with Connection(args.amqp_url) as conn:
+    producer = conn.Producer(serializer='json')
+    logging.info("sending to amqp url={} exchange={} routing_key={} -- {}".format(args.amqp_url, args.amqp_exchange, args.amqp_routing_key, payload))
+    producer.publish(payload, routing_key=args.amqp_routing_key, exchange=exchange)
+

umbra/controller.py

@@ -98,7 +98,17 @@ class AmqpBrowserController:
                 browser.start()
 
                 def callback(body, message):
-                    self._start_browsing_page(browser, message, body['clientId'], body['url'], body['metadata'])
+                    try:
+                        client_id, url, metadata = body['clientId'], body['url'], body['metadata']
+                    except:
+                        self.logger.error("unable to decipher message {}".format(message), exc_info=True)
+                        self.logger.error("discarding bad message")
+                        message.reject()
+                        browser.stop()
+                        self._browser_pool.release(browser)
+                        return
+                    self._start_browsing_page(browser, message, client_id, url, metadata)
+
                 consumer.callbacks = [callback]
 
                 while True: