Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 00:29:53 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add additional checks for weird chrome behavior, serialize requests

Browse Source
This commit is contained in:
Eldon 2014-01-23 18:39:17 +00:00
parent 6dc20e660f
commit 09b0060137
1 changed files with 101 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
umbra/umbra.py
View File

@ -14,8 +14,10 @@ class Umbra:
self.cmd_id = 0
self.chrome_debug_port = port
self.producer = None
self.current_socket = False
self.amqpurl = amqpurl
self.launch_tab_socket = self.get_websocket(self.on_open)
self.producer_lock = threading.Lock()
threading.Thread(target=self.launch_tab_socket.run_forever).start()
def get_websocket(self, on_open, url=None):
@ -23,11 +25,11 @@ class Umbra:
return loads(urllib.request.urlopen("http://localhost:%s/json" % self.chrome_debug_port).read().decode('utf-8').replace("\\n",""))
time.sleep(0.5)
debug_info = []
while not debug_info:
while not (debug_info and 'webSocketDebuggerUrl' in debug_info[0]):
debug_info = fetch_debugging_json()
if url:
debug_info = [x for x in debug_info if x['url'] == url]
return_socket = websocket.WebSocketApp(debug_info[0]['webSocketDebuggerUrl'], on_message = self.handle_message, on_error = print )
return_socket = websocket.WebSocketApp(debug_info[0]['webSocketDebuggerUrl'], on_message = self.handle_message)
return_socket.on_open = on_open
return return_socket
@ -35,6 +37,7 @@ class Umbra:
message = loads(message)
if "method" in message.keys() and message["method"] == "Network.requestWillBeSent":
request_queue = Queue('requests', routing_key='request', exchange=self.umbra_exchange)
with self.producer_lock:
self.producer.publish(message['params']['request'],routing_key='request', exchange=self.umbra_exchange, declare=[request_queue])
@ -61,6 +64,9 @@ class Umbra:
tab.send(dumps(command))
def fetch_url(self, body, message):
while self.current_socket:
time.sleep(1)
self.current_socket = True
url = body['url']
new_page = 'data:text/html;charset=utf-8,<html><body>%s</body></html>' % str(uuid.uuid4())
self.send_command(method="Runtime.evaluate", params={"expression":"window.open('%s');" % new_page})
@ -71,7 +77,8 @@ class Umbra:
def do_close():
time.sleep(10)
self.send_command(tab=ws, method="Runtime.evaluate", params={"expression":"window.open('', '_self', ''); window.close(); "})
#threading.Thread(target=do_close).start()
self.current_socket = False
threading.Thread(target=do_close).start()
socket = self.get_websocket(on_open, new_page)
message.ack()
threading.Thread(target=socket.run_forever).start()