mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-02-24 00:29:53 -05:00
Add additional checks for weird chrome behavior, serialize requests
This commit is contained in:
Eldon
parent
6dc20e660f
commit
09b0060137
195
umbra/umbra.py
195
umbra/umbra.py
@ -10,108 +10,115 @@ from kombu import Connection, Exchange, Queue
|
|||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
|
||||||
class Umbra:
|
class Umbra:
|
||||||
def __init__(self, port, amqpurl):
|
def __init__(self, port, amqpurl):
|
||||||
self.cmd_id = 0
|
self.cmd_id = 0
|
||||||
self.chrome_debug_port = port
|
self.chrome_debug_port = port
|
||||||
self.producer = None
|
self.producer = None
|
||||||
self.amqpurl = amqpurl
|
self.current_socket = False
|
||||||
self.launch_tab_socket = self.get_websocket(self.on_open)
|
self.amqpurl = amqpurl
|
||||||
threading.Thread(target=self.launch_tab_socket.run_forever).start()
|
self.launch_tab_socket = self.get_websocket(self.on_open)
|
||||||
|
self.producer_lock = threading.Lock()
|
||||||
def get_websocket(self, on_open, url=None):
|
threading.Thread(target=self.launch_tab_socket.run_forever).start()
|
||||||
def fetch_debugging_json():
|
|
||||||
return loads(urllib.request.urlopen("http://localhost:%s/json" % self.chrome_debug_port).read().decode('utf-8').replace("\\n",""))
|
|
||||||
time.sleep(0.5)
|
|
||||||
debug_info = []
|
|
||||||
while not debug_info:
|
|
||||||
debug_info = fetch_debugging_json()
|
|
||||||
if url:
|
|
||||||
debug_info = [x for x in debug_info if x['url'] == url]
|
|
||||||
return_socket = websocket.WebSocketApp(debug_info[0]['webSocketDebuggerUrl'], on_message = self.handle_message, on_error = print )
|
|
||||||
return_socket.on_open = on_open
|
|
||||||
return return_socket
|
|
||||||
|
|
||||||
def handle_message(self, ws, message):
|
|
||||||
message = loads(message)
|
|
||||||
if "method" in message.keys() and message["method"] == "Network.requestWillBeSent":
|
|
||||||
request_queue = Queue('requests', routing_key='request', exchange=self.umbra_exchange)
|
|
||||||
self.producer.publish(message['params']['request'],routing_key='request', exchange=self.umbra_exchange, declare=[request_queue])
|
|
||||||
|
|
||||||
|
def get_websocket(self, on_open, url=None):
|
||||||
def start_amqp(self):
|
def fetch_debugging_json():
|
||||||
self.umbra_exchange = Exchange('umbra', 'direct', durable=True)
|
return loads(urllib.request.urlopen("http://localhost:%s/json" % self.chrome_debug_port).read().decode('utf-8').replace("\\n",""))
|
||||||
url_queue = Queue('urls', routing_key='url', exchange=self.umbra_exchange)
|
time.sleep(0.5)
|
||||||
with Connection(self.amqpurl) as conn:
|
debug_info = []
|
||||||
self.producer = conn.Producer(serializer='json')
|
while not (debug_info and 'webSocketDebuggerUrl' in debug_info[0]):
|
||||||
with conn.Consumer(url_queue, callbacks=[self.fetch_url]) as consumer:
|
debug_info = fetch_debugging_json()
|
||||||
while True:
|
if url:
|
||||||
conn.drain_events()
|
debug_info = [x for x in debug_info if x['url'] == url]
|
||||||
|
return_socket = websocket.WebSocketApp(debug_info[0]['webSocketDebuggerUrl'], on_message = self.handle_message)
|
||||||
|
return_socket.on_open = on_open
|
||||||
|
return return_socket
|
||||||
|
|
||||||
def on_open(self, ws):
|
def handle_message(self, ws, message):
|
||||||
threading.Thread(target=self.start_amqp).start()
|
message = loads(message)
|
||||||
|
if "method" in message.keys() and message["method"] == "Network.requestWillBeSent":
|
||||||
|
request_queue = Queue('requests', routing_key='request', exchange=self.umbra_exchange)
|
||||||
|
with self.producer_lock:
|
||||||
|
self.producer.publish(message['params']['request'],routing_key='request', exchange=self.umbra_exchange, declare=[request_queue])
|
||||||
|
|
||||||
def send_command(self,tab=None, **kwargs):
|
|
||||||
if not tab:
|
def start_amqp(self):
|
||||||
tab = self.launch_tab_socket
|
self.umbra_exchange = Exchange('umbra', 'direct', durable=True)
|
||||||
command = {}
|
url_queue = Queue('urls', routing_key='url', exchange=self.umbra_exchange)
|
||||||
command.update(kwargs)
|
with Connection(self.amqpurl) as conn:
|
||||||
self.cmd_id += 1
|
self.producer = conn.Producer(serializer='json')
|
||||||
command['id'] = self.cmd_id
|
with conn.Consumer(url_queue, callbacks=[self.fetch_url]) as consumer:
|
||||||
tab.send(dumps(command))
|
while True:
|
||||||
|
conn.drain_events()
|
||||||
def fetch_url(self, body, message):
|
|
||||||
url = body['url']
|
def on_open(self, ws):
|
||||||
new_page = 'data:text/html;charset=utf-8,<html><body>%s</body></html>' % str(uuid.uuid4())
|
threading.Thread(target=self.start_amqp).start()
|
||||||
self.send_command(method="Runtime.evaluate", params={"expression":"window.open('%s');" % new_page})
|
|
||||||
def on_open(ws):
|
|
||||||
ws.on_message=self.handle_message
|
def send_command(self,tab=None, **kwargs):
|
||||||
self.send_command(tab=ws, method="Network.enable")
|
if not tab:
|
||||||
self.send_command(tab=ws, method="Runtime.evaluate", params={"expression":"document.location = '%s';" % url})
|
tab = self.launch_tab_socket
|
||||||
def do_close():
|
command = {}
|
||||||
time.sleep(10)
|
command.update(kwargs)
|
||||||
self.send_command(tab=ws, method="Runtime.evaluate", params={"expression":"window.open('', '_self', ''); window.close(); "})
|
self.cmd_id += 1
|
||||||
#threading.Thread(target=do_close).start()
|
command['id'] = self.cmd_id
|
||||||
socket = self.get_websocket(on_open, new_page)
|
tab.send(dumps(command))
|
||||||
message.ack()
|
|
||||||
threading.Thread(target=socket.run_forever).start()
|
def fetch_url(self, body, message):
|
||||||
|
while self.current_socket:
|
||||||
|
time.sleep(1)
|
||||||
|
self.current_socket = True
|
||||||
|
url = body['url']
|
||||||
|
new_page = 'data:text/html;charset=utf-8,<html><body>%s</body></html>' % str(uuid.uuid4())
|
||||||
|
self.send_command(method="Runtime.evaluate", params={"expression":"window.open('%s');" % new_page})
|
||||||
|
def on_open(ws):
|
||||||
|
ws.on_message=self.handle_message
|
||||||
|
self.send_command(tab=ws, method="Network.enable")
|
||||||
|
self.send_command(tab=ws, method="Runtime.evaluate", params={"expression":"document.location = '%s';" % url})
|
||||||
|
def do_close():
|
||||||
|
time.sleep(10)
|
||||||
|
self.send_command(tab=ws, method="Runtime.evaluate", params={"expression":"window.open('', '_self', ''); window.close(); "})
|
||||||
|
self.current_socket = False
|
||||||
|
threading.Thread(target=do_close).start()
|
||||||
|
socket = self.get_websocket(on_open, new_page)
|
||||||
|
message.ack()
|
||||||
|
threading.Thread(target=socket.run_forever).start()
|
||||||
|
|
||||||
class Chrome():
|
class Chrome():
|
||||||
def __init__(self, port, executable, browser_wait):
|
def __init__(self, port, executable, browser_wait):
|
||||||
self.port = port
|
self.port = port
|
||||||
self.executable = executable
|
self.executable = executable
|
||||||
self.browser_wait=browser_wait
|
self.browser_wait=browser_wait
|
||||||
|
|
||||||
def __enter__(self):
|
def __enter__(self):
|
||||||
import psutil, subprocess
|
import psutil, subprocess
|
||||||
self.chrome_process = subprocess.Popen([self.executable, "--disable-web-sockets", "--temp-profile", "--remote-debugging-port=%s" % self.port])
|
self.chrome_process = subprocess.Popen([self.executable, "--disable-web-sockets", "--temp-profile", "--remote-debugging-port=%s" % self.port])
|
||||||
start = time.time()
|
start = time.time()
|
||||||
import socket
|
import socket
|
||||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
while sock.connect_ex(('127.0.0.1',int(self.port))) != 0 and (time.time() - start) < float(self.browser_wait):
|
while sock.connect_ex(('127.0.0.1',int(self.port))) != 0 and (time.time() - start) < float(self.browser_wait):
|
||||||
time.sleep(0.1)
|
time.sleep(0.1)
|
||||||
|
|
||||||
|
|
||||||
def __exit__(self, *args):
|
|
||||||
self.chrome_process.kill()
|
def __exit__(self, *args):
|
||||||
|
self.chrome_process.kill()
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
arg_parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),
|
arg_parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),
|
||||||
description='umbra - Browser automation tool',
|
description='umbra - Browser automation tool',
|
||||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||||
arg_parser.add_argument('-w', '--browser-wait', dest='browser_wait', default='10',
|
arg_parser.add_argument('-w', '--browser-wait', dest='browser_wait', default='10',
|
||||||
help='Seconds to wait for browser initialization')
|
help='Seconds to wait for browser initialization')
|
||||||
arg_parser.add_argument('-e', '--executable', dest='executable', default='chromium-browser',
|
arg_parser.add_argument('-e', '--executable', dest='executable', default='chromium-browser',
|
||||||
help='Executable to use to invoke chrome')
|
help='Executable to use to invoke chrome')
|
||||||
arg_parser.add_argument('-p', '--port', dest='port', default='9222',
|
arg_parser.add_argument('-p', '--port', dest='port', default='9222',
|
||||||
help='Port to have invoked chrome listen on for debugging connections')
|
help='Port to have invoked chrome listen on for debugging connections')
|
||||||
arg_parser.add_argument('-u', '--url', dest='amqpurl', default='amqp://guest:guest@localhost:5672//',
|
arg_parser.add_argument('-u', '--url', dest='amqpurl', default='amqp://guest:guest@localhost:5672//',
|
||||||
help='URL identifying the amqp server to talk to')
|
help='URL identifying the amqp server to talk to')
|
||||||
args = arg_parser.parse_args(args=sys.argv[1:])
|
args = arg_parser.parse_args(args=sys.argv[1:])
|
||||||
with Chrome(args.port, args.executable, args.browser_wait):
|
with Chrome(args.port, args.executable, args.browser_wait):
|
||||||
Umbra(args.port, args.amqpurl)
|
Umbra(args.port, args.amqpurl)
|
||||||
while True:
|
while True:
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user