mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-02-24 16:49:56 -05:00
40 lines
1.6 KiB
Python
Executable File
40 lines
1.6 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# vim: set sw=4 et:
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
import logging
|
|
import umbra
|
|
|
|
arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__),
|
|
description='browse-url - open urls in chrome/chromium and run behaviors',
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
|
arg_parser.add_argument('urls', metavar='URL', nargs='+', help='URL(s) to browse')
|
|
arg_parser.add_argument('-w', '--browser-wait', dest='browser_wait', default='60',
|
|
help='seconds to wait for browser initialization')
|
|
arg_parser.add_argument('-e', '--executable', dest='chrome_exe', default='chromium-browser',
|
|
help='executable to use to invoke chrome')
|
|
arg_parser.add_argument('-v', '--verbose', dest='log_level',
|
|
action="store_const", default=logging.INFO, const=logging.DEBUG)
|
|
arg_parser.add_argument('--version', action='version',
|
|
version="umbra {} - {}".format(umbra.version, os.path.basename(__file__)))
|
|
args = arg_parser.parse_args(args=sys.argv[1:])
|
|
|
|
logging.basicConfig(stream=sys.stdout, level=args.log_level,
|
|
format='%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s')
|
|
|
|
frontier = list(args.urls)
|
|
|
|
def add_to_frontier(urls):
|
|
logging.info("adding {} urls to frontier".format(len(urls)))
|
|
frontier.extend(urls)
|
|
|
|
with umbra.Browser(chrome_exe=args.chrome_exe) as browser:
|
|
try:
|
|
while True:
|
|
browser.browse_page(frontier.pop(), on_outlinks=add_to_frontier)
|
|
except IndexError:
|
|
logging.info("finished, frontier is empty")
|
|
|