brozzler/bin/crawl-url

#!/usr/bin/env python
# vim: set sw=4 et:

import argparse
import os
import sys
import logging
import umbra
import umbra.frontier
import threading
import time

arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__),
        description='browse-url - open urls in chrome/chromium and run behaviors',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
arg_parser.add_argument('urls', metavar='URL', nargs='+', help='URL(s) to browse')
arg_parser.add_argument('-w', '--browser-wait', dest='browser_wait', default='60',
        help='seconds to wait for browser initialization')
arg_parser.add_argument('-e', '--executable', dest='chrome_exe', default='chromium-browser',
        help='executable to use to invoke chrome')
arg_parser.add_argument('-n', '--max-browsers', dest='max_browsers', default='1',
        help='Max number of chrome instances simultaneously browsing pages')
arg_parser.add_argument('-v', '--verbose', dest='log_level',
        action="store_const", default=logging.INFO, const=logging.DEBUG)
arg_parser.add_argument('--version', action='version',
        version="umbra {} - {}".format(umbra.version, os.path.basename(__file__)))
args = arg_parser.parse_args(args=sys.argv[1:])

logging.basicConfig(stream=sys.stdout, level=args.log_level,
        format='%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s')

frontier = umbra.frontier.Frontier()
for url in args.urls:
    frontier.schedule(umbra.frontier.CrawlUrl(url, priority=1000))

def frontier_schedule(urls):
    logging.info("scheduling {} urls".format(len(urls)))
    for url in urls:
        frontier.schedule(umbra.frontier.CrawlUrl(url))

def crawl_from_frontier(i):
    with umbra.Browser(chrome_port=9200 + i, chrome_exe=args.chrome_exe) as browser:
        while True:
            try:
                crawl_url = frontier.pop()
                browser.browse_page(crawl_url.url, on_outlinks=frontier_schedule)
            except KeyError:
                time.sleep(0.5)

for i in range(int(args.max_browsers)):
    th = threading.Thread(target=lambda: crawl_from_frontier(i), 
            name="BrowsingThread{}".format(i))
    th.start()

while True:
    time.sleep(0.5)