mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-02-24 16:49:56 -05:00
129 lines
4.4 KiB
Python
Executable File
129 lines
4.4 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# vim: set sw=4 et:
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
import logging
|
|
import umbra
|
|
import threading
|
|
import time
|
|
import sortedcontainers
|
|
import surt
|
|
|
|
arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__),
|
|
description='browse-url - open urls in chrome/chromium and run behaviors',
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
|
arg_parser.add_argument('urls', metavar='URL', nargs='+', help='URL(s) to browse')
|
|
arg_parser.add_argument('-w', '--browser-wait', dest='browser_wait', default='60',
|
|
help='seconds to wait for browser initialization')
|
|
arg_parser.add_argument('-e', '--executable', dest='chrome_exe', default='chromium-browser',
|
|
help='executable to use to invoke chrome')
|
|
arg_parser.add_argument('-n', '--max-browsers', dest='max_browsers', default='1',
|
|
help='Max number of chrome instances simultaneously browsing pages')
|
|
arg_parser.add_argument('-v', '--verbose', dest='log_level',
|
|
action="store_const", default=logging.INFO, const=logging.DEBUG)
|
|
arg_parser.add_argument('--version', action='version',
|
|
version="umbra {} - {}".format(umbra.version, os.path.basename(__file__)))
|
|
args = arg_parser.parse_args(args=sys.argv[1:])
|
|
|
|
logging.basicConfig(stream=sys.stdout, level=args.log_level,
|
|
format='%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s')
|
|
|
|
class CrawlUrl:
|
|
def __init__(self, url, priority=1):
|
|
self.url = url
|
|
self._surt = None
|
|
self.set_priority(priority)
|
|
|
|
def set_priority(self, priority):
|
|
# priority_key is both a sortable priority (higher value is higher
|
|
# priority) and a unique hash key
|
|
self.priority_key = (priority << 32) | (hash(self.surt) & (2**32 - 1))
|
|
|
|
@property
|
|
def surt(self):
|
|
if self._surt is None:
|
|
self._surt = surt.surt(self.url, canonicalizer=surt.GoogleURLCanonicalizer, trailing_comma=True)
|
|
return self._surt
|
|
|
|
@property
|
|
def priority(self):
|
|
return self.priority_key >> 32
|
|
|
|
class CrawlUrlQueue:
|
|
def __init__(self):
|
|
# {priority_key:CrawlUrl}
|
|
self._pq = sortedcontainers.SortedDict()
|
|
# {surt:CrawlUrl}
|
|
self._urls = {}
|
|
self.aggregate_priority = 0
|
|
|
|
def __len__(self):
|
|
assert len(self._urls) == len(self._pq)
|
|
return len(self._urls)
|
|
|
|
def schedule(self, crawl_url):
|
|
self.aggregate_priority += crawl_url.priority
|
|
|
|
try:
|
|
old_priority_key = self._urls.pop(crawl_url.surt).priority_key
|
|
old_crawl_url = self._pq.pop(old_priority_key)
|
|
|
|
# XXX dumb calculation of new priority, may not belong here
|
|
crawl_url.set_priority(crawl_url.priority + old_crawl_url.priority)
|
|
except KeyError:
|
|
pass
|
|
|
|
self._urls[crawl_url.surt] = crawl_url
|
|
self._pq[crawl_url.priority_key] = crawl_url
|
|
|
|
def next_url(self):
|
|
res0 = self._pq.popitem(last=True)[1]
|
|
res1 = self._urls.pop(res0.surt)
|
|
assert res0 is res1
|
|
|
|
new_low_priority = CrawlUrl(res0.url, -1000)
|
|
self.schedule(new_low_priority)
|
|
|
|
return res0
|
|
|
|
class Site:
|
|
"""A seed url, scope definition, and prioritized url queue."""
|
|
def __init__(self, seed_url):
|
|
self.seed = CrawlUrl(seed_url, priority=1000)
|
|
|
|
self.q = CrawlUrlQueue()
|
|
self.q.schedule(self.seed)
|
|
|
|
def is_in_scope(self, url):
|
|
surtt = surt.surt(url, canonicalizer=surt.GoogleURLCanonicalizer, trailing_comma=True)
|
|
return surtt.startswith(self.seed.surt)
|
|
|
|
def submit(self, urls):
|
|
for url in urls:
|
|
if self.is_in_scope(url):
|
|
logging.info("{} accepted {}".format(self.seed.surt, url))
|
|
self.q.schedule(CrawlUrl(url))
|
|
else:
|
|
logging.info("{} rejected {}".format(self.seed.surt, url))
|
|
|
|
# "browse" + "crawl" = "brozzle"
|
|
def brozzle_site(site, chrome_port):
|
|
with umbra.Browser(chrome_port=chrome_port, chrome_exe=args.chrome_exe) as browser:
|
|
while True:
|
|
crawl_url = site.q.next_url()
|
|
outlinks = browser.browse_page(crawl_url.url)
|
|
site.submit(outlinks)
|
|
|
|
chrome_port = 9200
|
|
for seed_url in args.urls:
|
|
site = Site(seed_url)
|
|
|
|
th = threading.Thread(target=lambda: brozzle_site(site, chrome_port),
|
|
name="BrowsingThread-{}".format(site.seed.surt))
|
|
th.start()
|
|
|
|
chrome_port += 1
|
|
|