#!/usr/bin/env python
# vim: set sw=4 et:

import argparse
import os
import sys
import logging
import umbra
import threading
import time
import sortedcontainers
import surt

arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__),
        description='browse-url - open urls in chrome/chromium and run behaviors',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
arg_parser.add_argument('urls', metavar='URL', nargs='+', help='URL(s) to browse')
arg_parser.add_argument('-w', '--browser-wait', dest='browser_wait', default='60',
        help='seconds to wait for browser initialization')
arg_parser.add_argument('-e', '--executable', dest='chrome_exe', default='chromium-browser',
        help='executable to use to invoke chrome')
arg_parser.add_argument('-n', '--max-browsers', dest='max_browsers', default='1',
        help='Max number of chrome instances simultaneously browsing pages')
arg_parser.add_argument('-v', '--verbose', dest='log_level',
        action="store_const", default=logging.INFO, const=logging.DEBUG)
arg_parser.add_argument('--version', action='version',
        version="umbra {} - {}".format(umbra.version, os.path.basename(__file__)))
args = arg_parser.parse_args(args=sys.argv[1:])

logging.basicConfig(stream=sys.stdout, level=args.log_level,
        format='%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s')

class CrawlUrl:
    def __init__(self, url, priority=1):
        self.url = url
        self._surt = None
        self.set_priority(priority)

    def set_priority(self, priority):
        # priority_key is both a sortable priority (higher value is higher
        # priority) and a unique hash key
        self.priority_key = (priority << 32) | (hash(self.surt) & (2**32 - 1))

    @property
    def surt(self):
        if self._surt is None:
            self._surt = surt.surt(self.url, canonicalizer=surt.GoogleURLCanonicalizer, trailing_comma=True)
        return self._surt

    @property
    def priority(self):
        return self.priority_key >> 32

class CrawlUrlQueue:
    def __init__(self):
        # {priority_key:CrawlUrl}
        self._pq = sortedcontainers.SortedDict()
        # {surt:CrawlUrl}
        self._urls = {}
        self.aggregate_priority = 0

    def __len__(self):
        assert len(self._urls) == len(self._pq)
        return len(self._urls)

    def schedule(self, crawl_url):
        self.aggregate_priority += crawl_url.priority

        try:
            old_priority_key = self._urls.pop(crawl_url.surt).priority_key
            old_crawl_url = self._pq.pop(old_priority_key)

            # XXX dumb calculation of new priority, may not belong here
            crawl_url.set_priority(crawl_url.priority + old_crawl_url.priority)
        except KeyError:
            pass

        self._urls[crawl_url.surt] = crawl_url
        self._pq[crawl_url.priority_key] = crawl_url

    def next_url(self):
        res0 = self._pq.popitem(last=True)[1]
        res1 = self._urls.pop(res0.surt)
        assert res0 is res1

        new_low_priority = CrawlUrl(res0.url, -1000)
        self.schedule(new_low_priority)

        return res0

class Site:
    """A seed url, scope definition, and prioritized url queue."""
    def __init__(self, seed_url):
        self.seed = CrawlUrl(seed_url, priority=1000)

        self.q = CrawlUrlQueue()
        self.q.schedule(self.seed)

    def is_in_scope(self, url):
        surtt = surt.surt(url, canonicalizer=surt.GoogleURLCanonicalizer, trailing_comma=True)
        return surtt.startswith(self.seed.surt)

    def submit(self, urls):
        for url in urls:
            if self.is_in_scope(url):
                logging.info("{} accepted {}".format(self.seed.surt, url))
                self.q.schedule(CrawlUrl(url))
            else:
                logging.info("{} rejected {}".format(self.seed.surt, url))

# "browse" + "crawl" = "brozzle"
def brozzle_site(site, chrome_port):
    with umbra.Browser(chrome_port=chrome_port, chrome_exe=args.chrome_exe) as browser:
        while True:
            crawl_url = site.q.next_url()
            outlinks = browser.browse_page(crawl_url.url)
            site.submit(outlinks)

chrome_port = 9200
for seed_url in args.urls:
    site = Site(seed_url)

    th = threading.Thread(target=lambda: brozzle_site(site, chrome_port), 
        name="BrowsingThread-{}".format(site.seed.surt))
    th.start()

    chrome_port += 1