#!/usr/bin/env python
# vim: set sw=4 et:

import argparse
import os
import sys
import logging
import brozzler
import kombu
import yaml
import json

arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__),
        description="brozzler-new-job - queue new job with brozzler",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
arg_parser.add_argument('job_conf_file', metavar='JOB_CONF_FILE', help='brozzler job configuration file in yaml')
arg_parser.add_argument('-u', '--amqp-url', dest='amqp_url', default='amqp://guest:guest@localhost:5672/%2f',
        help='URL identifying the amqp server to talk to')
arg_parser.add_argument("-v", "--verbose", dest="log_level",
        action="store_const", default=logging.INFO, const=logging.DEBUG)
arg_parser.add_argument("--version", action="version",
        version="brozzler {} - {}".format(brozzler.version, os.path.basename(__file__)))
args = arg_parser.parse_args(args=sys.argv[1:])

logging.basicConfig(stream=sys.stdout, level=args.log_level,
        format="%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s")

def merge(a, b):
    if isinstance(a, dict) and isinstance(b, dict):
        merged = dict(a)
        b_tmp = dict(b)
        for k in a:
            merged[k] = merge(a[k], b_tmp.pop(k, None))
        merged.update(b_tmp)
        return merged
    elif isinstance(a, list) and isinstance(b, list):
        return a + b
    else:
        return a

logging.info("loading %s", args.job_conf_file)
with open(args.job_conf_file) as f:
    job_conf = yaml.load(f)
logging.info("job_conf=%s", job_conf)

seeds = job_conf.pop("seeds")
# logging.info("=== global settings ===\n%s", yaml.dump(job_conf))

sites = []
for seed_conf in seeds:
    if "id" in seed_conf:
        seed_conf.pop("id")
    merged_conf = merge(seed_conf, job_conf)
    # XXX check for unknown settings, invalid url, etc
    # logging.info("merge(%s, %s) = %s", seed_conf, global_conf, merged_conf)
    # logging.info("=== seed_conf ===\n%s", yaml.dump(seed_conf))
    # logging.info("=== merged_conf ===\n%s", yaml.dump(merged_conf))

    extra_headers = None
    if "warcprox_meta" in merged_conf:
        warcprox_meta = json.dumps(merged_conf["warcprox_meta"], separators=(',', ':'))
        extra_headers = {"Warcprox-Meta":warcprox_meta}
    site = brozzler.Site(seed=merged_conf["url"],
            scope=merged_conf.get("scope"),
            time_limit=merged_conf.get("time_limit"),
            proxy=merged_conf.get("proxy"),
            ignore_robots=merged_conf.get("ignore_robots"),
            enable_warcprox_features=merged_conf.get("enable_warcprox_features"),
            extra_headers=extra_headers)
    sites.append(site)

with kombu.Connection(args.amqp_url) as conn:
    q = conn.SimpleQueue("brozzler.sites.new")
    for site in sites:
        d = site.to_dict()
        logging.info("feeding amqp queue %s with %s", repr(q.queue.name), repr(d))
        q.put(d)