mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-02-24 08:39:59 -05:00
76 lines
2.8 KiB
Python
Executable File
76 lines
2.8 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# vim: set sw=4 et:
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
import logging
|
|
import brozzler
|
|
import yaml
|
|
import json
|
|
|
|
arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__),
|
|
description="brozzler-new-job - queue new job with brozzler",
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
|
arg_parser.add_argument('job_conf_file', metavar='JOB_CONF_FILE', help='brozzler job configuration file in yaml')
|
|
arg_parser.add_argument("--db", dest="db", default="localhost",
|
|
help="comma-separated list of RethinkDB server addresses, e.g. db0.example.com,db0.example.com:39015,db1.example.com")
|
|
arg_parser.add_argument("-v", "--verbose", dest="log_level",
|
|
action="store_const", default=logging.INFO, const=logging.DEBUG)
|
|
arg_parser.add_argument("--version", action="version",
|
|
version="brozzler {} - {}".format(brozzler.version, os.path.basename(__file__)))
|
|
args = arg_parser.parse_args(args=sys.argv[1:])
|
|
|
|
logging.basicConfig(stream=sys.stdout, level=args.log_level,
|
|
format="%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s")
|
|
|
|
def merge(a, b):
|
|
if isinstance(a, dict) and isinstance(b, dict):
|
|
merged = dict(a)
|
|
b_tmp = dict(b)
|
|
for k in a:
|
|
merged[k] = merge(a[k], b_tmp.pop(k, None))
|
|
merged.update(b_tmp)
|
|
return merged
|
|
elif isinstance(a, list) and isinstance(b, list):
|
|
return a + b
|
|
else:
|
|
return a
|
|
|
|
logging.info("loading %s", args.job_conf_file)
|
|
with open(args.job_conf_file) as f:
|
|
job_conf = yaml.load(f)
|
|
logging.info("job_conf=%s", job_conf)
|
|
|
|
seeds = job_conf.pop("seeds")
|
|
# logging.info("=== global settings ===\n%s", yaml.dump(job_conf))
|
|
|
|
sites = []
|
|
for seed_conf in seeds:
|
|
if "id" in seed_conf:
|
|
seed_conf.pop("id")
|
|
merged_conf = merge(seed_conf, job_conf)
|
|
# XXX check for unknown settings, invalid url, etc
|
|
# logging.info("merge(%s, %s) = %s", seed_conf, global_conf, merged_conf)
|
|
# logging.info("=== seed_conf ===\n%s", yaml.dump(seed_conf))
|
|
# logging.info("=== merged_conf ===\n%s", yaml.dump(merged_conf))
|
|
|
|
extra_headers = None
|
|
if "warcprox_meta" in merged_conf:
|
|
warcprox_meta = json.dumps(merged_conf["warcprox_meta"], separators=(',', ':'))
|
|
extra_headers = {"Warcprox-Meta":warcprox_meta}
|
|
|
|
site = brozzler.Site(seed=merged_conf["url"],
|
|
scope=merged_conf.get("scope"),
|
|
time_limit=merged_conf.get("time_limit"),
|
|
proxy=merged_conf.get("proxy"),
|
|
ignore_robots=merged_conf.get("ignore_robots"),
|
|
enable_warcprox_features=merged_conf.get("enable_warcprox_features"),
|
|
extra_headers=extra_headers)
|
|
sites.append(site)
|
|
|
|
db = brozzler.BrozzlerRethinkDb(args.db.split(","))
|
|
for site in sites:
|
|
brozzler.new_site(db, site)
|
|
|