mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-02-24 08:39:59 -05:00
49 lines
2.3 KiB
Python
Executable File
49 lines
2.3 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# vim: set sw=4 et:
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
import logging
|
|
import brozzler
|
|
import re
|
|
|
|
arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__),
|
|
description="brozzler-new-site - register site to brozzle",
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
|
arg_parser.add_argument('seed', metavar='SEED', help='seed url')
|
|
arg_parser.add_argument("--db", dest="db", default="localhost",
|
|
help="comma-separated list of RethinkDB server addresses, e.g. db0.example.com,db0.example.com:39015,db1.example.com")
|
|
arg_parser.add_argument("--proxy", dest="proxy", default=None, help="http proxy for this site")
|
|
arg_parser.add_argument("--time-limit", dest="time_limit", default=None, help="time limit in seconds for this site")
|
|
arg_parser.add_argument("-H", "--extra-header", action="append",
|
|
dest="extra_headers", default=None, help="extra http header to send with every request for this site (may be used multiple times)")
|
|
arg_parser.add_argument("--ignore-robots", dest="ignore_robots",
|
|
action="store_true", help="ignore robots.txt for this site")
|
|
arg_parser.add_argument('--enable-warcprox-features', dest='enable_warcprox_features',
|
|
action='store_true', help='enable special features for this site that assume the configured proxy is warcprox')
|
|
arg_parser.add_argument("-v", "--verbose", dest="log_level",
|
|
action="store_const", default=logging.INFO, const=logging.DEBUG)
|
|
arg_parser.add_argument("--version", action="version",
|
|
version="brozzler {} - {}".format(brozzler.version, os.path.basename(__file__)))
|
|
args = arg_parser.parse_args(args=sys.argv[1:])
|
|
|
|
logging.basicConfig(stream=sys.stdout, level=args.log_level,
|
|
format="%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s")
|
|
|
|
extra_headers = {}
|
|
if args.extra_headers:
|
|
for hh in args.extra_headers:
|
|
[k,v] = re.split(r":\s*", hh, 1)
|
|
extra_headers[k] = v
|
|
|
|
site = brozzler.Site(seed=args.seed, proxy=args.proxy,
|
|
time_limit=int(args.time_limit) if args.time_limit else None,
|
|
ignore_robots=args.ignore_robots,
|
|
enable_warcprox_features=args.enable_warcprox_features,
|
|
extra_headers=extra_headers)
|
|
|
|
frontier = brozzler.RethinkDbFrontier(args.db.split(","))
|
|
brozzler.new_site(frontier, site)
|
|
|