brozzler/bin/brozzler-new-site

#!/usr/bin/env python
#
# brozzler-new-site - takes a seed url and creates a site and page object in
# rethinkdb, which brozzler-workers will look at and start crawling
#
# Copyright (C) 2014-2016 Internet Archive
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import argparse
import os
import sys
import logging
import brozzler
import re
import rethinkstuff
import warnings
import requests
import json

arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__),
        description="brozzler-new-site - register site to brozzle",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
arg_parser.add_argument('seed', metavar='SEED', help='seed url')
arg_parser.add_argument('--rethinkdb-servers', dest='rethinkdb_servers', default="localhost",
        help='rethinkdb servers, e.g. db0.foo.org,db0.foo.org:38015,db1.foo.org')
arg_parser.add_argument('--rethinkdb-db', dest='rethinkdb_db', default="brozzler",
        help='rethinkdb database name')
arg_parser.add_argument("--proxy", dest="proxy", default=None, help="http proxy for this site")
arg_parser.add_argument("--time-limit", dest="time_limit", default=None, help="time limit in seconds for this site")
arg_parser.add_argument("--ignore-robots", dest="ignore_robots",
        action="store_true", help="ignore robots.txt for this site")
arg_parser.add_argument('--enable-warcprox-features', dest='enable_warcprox_features',
                action='store_true', help='enable special features for this site that assume the configured proxy is warcprox')
arg_parser.add_argument(
        '--warcprox-meta', dest='warcprox_meta',
        help='Warcprox-Meta http request header to send with each request; '
        'must be a json blob, ignored unless warcprox features are enabled')
arg_parser.add_argument("-v", "--verbose", dest="log_level",
        action="store_const", default=logging.INFO, const=logging.DEBUG)
arg_parser.add_argument("--version", action="version",
        version="brozzler {} - {}".format(brozzler.__version__, os.path.basename(__file__)))
args = arg_parser.parse_args(args=sys.argv[1:])

logging.basicConfig(stream=sys.stdout, level=args.log_level,
        format="%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s")
logging.getLogger("requests.packages.urllib3").setLevel(logging.WARN)
warnings.simplefilter("ignore", category=requests.packages.urllib3.exceptions.InsecureRequestWarning)
warnings.simplefilter("ignore", category=requests.packages.urllib3.exceptions.InsecurePlatformWarning)

site = brozzler.Site(
        seed=args.seed, proxy=args.proxy,
        time_limit=int(args.time_limit) if args.time_limit else None,
        ignore_robots=args.ignore_robots,
        enable_warcprox_features=args.enable_warcprox_features,
        warcprox_meta=json.loads(args.warcprox_meta) if args.warcprox_meta else None)

r = rethinkstuff.Rethinker(args.rethinkdb_servers.split(","), args.rethinkdb_db)
frontier = brozzler.RethinkDbFrontier(r)
brozzler.new_site(frontier, site)