Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-23 14:24:07 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

generalized support for login doing automatic detection of login form on a page

Browse source
This commit is contained in:
Noah Levitt 2016-12-19 17:30:09 -08:00
parent bc6e0d243f
commit 86ac48d6c3
8 changed files with 213 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

18
brozzler/cli.py
View file

@ -126,6 +126,12 @@ def brozzle_page():
'json blob of parameters to populate the javascript behavior '
'template, e.g. {"parameter_username":"x",'
'"parameter_password":"y"}'))
arg_parser.add_argument(
'--username', dest='username', default=None,
help='use this username to try to log in if a login form is found')
arg_parser.add_argument(
'--password', dest='password', default=None,
help='use this password to try to log in if a login form is found')
arg_parser.add_argument(
'--proxy', dest='proxy', default=None,
help='http proxy')
@ -145,7 +151,8 @@ def brozzle_page():
site = brozzler.Site(
id=-1, seed=args.url, proxy=args.proxy,
enable_warcprox_features=args.enable_warcprox_features,
behavior_parameters=behavior_parameters)
behavior_parameters=behavior_parameters, username=args.username,
password=args.password)
page = brozzler.Page(url=args.url, site_id=site.id)
worker = brozzler.BrozzlerWorker(frontier=None)
@ -230,6 +237,12 @@ def brozzler_new_site():
'json blob of parameters to populate the javascript behavior '
'template, e.g. {"parameter_username":"x",'
'"parameter_password":"y"}'))
arg_parser.add_argument(
'--username', dest='username', default=None,
help='use this username to try to log in if a login form is found')
arg_parser.add_argument(
'--password', dest='password', default=None,
help='use this password to try to log in if a login form is found')
_add_common_options(arg_parser)
args = arg_parser.parse_args(args=sys.argv[1:])
@ -243,7 +256,8 @@ def brozzler_new_site():
warcprox_meta=json.loads(
args.warcprox_meta) if args.warcprox_meta else None,
behavior_parameters=json.loads(
args.behavior_parameters) if args.behavior_parameters else None)
args.behavior_parameters) if args.behavior_parameters else None,
username=args.username, password=args.password)
r = rethinkstuff.Rethinker(
args.rethinkdb_servers.split(","), args.rethinkdb_db)