diff --git a/brozzler/cli.py b/brozzler/cli.py index 188d591..458e447 100644 --- a/brozzler/cli.py +++ b/brozzler/cli.py @@ -2,7 +2,7 @@ ''' brozzler/cli.py - brozzler command line executables -Copyright (C) 2014-2017 Internet Archive +Copyright (C) 2014-2019 Internet Archive Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -605,6 +605,10 @@ def brozzler_purge(argv=None): '--site', dest='site', metavar='SITE_ID', help=( 'purge crawl state from rethinkdb for a site, including all ' 'pages')) + group.add_argument( + '--finished-before', dest='finished_before', metavar='YYYY-MM-DD', + help=('purge crawl state from rethinkdb for a jobs that ended ' + 'before this date')) arg_parser.add_argument( '--force', dest='force', action='store_true', help=( 'purge even if job or site is still has status ACTIVE')) @@ -653,6 +657,20 @@ def brozzler_purge(argv=None): '(override with --force)', site_id) sys.exit(1) _purge_site(rr, site_id) + elif args.finished_before: + finished_before = datetime.datetime.strptime( + args.finished_before, '%Y-%m-%d').replace( + tzinfo=doublethink.UTC) + reql = rr.table('jobs').filter( + r.row['finished'].default(r.maxval).lt(finished_before).or_( + r.row['starts_and_stops'].nth(-1)['stop'].default(r.maxval).lt(finished_before))) + logging.debug( + 'retrieving jobs older than %s: %s', finished_before, reql) + for job in reql.run(): + # logging.info('job %s finished=%s starts_and_stops[-1]["stop"]=%s', + # job['id'], job.get('finished'), + # job.get('starts_and_stops', [{'stop':None}])[-1]['stop']) + _purge_job(rr, job['id']) def _purge_site(rr, site_id): reql = rr.table('pages').between(