mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-08-03 20:14:19 -04:00
Use black, enforce with GitHub Actions
This commit is contained in:
parent
c4620c3018
commit
8b23430a87
23 changed files with 4048 additions and 2797 deletions
|
@ -1,5 +1,5 @@
|
|||
#!/usr/bin/env python
|
||||
'''
|
||||
"""
|
||||
vagrant-brozzler-new-job.py - runs brozzler-new-job inside the vagrant vm to
|
||||
queue a job for your vagrant brozzler deployment.
|
||||
|
||||
|
@ -20,30 +20,39 @@ distributed under the License is distributed on an "AS IS" BASIS,
|
|||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
'''
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import argparse
|
||||
import subprocess
|
||||
|
||||
|
||||
def main(argv=[]):
|
||||
arg_parser = argparse.ArgumentParser(prog=os.path.basename(argv[0]))
|
||||
arg_parser.add_argument(
|
||||
'job_conf_file', metavar='JOB_CONF_FILE',
|
||||
help='brozzler job configuration file in yaml')
|
||||
"job_conf_file",
|
||||
metavar="JOB_CONF_FILE",
|
||||
help="brozzler job configuration file in yaml",
|
||||
)
|
||||
args = arg_parser.parse_args(args=argv[1:])
|
||||
|
||||
# cd to path with Vagrantfile so "vagrant ssh" knows what to do
|
||||
os.chdir(os.path.dirname(__file__))
|
||||
|
||||
with open(args.job_conf_file, 'rb') as f:
|
||||
subprocess.call([
|
||||
'vagrant', 'ssh', '--',
|
||||
'f=`mktemp` && cat > $f && '
|
||||
'/home/vagrant/brozzler-ve3/bin/python '
|
||||
'/home/vagrant/brozzler-ve3/bin/brozzler-new-job $f'],
|
||||
stdin=f)
|
||||
with open(args.job_conf_file, "rb") as f:
|
||||
subprocess.call(
|
||||
[
|
||||
"vagrant",
|
||||
"ssh",
|
||||
"--",
|
||||
"f=`mktemp` && cat > $f && "
|
||||
"/home/vagrant/brozzler-ve3/bin/python "
|
||||
"/home/vagrant/brozzler-ve3/bin/brozzler-new-job $f",
|
||||
],
|
||||
stdin=f,
|
||||
)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv)
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
#!/usr/bin/env python
|
||||
'''
|
||||
"""
|
||||
vagrant-brozzler-new-site.py - runs brozzler-new-site inside the vagrant vm to
|
||||
queue a site for your vagrant brozzler deployment.
|
||||
|
||||
|
@ -23,61 +23,69 @@ distributed under the License is distributed on an "AS IS" BASIS,
|
|||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
'''
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import argparse
|
||||
import subprocess
|
||||
|
||||
try:
|
||||
from shlex import quote
|
||||
except:
|
||||
from pipes import quote
|
||||
|
||||
|
||||
def main(argv=[]):
|
||||
arg_parser = argparse.ArgumentParser(prog=os.path.basename(argv[0]))
|
||||
arg_parser.add_argument('seed', metavar='SEED', help='seed url')
|
||||
arg_parser.add_argument("seed", metavar="SEED", help="seed url")
|
||||
arg_parser.add_argument(
|
||||
'--time-limit', dest='time_limit', default=None,
|
||||
help='time limit in seconds for this site')
|
||||
"--time-limit",
|
||||
dest="time_limit",
|
||||
default=None,
|
||||
help="time limit in seconds for this site",
|
||||
)
|
||||
arg_parser.add_argument(
|
||||
'--ignore-robots', dest='ignore_robots', action='store_true',
|
||||
help='ignore robots.txt for this site')
|
||||
"--ignore-robots",
|
||||
dest="ignore_robots",
|
||||
action="store_true",
|
||||
help="ignore robots.txt for this site",
|
||||
)
|
||||
arg_parser.add_argument(
|
||||
'--warcprox-meta', dest='warcprox_meta',
|
||||
help=(
|
||||
'Warcprox-Meta http request header to send with each request; '
|
||||
'must be a json blob, ignored unless warcprox features are '
|
||||
'enabled'))
|
||||
arg_parser.add_argument(
|
||||
'-q', '--quiet', dest='quiet', action='store_true')
|
||||
arg_parser.add_argument(
|
||||
'-v', '--verbose', dest='verbose', action='store_true')
|
||||
"--warcprox-meta",
|
||||
dest="warcprox_meta",
|
||||
help=(
|
||||
"Warcprox-Meta http request header to send with each request; "
|
||||
"must be a json blob, ignored unless warcprox features are "
|
||||
"enabled"
|
||||
),
|
||||
)
|
||||
arg_parser.add_argument("-q", "--quiet", dest="quiet", action="store_true")
|
||||
arg_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true")
|
||||
|
||||
args = arg_parser.parse_args(args=argv[1:])
|
||||
|
||||
options = []
|
||||
if args.time_limit:
|
||||
options.append('--time-limit=%s' % args.time_limit)
|
||||
options.append("--time-limit=%s" % args.time_limit)
|
||||
if args.ignore_robots:
|
||||
options.append('--ignore-robots')
|
||||
options.append("--ignore-robots")
|
||||
if args.warcprox_meta:
|
||||
# I think this shell escaping is correct?
|
||||
options.append(
|
||||
'--warcprox-meta=%s' % quote(args.warcprox_meta))
|
||||
options.append("--warcprox-meta=%s" % quote(args.warcprox_meta))
|
||||
if args.quiet:
|
||||
options.append('--quiet')
|
||||
options.append("--quiet")
|
||||
if args.verbose:
|
||||
options.append('--verbose')
|
||||
options.append("--verbose")
|
||||
|
||||
# cd to path with Vagrantfile so "vagrant ssh" knows what to do
|
||||
os.chdir(os.path.dirname(__file__))
|
||||
|
||||
cmd = (
|
||||
'/opt/brozzler-ve3/bin/python /opt/brozzler-ve3/bin/brozzler-new-site '
|
||||
'%s %s') % (' '.join(options), args.seed)
|
||||
subprocess.call(['vagrant', 'ssh', '--', cmd])
|
||||
"/opt/brozzler-ve3/bin/python /opt/brozzler-ve3/bin/brozzler-new-site " "%s %s"
|
||||
) % (" ".join(options), args.seed)
|
||||
subprocess.call(["vagrant", "ssh", "--", cmd])
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv)
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue