Use black, enforce with GitHub Actions

This commit is contained in:
Alex Dempsey 2024-02-08 11:55:23 -08:00
parent c4620c3018
commit 8b23430a87
23 changed files with 4048 additions and 2797 deletions

View file

@ -1,5 +1,5 @@
#!/usr/bin/env python
'''
"""
vagrant-brozzler-new-job.py - runs brozzler-new-job inside the vagrant vm to
queue a job for your vagrant brozzler deployment.
@ -20,30 +20,39 @@ distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
"""
import sys
import os
import argparse
import subprocess
def main(argv=[]):
arg_parser = argparse.ArgumentParser(prog=os.path.basename(argv[0]))
arg_parser.add_argument(
'job_conf_file', metavar='JOB_CONF_FILE',
help='brozzler job configuration file in yaml')
"job_conf_file",
metavar="JOB_CONF_FILE",
help="brozzler job configuration file in yaml",
)
args = arg_parser.parse_args(args=argv[1:])
# cd to path with Vagrantfile so "vagrant ssh" knows what to do
os.chdir(os.path.dirname(__file__))
with open(args.job_conf_file, 'rb') as f:
subprocess.call([
'vagrant', 'ssh', '--',
'f=`mktemp` && cat > $f && '
'/home/vagrant/brozzler-ve3/bin/python '
'/home/vagrant/brozzler-ve3/bin/brozzler-new-job $f'],
stdin=f)
with open(args.job_conf_file, "rb") as f:
subprocess.call(
[
"vagrant",
"ssh",
"--",
"f=`mktemp` && cat > $f && "
"/home/vagrant/brozzler-ve3/bin/python "
"/home/vagrant/brozzler-ve3/bin/brozzler-new-job $f",
],
stdin=f,
)
if __name__ == '__main__':
if __name__ == "__main__":
main(sys.argv)

View file

@ -1,5 +1,5 @@
#!/usr/bin/env python
'''
"""
vagrant-brozzler-new-site.py - runs brozzler-new-site inside the vagrant vm to
queue a site for your vagrant brozzler deployment.
@ -23,61 +23,69 @@ distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
"""
import sys
import os
import argparse
import subprocess
try:
from shlex import quote
except:
from pipes import quote
def main(argv=[]):
arg_parser = argparse.ArgumentParser(prog=os.path.basename(argv[0]))
arg_parser.add_argument('seed', metavar='SEED', help='seed url')
arg_parser.add_argument("seed", metavar="SEED", help="seed url")
arg_parser.add_argument(
'--time-limit', dest='time_limit', default=None,
help='time limit in seconds for this site')
"--time-limit",
dest="time_limit",
default=None,
help="time limit in seconds for this site",
)
arg_parser.add_argument(
'--ignore-robots', dest='ignore_robots', action='store_true',
help='ignore robots.txt for this site')
"--ignore-robots",
dest="ignore_robots",
action="store_true",
help="ignore robots.txt for this site",
)
arg_parser.add_argument(
'--warcprox-meta', dest='warcprox_meta',
help=(
'Warcprox-Meta http request header to send with each request; '
'must be a json blob, ignored unless warcprox features are '
'enabled'))
arg_parser.add_argument(
'-q', '--quiet', dest='quiet', action='store_true')
arg_parser.add_argument(
'-v', '--verbose', dest='verbose', action='store_true')
"--warcprox-meta",
dest="warcprox_meta",
help=(
"Warcprox-Meta http request header to send with each request; "
"must be a json blob, ignored unless warcprox features are "
"enabled"
),
)
arg_parser.add_argument("-q", "--quiet", dest="quiet", action="store_true")
arg_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true")
args = arg_parser.parse_args(args=argv[1:])
options = []
if args.time_limit:
options.append('--time-limit=%s' % args.time_limit)
options.append("--time-limit=%s" % args.time_limit)
if args.ignore_robots:
options.append('--ignore-robots')
options.append("--ignore-robots")
if args.warcprox_meta:
# I think this shell escaping is correct?
options.append(
'--warcprox-meta=%s' % quote(args.warcprox_meta))
options.append("--warcprox-meta=%s" % quote(args.warcprox_meta))
if args.quiet:
options.append('--quiet')
options.append("--quiet")
if args.verbose:
options.append('--verbose')
options.append("--verbose")
# cd to path with Vagrantfile so "vagrant ssh" knows what to do
os.chdir(os.path.dirname(__file__))
cmd = (
'/opt/brozzler-ve3/bin/python /opt/brozzler-ve3/bin/brozzler-new-site '
'%s %s') % (' '.join(options), args.seed)
subprocess.call(['vagrant', 'ssh', '--', cmd])
"/opt/brozzler-ve3/bin/python /opt/brozzler-ve3/bin/brozzler-new-site " "%s %s"
) % (" ".join(options), args.seed)
subprocess.call(["vagrant", "ssh", "--", cmd])
if __name__ == '__main__':
if __name__ == "__main__":
main(sys.argv)