Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-20 12:54:23 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

finish vagrant-brozzler-new-job.py

Browse source
This commit is contained in:
Noah Levitt 2016-10-03 18:17:35 -07:00
parent 5bdf916a78
commit 15633be612
3 changed files with 25 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

2
setup.py
View file

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1b6.dev90', version='1.1b6.dev91',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',

38
vagrant/vagrant-brozzler-new-job.py
View file

@ -6,6 +6,20 @@ queue a job for your vagrant brozzler deployment.
This is a standalone script with no dependencies other than python, and should This is a standalone script with no dependencies other than python, and should
work with python 2.7 or python 3.2+. The only reason it's not a bash script is work with python 2.7 or python 3.2+. The only reason it's not a bash script is
so we can use the argparse library. so we can use the argparse library.
Copyright (C) 2016 Internet Archive
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
''' '''
import sys import sys
@ -20,23 +34,17 @@ def main(argv=[]):
help='brozzler job configuration file in yaml') help='brozzler job configuration file in yaml')
args = arg_parser.parse_args(args=argv[1:]) args = arg_parser.parse_args(args=argv[1:])
with open(args.job_conf_file, 'rb') as f:
yaml_bytes = f.read()
subprocess.call(
['vagrant', 'ssh', '--', 'f=`mktemp` && cat > $f'],
stdin=yaml_bytes)
# cd to path with Vagrantfile so "vagrant ssh" knows what to do # cd to path with Vagrantfile so "vagrant ssh" knows what to do
os.chdir(os.path.dirname(__file__)) os.chdir(os.path.dirname(__file__))
with open(args.job_conf_file, 'rb') as f:
subprocess.call([
'vagrant', 'ssh', '--',
'f=`mktemp` && cat > $f && '
'PYTHONPATH=/home/vagrant/brozzler-ve34/lib/python3.4/site-packages '
'/home/vagrant/brozzler-ve34/bin/python '
'/home/vagrant/brozzler-ve34/bin/brozzler-new-job $f'],
stdin=f)
if __name__ == '__main__': if __name__ == '__main__':
main(sys.argv) main(sys.argv)
## # cd to path with Vagrantfile so "vagrant ssh" knows what to do
## script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
## cd $script_dir
##
## vagrant ssh -- \
## PYTHONPATH=/home/vagrant/brozzler-ve34/lib/python3.4/site-packages \
## /home/vagrant/brozzler-ve34/bin/python \
## /home/vagrant/brozzler-ve34/bin/brozzler-new-job "$@"

2
vagrant/vagrant-brozzler-new-site.py
View file

@ -10,7 +10,7 @@ This is a standalone script with no dependencies other than python, and should
work with python 2.7 or python 3.2+. The only reason it's not a bash script is work with python 2.7 or python 3.2+. The only reason it's not a bash script is
so we can use the argparse library. so we can use the argparse library.
Copyright (C) 2014-2016 Internet Archive Copyright (C) 2016 Internet Archive
Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.