From 9a2f296fa2e6bd42f10b12a81dba2279b8482fcc Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 2 Aug 2016 20:42:30 +0100 Subject: [PATCH] Factor out some of the code shared between the sytest scripts (#974) * Factor out some of the code shared between the different sytest jenkins scripts * Exclude jenkins from the MANIFEST * Fix dendron build * Missing new line * Poke jenkins * Export the PORT_BASE and PORT_COUNT * Poke jenkins --- MANIFEST.in | 2 ++ jenkins-dendron-postgres.sh | 41 ++++++++----------------------------- jenkins-postgres.sh | 29 ++++++++------------------ jenkins-sqlite.sh | 25 ++++++++-------------- jenkins/clone.sh | 24 ++++++++++++++++++++++ jenkins/prepare_synapse.sh | 19 +++++++++++++++++ 6 files changed, 71 insertions(+), 69 deletions(-) create mode 100755 jenkins/clone.sh create mode 100755 jenkins/prepare_synapse.sh diff --git a/MANIFEST.in b/MANIFEST.in index 216df265b..981698143 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -24,5 +24,7 @@ recursive-include synapse/static *.js exclude jenkins.sh exclude jenkins*.sh +exclude jenkins* +recursive-exclude jenkins *.sh prune demo/etc diff --git a/jenkins-dendron-postgres.sh b/jenkins-dendron-postgres.sh index f715cd559..e6e94cc8b 100755 --- a/jenkins-dendron-postgres.sh +++ b/jenkins-dendron-postgres.sh @@ -22,24 +22,10 @@ export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished w rm .coverage* || echo "No coverage files to remove" -tox --notest -e py27 +./jenkins/prepare_synapse.sh -TOX_BIN=$WORKSPACE/.tox/py27/bin -python synapse/python_dependencies.py | xargs -n1 $TOX_BIN/pip install -$TOX_BIN/pip install psycopg2 -$TOX_BIN/pip install lxml - -: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"} - -if [[ ! -e .dendron-base ]]; then - git clone https://github.com/matrix-org/dendron.git .dendron-base --mirror -else - (cd .dendron-base; git fetch -p) -fi - -rm -rf dendron -git clone .dendron-base dendron --shared -cd dendron +./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git +./jenkins/clone.sh dendron https://github.com/matrix-org/dendron.git : ${GOPATH:=${WORKSPACE}/.gopath} if [[ "${GOPATH}" != *:* ]]; then @@ -48,35 +34,26 @@ if [[ "${GOPATH}" != *:* ]]; then fi export GOPATH -git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop) +cd dendron go get github.com/constabulary/gb/... gb generate gb build -cd .. - - -if [[ ! -e .sytest-base ]]; then - git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror -else - (cd .sytest-base; git fetch -p) -fi - -rm -rf sytest -git clone .sytest-base sytest --shared -cd sytest - -git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop) +cd ../sytest : ${PORT_BASE:=20000} : ${PORT_COUNT=100} +export PORT_BASE +export PORT_COUNT ./jenkins/prep_sytest_for_postgres.sh mkdir -p var echo >&2 "Running sytest with PostgreSQL"; + +TOX_BIN=$WORKSPACE/.tox/py27/bin ./jenkins/install_and_run.sh --python $TOX_BIN/python \ --synapse-directory $WORKSPACE \ --dendron $WORKSPACE/dendron/bin/dendron \ diff --git a/jenkins-postgres.sh b/jenkins-postgres.sh index 7a43df0d5..edf61a45b 100755 --- a/jenkins-postgres.sh +++ b/jenkins-postgres.sh @@ -22,37 +22,26 @@ export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished w rm .coverage* || echo "No coverage files to remove" -tox --notest -e py27 +./jenkins/prepare_synapse.sh -TOX_BIN=$WORKSPACE/.tox/py27/bin -python synapse/python_dependencies.py | xargs -n1 $TOX_BIN/pip install -$TOX_BIN/pip install psycopg2 -$TOX_BIN/pip install lxml - -: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"} - -if [[ ! -e .sytest-base ]]; then - git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror -else - (cd .sytest-base; git fetch -p) -fi - -rm -rf sytest -git clone .sytest-base sytest --shared -cd sytest - -git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop) +./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git : ${PORT_BASE:=20000} : ${PORT_COUNT=100} +export PORT_BASE +export PORT_COUNT + +cd sytest ./jenkins/prep_sytest_for_postgres.sh echo >&2 "Running sytest with PostgreSQL"; + +TOX_BIN=$WORKSPACE/.tox/py27/bin ./jenkins/install_and_run.sh --coverage \ --python $TOX_BIN/python \ --synapse-directory $WORKSPACE \ - --port-range ${PORT_BASE}:$((PORT_BASE+PORT_COUNT-1)) \ + --port-range ${PORT_BASE}:$((PORT_BASE+PORT_COUNT-1)) cd .. cp sytest/.coverage.* . diff --git a/jenkins-sqlite.sh b/jenkins-sqlite.sh index 27e61af6e..1c3530ebb 100755 --- a/jenkins-sqlite.sh +++ b/jenkins-sqlite.sh @@ -4,6 +4,7 @@ set -eux : ${WORKSPACE:="$(pwd)"} +export WORKSPACE export PYTHONDONTWRITEBYTECODE=yep export SYNAPSE_CACHE_FACTOR=1 @@ -22,28 +23,18 @@ export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished w rm .coverage* || echo "No coverage files to remove" -tox --notest -e py27 -TOX_BIN=$WORKSPACE/.tox/py27/bin -python synapse/python_dependencies.py | xargs -n1 $TOX_BIN/pip install -$TOX_BIN/pip install lxml +./jenkins/prepare_synapse.sh -: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"} - -if [[ ! -e .sytest-base ]]; then - git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror -else - (cd .sytest-base; git fetch -p) -fi - -rm -rf sytest -git clone .sytest-base sytest --shared -cd sytest - -git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop) +./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git : ${PORT_BASE:=20000} : ${PORT_COUNT=100} +export PORT_BASE +export PORT_COUNT +cd sytest + +TOX_BIN=$WORKSPACE/.tox/py27/bin ./jenkins/install_and_run.sh --coverage \ --python $TOX_BIN/python \ --synapse-directory $WORKSPACE \ diff --git a/jenkins/clone.sh b/jenkins/clone.sh new file mode 100755 index 000000000..f56d076ea --- /dev/null +++ b/jenkins/clone.sh @@ -0,0 +1,24 @@ +#! /bin/bash + +NAME=$1 +PROJECT=$2 +BASE=".$NAME-base" + +# update our clone +if [ ! -d .$NAME-base ]; then + git clone $PROJECT $BASE --mirror +else + (cd $BASE; git fetch -p) +fi + +rm -rf $NAME +git clone $BASE $NAME --shared + +: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"} +cd $NAME +# check out the relevant branch +git checkout "${GIT_BRANCH}" || ( + echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" + git checkout "origin/develop" +) +git clean -df . diff --git a/jenkins/prepare_synapse.sh b/jenkins/prepare_synapse.sh new file mode 100755 index 000000000..237223c81 --- /dev/null +++ b/jenkins/prepare_synapse.sh @@ -0,0 +1,19 @@ +#! /bin/bash + +cd "`dirname $0`/.." + +TOX_DIR=$WORKSPACE/.tox + +mkdir -p $TOX_DIR + +if ! [ $TOX_DIR -ef .tox ]; then + ln -s "$TOX_DIR" .tox +fi + +# set up the virtualenv +tox -e py27 --notest -v + +TOX_BIN=$TOX_DIR/py27/bin +python synapse/python_dependencies.py | xargs -n1 $TOX_BIN/pip install +$TOX_BIN/pip install lxml +$TOX_BIN/pip install psycopg2