From b2df100fe5f60cfe7ec36d7e8fbd68e7b10afaa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Tue, 24 Nov 2015 03:12:51 +0100 Subject: [PATCH] development-workflow: add script for inter-VM git and packages copy --- developers/building/development-workflow.md | 313 ++++++++++++++++++++ 1 file changed, 313 insertions(+) diff --git a/developers/building/development-workflow.md b/developers/building/development-workflow.md index 6c75239d..aafd8907 100644 --- a/developers/building/development-workflow.md +++ b/developers/building/development-workflow.md @@ -231,3 +231,316 @@ fname=`basename $file` qvm-run $domain 'mkdir /home/user/incoming/dom0 -p' cat $file| qvm-run --pass-io $domain "cat > /home/user/incoming/dom0/$fname" ~~~ + +## Git connection between VMs + +Sometimes it's useful to transfer git commits between VMs. You can use `git +format-patch` for that and simply copy the files. But you can also setup +custom qrexec service for it. + +Below example assumes that you use `builder-RX` directory in target VM to +store sources in qubes-builder layout (where `X` is some number). Make sure that +all the scripts are executable. + +Service file (save in `/usr/local/etc/qubes-rpc/local.Git` in target VM): + +~~~ +#!/bin/sh + +exec 2>/tmp/log2 + +read service rel repo +echo "Params: $service $rel $repo" >&2 +# Adjust regexps if needed +echo "$repo" | grep -q '^[A-Za-z0-9-]\+$' || exit 1 +echo "$rel" | grep -q '^[0-9.]\+$' || exit 1 +path="/home/user/builder-R$rel/qubes-src/$repo" +if [ "$repo" = "builder" ]; then + path="/home/user/builder-R$rel" +fi +case $service in + git-receive-pack|git-upload-pack) + echo "starting $service $path" >&2 + exec $service $path + ;; + *) + echo "Unsupported service: $service" >&2 + ;; +esac +~~~ + +Client script (save in `~/bin/git-qrexec` in source VM): + +~~~ +#!/bin/sh + +VMNAME=$1 + +(echo $GIT_EXT_SERVICE $2 $3; exec cat) | qrexec-client-vm $VMNAME local.Git +~~~ + +You will also need to setup qrexec policy in dom0 (`/etc/qubes-rpc/policy/local.Git`). + +Usage: + +~~~ +[user@source core-agent-linux]$ git remote add testbuilder "ext::git-qrexec testbuilder 3 core-agent-linux" +[user@source core-agent-linux]$ git push testbuilder master +~~~ + +You can create `~/bin/add-remote` script to ease adding remotes: + +~~~ +#!/bin/sh + +[ -n "$1" ] || exit 1 + +if [ "$1" = "tb" ]; then + git remote add $1 "ext::git-qrexec testbuilder 3 `basename $PWD`" + exit $? +fi + +git remote add $1 git@github.com:$1/qubes-`basename $PWD` +~~~ + +It should be executed from component top level directory. This script takes one +argument - remote name. If it is `tb`, then it creates qrexec-based git remote +to `testbuilder` VM. Otherwise it creates remote pointing at github account of +the same name. In any case it points at repository matching current directory +name. + + +## Sending packages to different VM + +Other useful script(s) can be used to setup local package repository hosted in +some VM. This way you can keep your development VM behind firewall, while +having an option to expose some yum/apt repository to the local network (to +have them installed on test machine). + +To achieve this goal, a dummy repository can be created, which instead of +populating metadata locally, will upload the packages to some other VM and +trigger repository update there (using qrexec). You can use `unstable` +repository flavor, because there is no release managing rules bundled (unlike +current and current-testing). + +### RPM packages - yum repo + +In source VM, grab [linux-yum] repository (below is assumed you've made it in +`~/repo-yum-upload` directory) and replace `update_repo.sh` script with: + +~~~ +#!/bin/sh + +VMNAME=repo-vm + +set -e +qvm-copy-to-vm $VMNAME $1 +# remove only files, leave directory structure +find -type f -name '*.rpm' -delete +# trigger repo update +qrexec-client-vm $VMNAME local.UpdateYum +~~~ + +In target VM, setup actual yum repository (also based on [linux-yum], this time +without modifications). You will also need to setup some gpg key for signing +packages (it is possible to force yum to install unsigned packages, but it +isn't possible for `qubes-dom0-update` tool). Fill `~/.rpmmacros` with +key description: + +~~~ +%_gpg_name Test packages signing key +~~~ + +Then setup `local.UpdateYum` qrexec service (`/usr/local/etc/qubes-rpc/local.UpdateYum`): + +~~~ +#!/bin/sh + +if [ -z "$QREXEC_REMOTE_DOMAIN" ]; then + exit 1 +fi + +real_repository=/home/user/linux-yum +incoming=/home/user/QubesIncoming/$QREXEC_REMOTE_DOMAIN + +find $incoming -name '*.rpm' |xargs rpm -K |grep -iv pgp |cut -f1 -d: |xargs -r setsid -w rpm --addsign 2>&1 + +rsync -lr --remove-source-files $incoming/ $real_repository +cd $real_repository +export SKIP_REPO_CHECK=1 +if [ -d $incoming/r3.1 ]; then + ./update_repo-unstable.sh r3.1 +fi + +if [ -d $incoming/r3.0 ]; then + ./update_repo-unstable.sh r3.0 +fi + +if [ -d $incoming/r2 ]; then + ./update_repo-unstable.sh r2 +fi +find $incoming -type d -empty -delete +exit 0 +~~~ + +Of course you will also need to setup qrexec policy in dom0 +`/etc/qubes-rpc/policy/local.UpdateYum`. + +If you want to access the repository from network, you need to setup HTTP +server serving it, and configure the system to let other machines actually +reach this HTTP server. You can use for example using [port +forwarding][port-forwarding] or setting up Tor hidden service. Configuration +details of those services are outside of the scope of this page. + +Usage: setup `builder.conf` in source VM to use your dummy-uploader repository: + +~~~ +LINUX_REPO_BASEDIR = ../../repo-yum-upload/r3.1 +~~~ + +Then use `make update-repo-unstable` to upload the packages. You can also +specify selected components on command line, then build them and upload to the +repository: + +~~~ +make COMPONENTS="core-agent-linux gui-agent-linux linux-utils" qubes update-repo-unstable +~~~ + +On the test machine, add yum repository (`/etc/yum.repos.d`) pointing at just +configured HTTP server. For example: + +~~~ +[local-test] +name=Test +baseurl=http://local-test.lan/linux-yum/r$releasever/unstable/dom0/fc20 +~~~ + +Remember to also import gpg public key using `rpm --import`. + +### Deb packages - Apt repo + +Steps are mostly the same as in case of yum repo. Only details differs: + + - use [linux-deb] instead of [linux-yum] as a base - both in source and target VM + - use different `update_repo.sh` script in source VM (below) + - use `local.UpdateApt` qrexec service in target VM (code below) + - in target VM additionally place `update-local-repo.sh` script in repository dir (code below) + +`update_repo.sh` script: + +~~~ +#!/bin/sh + +set -e + +current_release=$1 +VMNAME=repo-vm + +qvm-copy-to-vm $VMNAME $1 +find $current_release -type f -name '*.deb' -delete +rm -f $current_release/vm/db/* +qrexec-client-vm $VMNAME local.UpdateApt +~~~ + +`local.UpdateApt` service code (`/usr/local/etc/qubes-rpc/local.UpdateApt` in repo-serving VM): + +~~~ +#!/bin/sh + +if [ -z "$QREXEC_REMOTE_DOMAIN" ]; then + exit 1 +fi + +incoming=/home/user/QubesIncoming/$QREXEC_REMOTE_DOMAIN + +rsync -lr --remove-source-files $incoming/ /home/user/linux-deb/ +cd /home/user/linux-deb +export SKIP_REPO_CHECK=1 +if [ -d $incoming/r3.1 ]; then + for dist in `ls r3.1/vm/dists`; do + ./update-local-repo.sh r3.1/vm $dist + done +fi + +if [ -d $incoming/r3.0 ]; then + for dist in `ls r3.0/vm/dists`; do + ./update-local-repo.sh r3.0/vm $dist + done +fi + +if [ -d $incoming/r2 ]; then + for dist in `ls r2/vm/dists`; do + ./update-local-repo.sh r2/vm $dist + done +fi +find $incoming -type d -empty -delete +exit 0 +~~~ + +`update-local-repo.sh`: + +~~~ +#!/bin/sh + +set -e + +# Set this to your local repository signing key +SIGN_KEY=01ABCDEF + +[ -z "$1" ] && { echo "Usage: $0 "; exit 1; } + +REPO_DIR=$1 +DIST=$2 + +if [ "$DIST" = "wheezy-unstable" ]; then + DIST_TAG=deb7 +elif [ "$DIST" = "jessie-unstable" ]; then + DIST_TAG=deb8 +elif [ "$DIST" = "stretch-unstable" ]; then + DIST_TAG=deb9 +fi + +pushd $REPO_DIR +mkdir -p dists/$DIST/main/binary-amd64 +dpkg-scanpackages --multiversion --arch "*$DIST_TAG*" . > dists/$DIST/main/binary-amd64/Packages +gzip -9c dists/$DIST/main/binary-amd64/Packages > dists/$DIST/main/binary-amd64/Packages.gz +cat > dists/$DIST/Release <> dists/$DIST/Release + +rm -f $DIST/Release.gpg +rm -f $DIST/InRelease +gpg -abs -u "$SIGN_KEY" \ + < dists/$DIST/Release > dists/$DIST/Release.gpg +gpg -a -s --clearsign -u "$SIGN_KEY" \ + < dists/$DIST/Release > dists/$DIST/InRelease +popd + +if [ `id -u` -eq 0 ]; then + chown -R --reference=$REPO_DIR $REPO_DIR +fi +~~~ + +Usage: add this line to `/etc/apt/sources.list` on test machine (adjust host and path): + +~~~ +deb http://local-test.lan/linux-deb/r3.1 jessie-unstable main +~~~ + +[port-forwarding]: /doc/qubes-firewall/#tocAnchor-1-1-5 +[linux-yum]: https://github.com/QubesOS/qubes-linux-yum +[linux-deb]: https://github.com/QubesOS/qubes-linux-deb