From 5438dbf13c99bd7abd4f97aed475b092450d5490 Mon Sep 17 00:00:00 2001 From: LinuxServer-CI Date: Sat, 29 Apr 2023 03:16:05 +0000 Subject: [PATCH 1/4] Bot Updating Templated Files --- Jenkinsfile | 91 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b859cf3..86696fe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,10 +40,11 @@ pipeline { // Setup all the basic environment variables needed for the build stage("Set ENV Variables base"){ steps{ + sh '''docker pull quay.io/skopeo/stable:v1 || : ''' script{ env.EXIT_STATUS = '' env.LS_RELEASE = sh( - script: '''docker run --rm ghcr.io/linuxserver/alexeiled-skopeo sh -c 'skopeo inspect docker://docker.io/'${DOCKERHUB_IMAGE}':latest 2>/dev/null' | jq -r '.Labels.build_version' | awk '{print $3}' | grep '\\-ls' || : ''', + script: '''docker run --rm quay.io/skopeo/stable:v1 inspect docker://ghcr.io/${LS_USER}/${CONTAINER_NAME}:latest 2>/dev/null | jq -r '.Labels.build_version' | awk '{print $3}' | grep '\\-ls' || : ''', returnStdout: true).trim() env.LS_RELEASE_NOTES = sh( script: '''cat readme-vars.yml | awk -F \\" '/date: "[0-9][0-9].[0-9][0-9].[0-9][0-9]:/ {print $4;exit;}' | sed -E ':a;N;$!ba;s/\\r{0,1}\\n/\\\\n/g' ''', @@ -228,7 +229,7 @@ pipeline { script{ env.SHELLCHECK_URL = 'https://ci-tests.linuxserver.io/' + env.IMAGE + '/' + env.META_TAG + '/shellcheck-result.xml' } - sh '''curl -sL https://raw.githubusercontent.com/linuxserver/docker-shellcheck/master/checkrun.sh | /bin/bash''' + sh '''curl -sL https://raw.githubusercontent.com/linuxserver/docker-jenkins-builder/master/checkrun.sh | /bin/bash''' sh '''#! /bin/bash docker run --rm \ -v ${WORKSPACE}:/mnt \ @@ -376,6 +377,26 @@ pipeline { } } } + // If this is a master build check the S6 service file perms + stage("Check S6 Service file Permissions"){ + when { + branch "master" + environment name: 'CHANGE_ID', value: '' + environment name: 'EXIT_STATUS', value: '' + } + steps { + script{ + sh '''#! /bin/bash + WRONG_PERM=$(find ./ -path "./.git" -prune -o \\( -name "run" -o -name "finish" -o -name "check" \\) -not -perm -u=x,g=x,o=x -print) + if [[ -n "${WRONG_PERM}" ]]; then + echo "The following S6 service files are missing the executable bit; canceling the faulty build: ${WRONG_PERM}" + exit 1 + else + echo "S6 service file perms look good." + fi ''' + } + } + } /* ####################### GitLab Mirroring ####################### */ @@ -668,6 +689,7 @@ pipeline { ]) { script{ env.CI_URL = 'https://ci-tests.linuxserver.io/' + env.IMAGE + '/' + env.META_TAG + '/index.html' + env.CI_JSON_URL = 'https://ci-tests.linuxserver.io/' + env.IMAGE + '/' + env.META_TAG + '/report.json' } sh '''#! /bin/bash set -e @@ -694,8 +716,6 @@ pipeline { -e WEB_SCREENSHOT=\"${CI_WEB}\" \ -e WEB_AUTH=\"${CI_AUTH}\" \ -e WEB_PATH=\"${CI_WEBPATH}\" \ - -e DO_REGION="ams3" \ - -e DO_BUCKET="lsio-ci" \ -t ghcr.io/linuxserver/ci:latest \ python3 test_build.py''' } @@ -949,8 +969,67 @@ pipeline { environment name: 'EXIT_STATUS', value: '' } steps { - sh '''curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST https://api.github.com/repos/${LS_USER}/${LS_REPO}/issues/${PULL_REQUEST}/comments \ - -d '{"body": "I am a bot, here are the test results for this PR: \\n'${CI_URL}' \\n'${SHELLCHECK_URL}'"}' ''' + sh '''#! /bin/bash + # Function to retrieve JSON data from URL + get_json() { + local url="$1" + local response=$(curl -s "$url") + if [ $? -ne 0 ]; then + echo "Failed to retrieve JSON data from $url" + return 1 + fi + local json=$(echo "$response" | jq .) + if [ $? -ne 0 ]; then + echo "Failed to parse JSON data from $url" + return 1 + fi + echo "$json" + } + + build_table() { + local data="$1" + + # Get the keys in the JSON data + local keys=$(echo "$data" | jq -r 'to_entries | map(.key) | .[]') + + # Check if keys are empty + if [ -z "$keys" ]; then + echo "JSON report data does not contain any keys or the report does not exist." + return 1 + fi + + # Build table header + local header="| Tag | Passed |\\n| --- | --- |\\n" + + # Loop through the JSON data to build the table rows + local rows="" + for build in $keys; do + local status=$(echo "$data" | jq -r ".[\\"$build\\"].test_success") + if [ "$status" = "true" ]; then + status="✅" + else + status="❌" + fi + local row="| "$build" | "$status" |\\n" + rows="${rows}${row}" + done + + local table="${header}${rows}" + local escaped_table=$(echo "$table" | sed 's/\"/\\\\"/g') + echo "$escaped_table" + } + + # Retrieve JSON data from URL + data=$(get_json "$CI_JSON_URL") + # Create table from JSON data + table=$(build_table "$data") + echo -e "$table" + + curl -X POST -H "Authorization: token $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/$LS_USER/$LS_REPO/issues/$PULL_REQUEST/comments" \ + -d "{\\"body\\": \\"I am a bot, here are the test results for this PR: \\n${CI_URL}\\n${SHELLCHECK_URL}\\n${table}\\"}"''' + } } } From dee14d718b1fe2293220ece8791cd339fa6f7268 Mon Sep 17 00:00:00 2001 From: LinuxServer-CI Date: Sat, 29 Apr 2023 03:17:04 +0000 Subject: [PATCH 2/4] Bot Updating Templated Files --- .github/workflows/call_issue_pr_tracker.yml | 6 ++++-- .github/workflows/external_trigger.yml | 8 ++++++++ .../workflows/external_trigger_scheduler.yml | 18 ++++++++++-------- .github/workflows/package_trigger.yml | 4 ++++ .../workflows/package_trigger_scheduler.yml | 8 ++++---- 5 files changed, 30 insertions(+), 14 deletions(-) diff --git a/.github/workflows/call_issue_pr_tracker.yml b/.github/workflows/call_issue_pr_tracker.yml index 87243e2..2c30784 100755 --- a/.github/workflows/call_issue_pr_tracker.yml +++ b/.github/workflows/call_issue_pr_tracker.yml @@ -2,9 +2,11 @@ name: Issue & PR Tracker on: issues: - types: [opened,reopened,labeled,unlabeled] + types: [opened,reopened,labeled,unlabeled,closed] pull_request_target: - types: [opened,reopened,review_requested,review_request_removed,labeled,unlabeled] + types: [opened,reopened,review_requested,review_request_removed,labeled,unlabeled,closed] + pull_request_review: + types: [submitted,edited,dismissed] jobs: manage-project: diff --git a/.github/workflows/external_trigger.yml b/.github/workflows/external_trigger.yml index 83b56e0..964c9b3 100644 --- a/.github/workflows/external_trigger.yml +++ b/.github/workflows/external_trigger.yml @@ -14,9 +14,11 @@ jobs: run: | if [ -n "${{ secrets.PAUSE_EXTERNAL_TRIGGER_SWAG_MASTER }}" ]; then echo "**** Github secret PAUSE_EXTERNAL_TRIGGER_SWAG_MASTER is set; skipping trigger. ****" + echo "Github secret \`PAUSE_EXTERNAL_TRIGGER_SWAG_MASTER\` is set; skipping trigger." >> $GITHUB_STEP_SUMMARY exit 0 fi echo "**** External trigger running off of master branch. To disable this trigger, set a Github secret named \"PAUSE_EXTERNAL_TRIGGER_SWAG_MASTER\". ****" + echo "External trigger running off of master branch. To disable this trigger, set a Github secret named \`PAUSE_EXTERNAL_TRIGGER_SWAG_MASTER\`" >> $GITHUB_STEP_SUMMARY echo "**** Retrieving external version ****" EXT_RELEASE=$(curl -sL "https://pypi.python.org/pypi/certbot/json" |jq -r '. | .info.version') if [ -z "${EXT_RELEASE}" ] || [ "${EXT_RELEASE}" == "null" ]; then @@ -30,6 +32,7 @@ jobs: fi EXT_RELEASE=$(echo ${EXT_RELEASE} | sed 's/[~,%@+;:/]//g') echo "**** External version: ${EXT_RELEASE} ****" + echo "External version: ${EXT_RELEASE}" >> $GITHUB_STEP_SUMMARY echo "**** Retrieving last pushed version ****" image="linuxserver/swag" tag="latest" @@ -65,14 +68,18 @@ jobs: exit 1 fi echo "**** Last pushed version: ${IMAGE_VERSION} ****" + echo "Last pushed version: ${IMAGE_VERSION}" >> $GITHUB_STEP_SUMMARY if [ "${EXT_RELEASE}" == "${IMAGE_VERSION}" ]; then echo "**** Version ${EXT_RELEASE} already pushed, exiting ****" + echo "Version ${EXT_RELEASE} already pushed, exiting" >> $GITHUB_STEP_SUMMARY exit 0 elif [ $(curl -s https://ci.linuxserver.io/job/Docker-Pipeline-Builders/job/docker-swag/job/master/lastBuild/api/json | jq -r '.building') == "true" ]; then echo "**** New version ${EXT_RELEASE} found; but there already seems to be an active build on Jenkins; exiting ****" + echo "New version ${EXT_RELEASE} found; but there already seems to be an active build on Jenkins; exiting" >> $GITHUB_STEP_SUMMARY exit 0 else echo "**** New version ${EXT_RELEASE} found; old version was ${IMAGE_VERSION}. Triggering new build ****" + echo "New version ${EXT_RELEASE} found; old version was ${IMAGE_VERSION}. Triggering new build" >> $GITHUB_STEP_SUMMARY response=$(curl -iX POST \ https://ci.linuxserver.io/job/Docker-Pipeline-Builders/job/docker-swag/job/master/buildWithParameters?PACKAGE_CHECK=false \ --user ${{ secrets.JENKINS_USER }}:${{ secrets.JENKINS_TOKEN }} | grep -i location | sed "s|^[L|l]ocation: \(.*\)|\1|") @@ -82,6 +89,7 @@ jobs: buildurl=$(curl -s "${response%$'\r'}api/json" | jq -r '.executable.url') buildurl="${buildurl%$'\r'}" echo "**** Jenkins job build url: ${buildurl} ****" + echo "Jenkins job build url: ${buildurl}" >> $GITHUB_STEP_SUMMARY echo "**** Attempting to change the Jenkins job description ****" curl -iX POST \ "${buildurl}submitDescription" \ diff --git a/.github/workflows/external_trigger_scheduler.yml b/.github/workflows/external_trigger_scheduler.yml index 05db26d..353eb4b 100644 --- a/.github/workflows/external_trigger_scheduler.yml +++ b/.github/workflows/external_trigger_scheduler.yml @@ -2,7 +2,7 @@ name: External Trigger Scheduler on: schedule: - - cron: '50 * * * *' + - cron: '2 * * * *' workflow_dispatch: jobs: @@ -17,18 +17,18 @@ jobs: run: | echo "**** Branches found: ****" git for-each-ref --format='%(refname:short)' refs/remotes - echo "**** Pulling the yq docker image ****" - docker pull ghcr.io/linuxserver/yq for br in $(git for-each-ref --format='%(refname:short)' refs/remotes) do br=$(echo "$br" | sed 's|origin/||g') echo "**** Evaluating branch ${br} ****" - ls_branch=$(curl -sX GET https://raw.githubusercontent.com/linuxserver/docker-swag/${br}/jenkins-vars.yml \ - | docker run --rm -i --entrypoint yq ghcr.io/linuxserver/yq -r .ls_branch) - if [ "$br" == "$ls_branch" ]; then - echo "**** Branch ${br} appears to be live; checking workflow. ****" + ls_jenkins_vars=$(curl -sX GET https://raw.githubusercontent.com/linuxserver/docker-swag/${br}/jenkins-vars.yml) + ls_branch=$(echo "${ls_jenkins_vars}" | yq -r '.ls_branch') + ls_trigger=$(echo "${ls_jenkins_vars}" | yq -r '.external_type') + if [[ "${br}" == "${ls_branch}" ]] && [[ "${ls_trigger}" != "os" ]]; then + echo "**** Branch ${br} appears to be live and trigger is not os; checking workflow. ****" if curl -sfX GET https://raw.githubusercontent.com/linuxserver/docker-swag/${br}/.github/workflows/external_trigger.yml > /dev/null 2>&1; then echo "**** Workflow exists. Triggering external trigger workflow for branch ${br} ****." + echo "Triggering external trigger workflow for branch ${br}" >> $GITHUB_STEP_SUMMARY curl -iX POST \ -H "Authorization: token ${{ secrets.CR_PAT }}" \ -H "Accept: application/vnd.github.v3+json" \ @@ -36,8 +36,10 @@ jobs: https://api.github.com/repos/linuxserver/docker-swag/actions/workflows/external_trigger.yml/dispatches else echo "**** Workflow doesn't exist; skipping trigger. ****" + echo "Skipping branch ${br} due to no external trigger workflow present." >> $GITHUB_STEP_SUMMARY fi else - echo "**** ${br} appears to be a dev branch; skipping trigger. ****" + echo "**** ${br} is either a dev branch, or has no external version; skipping trigger. ****" + echo "Skipping branch ${br} due to being detected as dev branch or having no external version." >> $GITHUB_STEP_SUMMARY fi done diff --git a/.github/workflows/package_trigger.yml b/.github/workflows/package_trigger.yml index 35847d5..7e71dfb 100644 --- a/.github/workflows/package_trigger.yml +++ b/.github/workflows/package_trigger.yml @@ -14,13 +14,16 @@ jobs: run: | if [ -n "${{ secrets.PAUSE_PACKAGE_TRIGGER_SWAG_MASTER }}" ]; then echo "**** Github secret PAUSE_PACKAGE_TRIGGER_SWAG_MASTER is set; skipping trigger. ****" + echo "Github secret \`PAUSE_PACKAGE_TRIGGER_SWAG_MASTER\` is set; skipping trigger." >> $GITHUB_STEP_SUMMARY exit 0 fi if [ $(curl -s https://ci.linuxserver.io/job/Docker-Pipeline-Builders/job/docker-swag/job/master/lastBuild/api/json | jq -r '.building') == "true" ]; then echo "**** There already seems to be an active build on Jenkins; skipping package trigger ****" + echo "There already seems to be an active build on Jenkins; skipping package trigger" >> $GITHUB_STEP_SUMMARY exit 0 fi echo "**** Package trigger running off of master branch. To disable, set a Github secret named \"PAUSE_PACKAGE_TRIGGER_SWAG_MASTER\". ****" + echo "Package trigger running off of master branch. To disable, set a Github secret named \`PAUSE_PACKAGE_TRIGGER_SWAG_MASTER\`" >> $GITHUB_STEP_SUMMARY response=$(curl -iX POST \ https://ci.linuxserver.io/job/Docker-Pipeline-Builders/job/docker-swag/job/master/buildWithParameters?PACKAGE_CHECK=true \ --user ${{ secrets.JENKINS_USER }}:${{ secrets.JENKINS_TOKEN }} | grep -i location | sed "s|^[L|l]ocation: \(.*\)|\1|") @@ -30,6 +33,7 @@ jobs: buildurl=$(curl -s "${response%$'\r'}api/json" | jq -r '.executable.url') buildurl="${buildurl%$'\r'}" echo "**** Jenkins job build url: ${buildurl} ****" + echo "Jenkins job build url: ${buildurl}" >> $GITHUB_STEP_SUMMARY echo "**** Attempting to change the Jenkins job description ****" curl -iX POST \ "${buildurl}submitDescription" \ diff --git a/.github/workflows/package_trigger_scheduler.yml b/.github/workflows/package_trigger_scheduler.yml index b92d6f4..b912788 100644 --- a/.github/workflows/package_trigger_scheduler.yml +++ b/.github/workflows/package_trigger_scheduler.yml @@ -17,18 +17,16 @@ jobs: run: | echo "**** Branches found: ****" git for-each-ref --format='%(refname:short)' refs/remotes - echo "**** Pulling the yq docker image ****" - docker pull ghcr.io/linuxserver/yq for br in $(git for-each-ref --format='%(refname:short)' refs/remotes) do br=$(echo "$br" | sed 's|origin/||g') echo "**** Evaluating branch ${br} ****" - ls_branch=$(curl -sX GET https://raw.githubusercontent.com/linuxserver/docker-swag/${br}/jenkins-vars.yml \ - | docker run --rm -i --entrypoint yq ghcr.io/linuxserver/yq -r .ls_branch) + ls_branch=$(curl -sX GET https://raw.githubusercontent.com/linuxserver/docker-swag/${br}/jenkins-vars.yml | yq -r '.ls_branch') if [ "${br}" == "${ls_branch}" ]; then echo "**** Branch ${br} appears to be live; checking workflow. ****" if curl -sfX GET https://raw.githubusercontent.com/linuxserver/docker-swag/${br}/.github/workflows/package_trigger.yml > /dev/null 2>&1; then echo "**** Workflow exists. Triggering package trigger workflow for branch ${br}. ****" + echo "Triggering package trigger workflow for branch ${br}" >> $GITHUB_STEP_SUMMARY triggered_branches="${triggered_branches}${br} " curl -iX POST \ -H "Authorization: token ${{ secrets.CR_PAT }}" \ @@ -38,9 +36,11 @@ jobs: sleep 30 else echo "**** Workflow doesn't exist; skipping trigger. ****" + echo "Skipping branch ${br} due to no package trigger workflow present." >> $GITHUB_STEP_SUMMARY fi else echo "**** ${br} appears to be a dev branch; skipping trigger. ****" + echo "Skipping branch ${br} due to being detected as dev branch." >> $GITHUB_STEP_SUMMARY fi done echo "**** Package check build(s) triggered for branch(es): ${triggered_branches} ****" From 24f3353692d616afa1b66e4a45e9ebf4fcc5aa3e Mon Sep 17 00:00:00 2001 From: LinuxServer-CI Date: Sat, 29 Apr 2023 03:20:15 +0000 Subject: [PATCH 3/4] Bot Updating Package Versions --- package_versions.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/package_versions.txt b/package_versions.txt index 7852381..2b0fb57 100755 --- a/package_versions.txt +++ b/package_versions.txt @@ -21,8 +21,8 @@ azure-mgmt-core 1.4.0 python azure-mgmt-dns 8.0.0 python bash 5.2.15-r0 apk beautifulsoup4 4.12.2 python -boto3 1.26.118 python -botocore 1.29.118 python +boto3 1.26.123 python +botocore 1.29.123 python brotli-libs 1.0.9-r9 apk bs4 0.0.1 python busybox 1.35.0 binary @@ -93,8 +93,8 @@ fontconfig 2.14.1-r0 apk freetype 2.12.1-r0 apk future 0.18.3 python gdbm 1.23-r0 apk -git 2.38.4-r1 apk -git-perl 2.38.4-r1 apk +git 2.38.5-r0 apk +git-perl 2.38.5-r0 apk gmp 6.2.1-r2 apk gnupg 2.2.40-r0 apk gnupg-dirmngr 2.2.40-r0 apk @@ -116,7 +116,7 @@ httplib2 0.22.0 python icu-data-en 72.1-r1 apk icu-libs 72.1-r1 apk idna 3.4 python -importlib-metadata 6.5.1 python +importlib-metadata 6.6.0 python ip6tables 1.8.8-r2 apk iptables 1.8.8-r2 apk isodate 0.6.1 python @@ -232,7 +232,7 @@ pcre 8.45-r2 apk pcre2 10.42-r0 apk perl 5.36.0-r1 apk perl-error 0.17029-r1 apk -perl-git 2.38.4-r1 apk +perl-git 2.38.5-r0 apk php-cli 8.1.18 binary php-fpm 8.1.18 binary php81 8.1.18-r0 apk @@ -286,7 +286,7 @@ php81-xmlwriter 8.1.18-r0 apk php81-xsl 8.1.18-r0 apk php81-zip 8.1.18-r0 apk pinentry 1.2.1-r0 apk -pip 23.1 python +pip 23.1.2 python pkb-client 1.2 python popt 1.19-r0 apk portalocker 2.7.0 python @@ -307,7 +307,7 @@ python-transip 0.6.0 python python3 3.10.11-r0 apk pytz 2023.3 python readline 8.2.0-r0 apk -requests 2.28.2 python +requests 2.29.0 python requests-file 1.5.1 python requests-mock 1.10.0 python requests-oauthlib 1.3.1 python @@ -322,7 +322,7 @@ soupsieve 2.4.1 python sqlite-libs 3.40.1-r0 apk ssl_client 1.35.0-r29 apk tiff 4.4.0-r3 apk -tldextract 3.4.0 python +tldextract 3.4.1 python typing-inspect 0.8.0 python typing_extensions 4.5.0 python tzdata 2023c-r0 apk From 10d1bf7b96f1213da2c532451dee72468edd223d Mon Sep 17 00:00:00 2001 From: LinuxServer-CI Date: Sat, 6 May 2023 03:20:14 +0000 Subject: [PATCH 4/4] Bot Updating Package Versions --- package_versions.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package_versions.txt b/package_versions.txt index 2b0fb57..886f3ea 100755 --- a/package_versions.txt +++ b/package_versions.txt @@ -21,8 +21,8 @@ azure-mgmt-core 1.4.0 python azure-mgmt-dns 8.0.0 python bash 5.2.15-r0 apk beautifulsoup4 4.12.2 python -boto3 1.26.123 python -botocore 1.29.123 python +boto3 1.26.129 python +botocore 1.29.129 python brotli-libs 1.0.9-r9 apk bs4 0.0.1 python busybox 1.35.0 binary @@ -269,7 +269,7 @@ php81-pecl-mailparse 3.1.4-r0 apk php81-pecl-mcrypt 1.0.6-r0 apk php81-pecl-memcached 3.2.0-r0 apk php81-pecl-redis 5.3.7-r0 apk -php81-pecl-xmlrpc 1.0.0_rc3-r0 apk +php81-pecl-xmlrpc 1.0.0_rc3-r1 apk php81-pgsql 8.1.18-r0 apk php81-phar 8.1.18-r0 apk php81-posix 8.1.18-r0 apk @@ -291,7 +291,7 @@ pkb-client 1.2 python popt 1.19-r0 apk portalocker 2.7.0 python procps 3.3.17-r2 apk -protobuf 4.22.3 python +protobuf 4.22.4 python publicsuffixlist 0.9.4 python pyOpenSSL 23.1.1 python pyRFC3339 1.1 python @@ -307,12 +307,12 @@ python-transip 0.6.0 python python3 3.10.11-r0 apk pytz 2023.3 python readline 8.2.0-r0 apk -requests 2.29.0 python +requests 2.30.0 python requests-file 1.5.1 python requests-mock 1.10.0 python requests-oauthlib 1.3.1 python rsa 4.9 python -s3transfer 0.6.0 python +s3transfer 0.6.1 python scanelf 1.3.5-r1 apk setuptools 65.5.0 python shadow 4.13-r0 apk