Merge branch 'master' into certbot-revamp-config-file

.github/workflows/call_issue_pr_tracker.yml

@@ -2,9 +2,11 @@ name: Issue & PR Tracker
 
 on:
   issues:
-    types: [opened,reopened,labeled,unlabeled]
+    types: [opened,reopened,labeled,unlabeled,closed]
   pull_request_target:
-    types: [opened,reopened,review_requested,review_request_removed,labeled,unlabeled]
+    types: [opened,reopened,review_requested,review_request_removed,labeled,unlabeled,closed]
+  pull_request_review:
+    types: [submitted,edited,dismissed]
 
 jobs:
   manage-project:

.github/workflows/external_trigger.yml

@@ -14,9 +14,11 @@ jobs:
         run: |
           if [ -n "${{ secrets.PAUSE_EXTERNAL_TRIGGER_SWAG_MASTER }}" ]; then
             echo "**** Github secret PAUSE_EXTERNAL_TRIGGER_SWAG_MASTER is set; skipping trigger. ****"
+            echo "Github secret \`PAUSE_EXTERNAL_TRIGGER_SWAG_MASTER\` is set; skipping trigger." >> $GITHUB_STEP_SUMMARY
             exit 0
           fi
           echo "**** External trigger running off of master branch. To disable this trigger, set a Github secret named \"PAUSE_EXTERNAL_TRIGGER_SWAG_MASTER\". ****"
+          echo "External trigger running off of master branch. To disable this trigger, set a Github secret named \`PAUSE_EXTERNAL_TRIGGER_SWAG_MASTER\`" >> $GITHUB_STEP_SUMMARY
           echo "**** Retrieving external version ****"
           EXT_RELEASE=$(curl -sL "https://pypi.python.org/pypi/certbot/json" |jq -r '. | .info.version')
           if [ -z "${EXT_RELEASE}" ] || [ "${EXT_RELEASE}" == "null" ]; then
@@ -30,6 +32,7 @@ jobs:
           fi
           EXT_RELEASE=$(echo ${EXT_RELEASE} | sed 's/[~,%@+;:/]//g')
           echo "**** External version: ${EXT_RELEASE} ****"
+          echo "External version: ${EXT_RELEASE}" >> $GITHUB_STEP_SUMMARY
           echo "**** Retrieving last pushed version ****"
           image="linuxserver/swag"
           tag="latest"
@@ -65,14 +68,18 @@ jobs:
             exit 1
           fi
           echo "**** Last pushed version: ${IMAGE_VERSION} ****"
+          echo "Last pushed version: ${IMAGE_VERSION}" >> $GITHUB_STEP_SUMMARY
           if [ "${EXT_RELEASE}" == "${IMAGE_VERSION}" ]; then
             echo "**** Version ${EXT_RELEASE} already pushed, exiting ****"
+            echo "Version ${EXT_RELEASE} already pushed, exiting" >> $GITHUB_STEP_SUMMARY
             exit 0
           elif [ $(curl -s https://ci.linuxserver.io/job/Docker-Pipeline-Builders/job/docker-swag/job/master/lastBuild/api/json | jq -r '.building') == "true" ]; then
             echo "**** New version ${EXT_RELEASE} found; but there already seems to be an active build on Jenkins; exiting ****"
+            echo "New version ${EXT_RELEASE} found; but there already seems to be an active build on Jenkins; exiting" >> $GITHUB_STEP_SUMMARY
             exit 0
           else
             echo "**** New version ${EXT_RELEASE} found; old version was ${IMAGE_VERSION}. Triggering new build ****"
+            echo "New version ${EXT_RELEASE} found; old version was ${IMAGE_VERSION}. Triggering new build" >> $GITHUB_STEP_SUMMARY
             response=$(curl -iX POST \
               https://ci.linuxserver.io/job/Docker-Pipeline-Builders/job/docker-swag/job/master/buildWithParameters?PACKAGE_CHECK=false \
               --user ${{ secrets.JENKINS_USER }}:${{ secrets.JENKINS_TOKEN }} | grep -i location | sed "s|^[L|l]ocation: \(.*\)|\1|")
@@ -82,6 +89,7 @@ jobs:
             buildurl=$(curl -s "${response%$'\r'}api/json" | jq -r '.executable.url')
             buildurl="${buildurl%$'\r'}"
             echo "**** Jenkins job build url: ${buildurl} ****"
+            echo "Jenkins job build url: ${buildurl}" >> $GITHUB_STEP_SUMMARY
             echo "**** Attempting to change the Jenkins job description ****"
             curl -iX POST \
               "${buildurl}submitDescription" \

.github/workflows/external_trigger_scheduler.yml

@@ -2,7 +2,7 @@ name: External Trigger Scheduler
 
 on:
   schedule:
-    - cron:  '50 * * * *'
+    - cron:  '2 * * * *'
   workflow_dispatch:
 
 jobs:
@@ -17,18 +17,18 @@ jobs:
         run: |
           echo "**** Branches found: ****"
           git for-each-ref --format='%(refname:short)' refs/remotes
-          echo "**** Pulling the yq docker image ****"
-          docker pull ghcr.io/linuxserver/yq
           for br in $(git for-each-ref --format='%(refname:short)' refs/remotes)
           do
             br=$(echo "$br" | sed 's|origin/||g')
             echo "**** Evaluating branch ${br} ****"
-            ls_branch=$(curl -sX GET https://raw.githubusercontent.com/linuxserver/docker-swag/${br}/jenkins-vars.yml \
+            ls_jenkins_vars=$(curl -sX GET https://raw.githubusercontent.com/linuxserver/docker-swag/${br}/jenkins-vars.yml)
-              | docker run --rm -i --entrypoint yq ghcr.io/linuxserver/yq -r .ls_branch)
+            ls_branch=$(echo "${ls_jenkins_vars}" | yq -r '.ls_branch')
-            if [ "$br" == "$ls_branch" ]; then
+            ls_trigger=$(echo "${ls_jenkins_vars}" | yq -r '.external_type')
-              echo "**** Branch ${br} appears to be live; checking workflow. ****"
+            if [[ "${br}" == "${ls_branch}" ]] && [[ "${ls_trigger}" != "os" ]]; then
+              echo "**** Branch ${br} appears to be live and trigger is not os; checking workflow. ****"
               if curl -sfX GET https://raw.githubusercontent.com/linuxserver/docker-swag/${br}/.github/workflows/external_trigger.yml > /dev/null 2>&1; then
                 echo "**** Workflow exists. Triggering external trigger workflow for branch ${br} ****."
+                echo "Triggering external trigger workflow for branch ${br}" >> $GITHUB_STEP_SUMMARY
                 curl -iX POST \
                   -H "Authorization: token ${{ secrets.CR_PAT }}" \
                   -H "Accept: application/vnd.github.v3+json" \
@@ -36,8 +36,10 @@ jobs:
                   https://api.github.com/repos/linuxserver/docker-swag/actions/workflows/external_trigger.yml/dispatches
               else
                 echo "**** Workflow doesn't exist; skipping trigger. ****"
+                echo "Skipping branch ${br} due to no external trigger workflow present." >> $GITHUB_STEP_SUMMARY
               fi
             else
-              echo "**** ${br} appears to be a dev branch; skipping trigger. ****"
+              echo "**** ${br} is either a dev branch, or has no external version; skipping trigger. ****"
+              echo "Skipping branch ${br} due to being detected as dev branch or having no external version." >> $GITHUB_STEP_SUMMARY
             fi
           done

.github/workflows/package_trigger.yml

@@ -14,13 +14,16 @@ jobs:
         run: |
           if [ -n "${{ secrets.PAUSE_PACKAGE_TRIGGER_SWAG_MASTER }}" ]; then
             echo "**** Github secret PAUSE_PACKAGE_TRIGGER_SWAG_MASTER is set; skipping trigger. ****"
+            echo "Github secret \`PAUSE_PACKAGE_TRIGGER_SWAG_MASTER\` is set; skipping trigger." >> $GITHUB_STEP_SUMMARY
             exit 0
           fi
           if [ $(curl -s https://ci.linuxserver.io/job/Docker-Pipeline-Builders/job/docker-swag/job/master/lastBuild/api/json | jq -r '.building') == "true" ]; then
             echo "**** There already seems to be an active build on Jenkins; skipping package trigger ****"
+            echo "There already seems to be an active build on Jenkins; skipping package trigger" >> $GITHUB_STEP_SUMMARY
             exit 0
           fi
           echo "**** Package trigger running off of master branch. To disable, set a Github secret named \"PAUSE_PACKAGE_TRIGGER_SWAG_MASTER\". ****"
+          echo "Package trigger running off of master branch. To disable, set a Github secret named \`PAUSE_PACKAGE_TRIGGER_SWAG_MASTER\`" >> $GITHUB_STEP_SUMMARY
           response=$(curl -iX POST \
             https://ci.linuxserver.io/job/Docker-Pipeline-Builders/job/docker-swag/job/master/buildWithParameters?PACKAGE_CHECK=true \
             --user ${{ secrets.JENKINS_USER }}:${{ secrets.JENKINS_TOKEN }} | grep -i location | sed "s|^[L|l]ocation: \(.*\)|\1|")
@@ -30,6 +33,7 @@ jobs:
           buildurl=$(curl -s "${response%$'\r'}api/json" | jq -r '.executable.url')
           buildurl="${buildurl%$'\r'}"
           echo "**** Jenkins job build url: ${buildurl} ****"
+          echo "Jenkins job build url: ${buildurl}" >> $GITHUB_STEP_SUMMARY
           echo "**** Attempting to change the Jenkins job description ****"
           curl -iX POST \
             "${buildurl}submitDescription" \

.github/workflows/package_trigger_scheduler.yml

@@ -17,18 +17,16 @@ jobs:
         run: |
           echo "**** Branches found: ****"
           git for-each-ref --format='%(refname:short)' refs/remotes
-          echo "**** Pulling the yq docker image ****"
-          docker pull ghcr.io/linuxserver/yq
           for br in $(git for-each-ref --format='%(refname:short)' refs/remotes)
           do
             br=$(echo "$br" | sed 's|origin/||g')
             echo "**** Evaluating branch ${br} ****"
-            ls_branch=$(curl -sX GET https://raw.githubusercontent.com/linuxserver/docker-swag/${br}/jenkins-vars.yml \
+            ls_branch=$(curl -sX GET https://raw.githubusercontent.com/linuxserver/docker-swag/${br}/jenkins-vars.yml | yq -r '.ls_branch')
-              | docker run --rm -i --entrypoint yq ghcr.io/linuxserver/yq -r .ls_branch)
             if [ "${br}" == "${ls_branch}" ]; then
               echo "**** Branch ${br} appears to be live; checking workflow. ****"
               if curl -sfX GET https://raw.githubusercontent.com/linuxserver/docker-swag/${br}/.github/workflows/package_trigger.yml > /dev/null 2>&1; then
                 echo "**** Workflow exists. Triggering package trigger workflow for branch ${br}. ****"
+                echo "Triggering package trigger workflow for branch ${br}" >> $GITHUB_STEP_SUMMARY
                 triggered_branches="${triggered_branches}${br} "
                 curl -iX POST \
                   -H "Authorization: token ${{ secrets.CR_PAT }}" \
@@ -38,9 +36,11 @@ jobs:
                 sleep 30
               else
                 echo "**** Workflow doesn't exist; skipping trigger. ****"
+                echo "Skipping branch ${br} due to no package trigger workflow present." >> $GITHUB_STEP_SUMMARY
               fi
             else
               echo "**** ${br} appears to be a dev branch; skipping trigger. ****"
+              echo "Skipping branch ${br} due to being detected as dev branch." >> $GITHUB_STEP_SUMMARY
             fi
           done
           echo "**** Package check build(s) triggered for branch(es): ${triggered_branches} ****"

Jenkinsfile

@@ -40,10 +40,11 @@ pipeline {
     // Setup all the basic environment variables needed for the build
     stage("Set ENV Variables base"){
       steps{
+        sh '''docker pull quay.io/skopeo/stable:v1 || : '''
         script{
           env.EXIT_STATUS = ''
           env.LS_RELEASE = sh(
-            script: '''docker run --rm ghcr.io/linuxserver/alexeiled-skopeo sh -c 'skopeo inspect docker://docker.io/'${DOCKERHUB_IMAGE}':latest 2>/dev/null' | jq -r '.Labels.build_version' | awk '{print $3}' | grep '\\-ls' || : ''',
+            script: '''docker run --rm quay.io/skopeo/stable:v1 inspect docker://ghcr.io/${LS_USER}/${CONTAINER_NAME}:latest 2>/dev/null | jq -r '.Labels.build_version' | awk '{print $3}' | grep '\\-ls' || : ''',
             returnStdout: true).trim()
           env.LS_RELEASE_NOTES = sh(
             script: '''cat readme-vars.yml | awk -F \\" '/date: "[0-9][0-9].[0-9][0-9].[0-9][0-9]:/ {print $4;exit;}' | sed -E ':a;N;$!ba;s/\\r{0,1}\\n/\\\\n/g' ''',
@@ -228,7 +229,7 @@ pipeline {
           script{
             env.SHELLCHECK_URL = 'https://ci-tests.linuxserver.io/' + env.IMAGE + '/' + env.META_TAG + '/shellcheck-result.xml'
           }
-          sh '''curl -sL https://raw.githubusercontent.com/linuxserver/docker-shellcheck/master/checkrun.sh | /bin/bash'''
+          sh '''curl -sL https://raw.githubusercontent.com/linuxserver/docker-jenkins-builder/master/checkrun.sh | /bin/bash'''
           sh '''#! /bin/bash
                 docker run --rm \
                   -v ${WORKSPACE}:/mnt \
@@ -376,6 +377,26 @@ pipeline {
         }
       }
     }
+    // If this is a master build check the S6 service file perms
+    stage("Check S6 Service file Permissions"){
+      when {
+        branch "master"
+        environment name: 'CHANGE_ID', value: ''
+        environment name: 'EXIT_STATUS', value: ''
+      }
+      steps {
+        script{
+          sh '''#! /bin/bash
+            WRONG_PERM=$(find ./  -path "./.git" -prune -o \\( -name "run" -o -name "finish" -o -name "check" \\) -not -perm -u=x,g=x,o=x -print)
+            if [[ -n "${WRONG_PERM}" ]]; then
+              echo "The following S6 service files are missing the executable bit; canceling the faulty build: ${WRONG_PERM}"
+              exit 1
+            else
+              echo "S6 service file perms look good."
+            fi '''
+        }
+      }
+    }
     /* #######################
            GitLab Mirroring
        ####################### */
@@ -668,6 +689,7 @@ pipeline {
         ]) {
           script{
             env.CI_URL = 'https://ci-tests.linuxserver.io/' + env.IMAGE + '/' + env.META_TAG + '/index.html'
+            env.CI_JSON_URL = 'https://ci-tests.linuxserver.io/' + env.IMAGE + '/' + env.META_TAG + '/report.json'
           }
           sh '''#! /bin/bash
                 set -e
@@ -694,8 +716,6 @@ pipeline {
                 -e WEB_SCREENSHOT=\"${CI_WEB}\" \
                 -e WEB_AUTH=\"${CI_AUTH}\" \
                 -e WEB_PATH=\"${CI_WEBPATH}\" \
-                -e DO_REGION="ams3" \
-                -e DO_BUCKET="lsio-ci" \
                 -t ghcr.io/linuxserver/ci:latest \
                 python3 test_build.py'''
         }
@@ -949,8 +969,67 @@ pipeline {
         environment name: 'EXIT_STATUS', value: ''
       }
       steps {
-        sh '''curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST https://api.github.com/repos/${LS_USER}/${LS_REPO}/issues/${PULL_REQUEST}/comments \
+        sh '''#! /bin/bash
-        -d '{"body": "I am a bot, here are the test results for this PR: \\n'${CI_URL}' \\n'${SHELLCHECK_URL}'"}' '''
+            # Function to retrieve JSON data from URL
+            get_json() {
+              local url="$1"
+              local response=$(curl -s "$url")
+              if [ $? -ne 0 ]; then
+                echo "Failed to retrieve JSON data from $url"
+                return 1
+              fi
+              local json=$(echo "$response" | jq .)
+              if [ $? -ne 0 ]; then
+                echo "Failed to parse JSON data from $url"
+                return 1
+              fi
+              echo "$json"
+            }
+
+            build_table() {
+              local data="$1"
+
+              # Get the keys in the JSON data
+              local keys=$(echo "$data" | jq -r 'to_entries | map(.key) | .[]')
+
+              # Check if keys are empty
+              if [ -z "$keys" ]; then
+                echo "JSON report data does not contain any keys or the report does not exist."
+                return 1
+              fi
+
+              # Build table header
+              local header="| Tag | Passed |\\n| --- | --- |\\n"
+
+              # Loop through the JSON data to build the table rows
+              local rows=""
+              for build in $keys; do
+                local status=$(echo "$data" | jq -r ".[\\"$build\\"].test_success")
+                if [ "$status" = "true" ]; then
+                  status="✅"
+                else
+                  status="❌"
+                fi
+                local row="| "$build" | "$status" |\\n"
+                rows="${rows}${row}"
+              done
+
+              local table="${header}${rows}"
+              local escaped_table=$(echo "$table" | sed 's/\"/\\\\"/g')
+              echo "$escaped_table"
+            }
+
+            # Retrieve JSON data from URL
+            data=$(get_json "$CI_JSON_URL")
+            # Create table from JSON data
+            table=$(build_table "$data")
+            echo -e "$table"
+
+            curl -X POST -H "Authorization: token $GITHUB_TOKEN" \
+              -H "Accept: application/vnd.github.v3+json" \
+              "https://api.github.com/repos/$LS_USER/$LS_REPO/issues/$PULL_REQUEST/comments" \
+              -d "{\\"body\\": \\"I am a bot, here are the test results for this PR: \\n${CI_URL}\\n${SHELLCHECK_URL}\\n${table}\\"}"'''
+
       }
     }
   }

package_versions.txt

@@ -21,8 +21,8 @@ azure-mgmt-core                 1.4.0                  python
 azure-mgmt-dns                  8.0.0                  python  
 bash                            5.2.15-r0              apk     
 beautifulsoup4                  4.12.2                 python  
-boto3                           1.26.118               python  
+boto3                           1.26.129               python  
-botocore                        1.29.118               python  
+botocore                        1.29.129               python  
 brotli-libs                     1.0.9-r9               apk     
 bs4                             0.0.1                  python  
 busybox                         1.35.0                 binary  
@@ -93,8 +93,8 @@ fontconfig                      2.14.1-r0              apk
 freetype                        2.12.1-r0              apk     
 future                          0.18.3                 python  
 gdbm                            1.23-r0                apk     
-git                             2.38.4-r1              apk     
+git                             2.38.5-r0              apk     
-git-perl                        2.38.4-r1              apk     
+git-perl                        2.38.5-r0              apk     
 gmp                             6.2.1-r2               apk     
 gnupg                           2.2.40-r0              apk     
 gnupg-dirmngr                   2.2.40-r0              apk     
@@ -116,7 +116,7 @@ httplib2                        0.22.0                 python
 icu-data-en                     72.1-r1                apk     
 icu-libs                        72.1-r1                apk     
 idna                            3.4                    python  
-importlib-metadata              6.5.1                  python  
+importlib-metadata              6.6.0                  python  
 ip6tables                       1.8.8-r2               apk     
 iptables                        1.8.8-r2               apk     
 isodate                         0.6.1                  python  
@@ -232,7 +232,7 @@ pcre                            8.45-r2                apk
 pcre2                           10.42-r0               apk     
 perl                            5.36.0-r1              apk     
 perl-error                      0.17029-r1             apk     
-perl-git                        2.38.4-r1              apk     
+perl-git                        2.38.5-r0              apk     
 php-cli                         8.1.18                 binary  
 php-fpm                         8.1.18                 binary  
 php81                           8.1.18-r0              apk     
@@ -269,7 +269,7 @@ php81-pecl-mailparse            3.1.4-r0               apk
 php81-pecl-mcrypt               1.0.6-r0               apk     
 php81-pecl-memcached            3.2.0-r0               apk     
 php81-pecl-redis                5.3.7-r0               apk     
-php81-pecl-xmlrpc               1.0.0_rc3-r0           apk     
+php81-pecl-xmlrpc               1.0.0_rc3-r1           apk     
 php81-pgsql                     8.1.18-r0              apk     
 php81-phar                      8.1.18-r0              apk     
 php81-posix                     8.1.18-r0              apk     
@@ -286,12 +286,12 @@ php81-xmlwriter                 8.1.18-r0              apk
 php81-xsl                       8.1.18-r0              apk     
 php81-zip                       8.1.18-r0              apk     
 pinentry                        1.2.1-r0               apk     
-pip                             23.1                   python  
+pip                             23.1.2                 python  
 pkb-client                      1.2                    python  
 popt                            1.19-r0                apk     
 portalocker                     2.7.0                  python  
 procps                          3.3.17-r2              apk     
-protobuf                        4.22.3                 python  
+protobuf                        4.22.4                 python  
 publicsuffixlist                0.9.4                  python  
 pyOpenSSL                       23.1.1                 python  
 pyRFC3339                       1.1                    python  
@@ -307,12 +307,12 @@ python-transip                  0.6.0                  python
 python3                         3.10.11-r0             apk     
 pytz                            2023.3                 python  
 readline                        8.2.0-r0               apk     
-requests                        2.28.2                 python  
+requests                        2.30.0                 python  
 requests-file                   1.5.1                  python  
 requests-mock                   1.10.0                 python  
 requests-oauthlib               1.3.1                  python  
 rsa                             4.9                    python  
-s3transfer                      0.6.0                  python  
+s3transfer                      0.6.1                  python  
 scanelf                         1.3.5-r1               apk     
 setuptools                      65.5.0                 python  
 shadow                          4.13-r0                apk     
@@ -322,7 +322,7 @@ soupsieve                       2.4.1                  python
 sqlite-libs                     3.40.1-r0              apk     
 ssl_client                      1.35.0-r29             apk     
 tiff                            4.4.0-r3               apk     
-tldextract                      3.4.0                  python  
+tldextract                      3.4.1                  python  
 typing-inspect                  0.8.0                  python  
 typing_extensions               4.5.0                  python  
 tzdata                          2023c-r0               apk