.gitlab-ci.yml

 include:
-  local: /gitlab-ci-template.yml
+  - local: /gitlab-ci-template.yml
+  - local: /resources/shell-tools/.gitlab-ci.yml
 
 stages:
+  - shell-tools-lint
+  - shell-tools-test
   - build
   - test
   - tag
 
 variables:
-  CI_REGISTRY_IMAGE_VERSION: "1.3.0"
-  CI_REGISTRY_BUILD_ARG: "--build-arg SI_TOOLS_VERSION=0.3.1"
-
+  CI_REGISTRY_IMAGE_VERSION: "2.0.0"
+  CI_CONTAINER_BUILD_ARCHS: "amd64:arm64"
 
 container-test:
   stage: test
   inherit:
     default: false
-    variables:
-      - CI_REGISTRY
-      - CI_REGISTRY_USER
-      - CI_REGISTRY_PASSWORD
-      - CI_REGISTRY_IMAGE
-      - CI_REGISTRY_BUILD_ARGS
-  image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+    variables: true
+  image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA-amd64
   before_script:
     - podman login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
     - podman build --pull $CI_REGISTRY_BUILD_ARGS -t "build-image:test" .
-    - si-tagging -l "build-image:test" "example.com/tagging-image" "0.1.2.3"
-    - podman images --format "{{.Repository}}:{{.Tag}}" | grep -Pe "^example.com/tagging-image:0.1.2$"
-    - podman images --format "{{.Repository}}:{{.Tag}}" | grep "latest"
-    - podman images --format "{{.Repository}}:{{.Tag}}" | grep -Pe "^example.com/tagging-image:0$"
-
 
 container-tagging:
   stage: tag

Dockerfile

-FROM docker.io/library/fedora:32
+FROM docker.io/library/fedora:34
 
 COPY resources/storage.conf /etc/containers/
 
-ARG SI_TOOLS_VERSION=master
+COPY resources/shell-tools/ /shell-tools
 
 RUN true\
    && dnf -y upgrade \
    && dnf -y install podman buildah findutils git \
    && dnf clean all \
    && sed -e 's/.* cgroup_manager =.*/cgroup_manager = "cgroupfs"/' /usr/share/containers/containers.conf > /etc/containers/containers.conf \
-   && git clone -b "$SI_TOOLS_VERSION" --depth=3 https://git.shivering-isles.com/shivering-isles/shell-tools.git ./shell-tools \
-   && ./shell-tools/install.sh \
-   && rm -rf ./shell-tools \
+   && /shell-tools/install.sh \
    && true
 
 CMD ["/bin/bash"]

gitlab-ci-template.yml

@@ -4,45 +4,72 @@ stages:
 
 variables:
   CI_REGISTRY_BUILD_ARGS: ""
+  CI_CONTAINER_BUILD_ARCHS: "amd64"
 
-container-build:
+.container-build:
   stage: build
+  variables:
+    CI_REGISTRY_CONTAINER_FORMAT: "docker"
   inherit:
     default: false
-    variables:
-      - CI_REGISTRY
-      - CI_REGISTRY_USER
-      - CI_REGISTRY_PASSWORD
-      - CI_REGISTRY_IMAGE
-      - CI_REGISTRY_BUILD_DOCKERFILE
-      - CI_REGISTRY_BUILD_ARGS
-  image: quay.io/sheogorath/build-ah-engine:1.3.0
+    variables: true
+  image: quay.io/sheogorath/build-ah-engine:2.0.0
   before_script:
     - podman login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
-    - export BUILD_DATE=$(date --rfc-3339 ns)
-    - export VCS_REF=$CI_COMMIT_SHA
-    - export VCS_URL=$CI_PROJECT_URL
+    - export VCS_REF="$CI_COMMIT_SHA"
+    - export VCS_URL="$CI_PROJECT_URL"
+    - export BUILD_DATE="$(date --rfc-3339 ns)"
   script:
-    - si-fix "$(CI_REGISTRY_BUILD_DOCKERFILE:-./Dockerfile)"
-    - podman build --pull $CI_REGISTRY_BUILD_ARGS -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA" .
-    - podman push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA"
+    - si-fix "${CI_REGISTRY_BUILD_DOCKERFILE:-./Dockerfile}"
+    - podman build --pull
+      --label "org.opencontainers.image.source=$CI_PROJECT_URL"
+      --label "org.opencontainers.image.revision=$CI_COMMIT_SHA"
+      --label "org.opencontainers.image.created=$(date --rfc-3339 ns)"
+      --label "org.opencontainers.image.title=$CI_PROJECT_TITLE"
+      $CI_REGISTRY_BUILD_ARGS
+      -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA-$CI_REGISTRY_IMAGE_ARCH"
+      --format docker
+      .
+    - podman push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA-$CI_REGISTRY_IMAGE_ARCH"
+
+container-build-amd64:
+  extends: .container-build
+  variables:
+    CI_REGISTRY_IMAGE_ARCH: amd64
+  tags:
+    - amd64
+  rules:
+    - if: '$CI_CONTAINER_BUILD_ARCHS =~ /:?amd64:?/ && $CI_PIPELINE_SOURCE != "merge_request_event"'
+
+container-build-arm64:
+  extends: .container-build
+  variables:
+    CI_REGISTRY_IMAGE_ARCH: arm64
+  tags:
+    - arm64
+  rules:
+    - if: '$CI_CONTAINER_BUILD_ARCHS =~ /:?arm64:?/ && $CI_PIPELINE_SOURCE != "merge_request_event"'
+
+container-build-aarch64:
+  extends: .container-build
+  variables:
+    CI_REGISTRY_IMAGE_ARCH: aarch64
+  tags:
+    - aaarch64
+  rules:
+    - if: '$CI_CONTAINER_BUILD_ARCHS =~ /:?aarch64:?/ && $CI_PIPELINE_SOURCE != "merge_request_event"'
 
 container-tagging:
   stage: tag
   inherit:
     default: false
-    variables:
-      - CI_REGISTRY
-      - CI_REGISTRY_USER
-      - CI_REGISTRY_PASSWORD
-      - CI_REGISTRY_IMAGE
-      - CI_REGISTRY_IMAGE_VERSION
-  image: quay.io/sheogorath/build-ah-engine:1.3.0
+    variables: true
+  image: quay.io/sheogorath/build-ah-engine:2.0.0
   before_script:
     - podman login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
     - export BUILD_DATE="$(date --rfc-3339 ns)"
   script:
-    - podman pull "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA"
+    - si-pull "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA"
     - si-tagging -l "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA" "$CI_REGISTRY_IMAGE" "${CI_REGISTRY_IMAGE_VERSION}"
     - si-push "$CI_REGISTRY_IMAGE"
   resource_group: latest

resources/shell-tools/.gitlab-ci.yml

+stages:
+  - shell-tools-lint
+  - shell-tools-test
+
+
+shell-tools-shellcheck:
+  image: docker.io/library/fedora:36
+  stage: shell-tools-lint
+  before_script:
+    - dnf install -y ShellCheck
+  script:
+    - shellcheck ./resources/shell-tools/**/*.sh
+
+shell-tools-test:
+  image: $CI_REGISTRY_IMAGE:latest
+  stage: shell-tools-test
+  script:
+    - ./resources/shell-tools/test/latest.sh
+    - ./resources/shell-tools/test/fix-dockerfile-pinning.sh
+    - ./resources/shell-tools/test/multiarch.sh

resources/shell-tools/bin/fix-dockerfile.sh

+#!/bin/bash
+
+set -o pipefail
+set -u
+set -e
+
+VERSION=0.1.0
+
+printVersion() {
+    echo "$VERSION"
+}
+
+printUsage() {
+    printVersion
+    echo "
+    Shivering-Isles fix Dockerfile tool
+
+    This tool fix the Dockerfile for podman to be able to handle pinned tags properly
+
+    Usage of $0:
+        $0 [<DOCKERFILE>]
+
+    Example:
+        $0 ./Dockerfile
+    "
+    exit 1
+}
+
+CONTAINER_CMD=podman
+
+
+if ! command -v "$CONTAINER_CMD" >/dev/null 2>&1; then
+    CONTAINER_CMD=docker
+fi
+
+for i in "$@"
+do
+case $i in
+    --container-cmd=*)
+    CONTAINER_CMD="${i#*=}"
+    shift # past argument with no value
+    ;;
+    -v|--version)
+    printVersion
+    exit 0
+    shift # past argument with no value
+    ;;
+    -h|--help)
+    printUsage
+    shift
+    ;;
+    *)
+          # further/unknown options
+    ;;
+esac
+done
+
+if [ "$CONTAINER_CMD" != "podman" ]; then
+  exit 0
+fi
+
+DOCKERFILE="${1:-./Dockerfile}"
+
+sed -Ei 's/^(FROM [^/]+(\/[^/:]+)+)(:[^@]+)@/\1@/g' "$DOCKERFILE"

resources/shell-tools/bin/pull.sh

+#!/bin/bash
+
+set -o pipefail
+set -u
+set -e
+
+VERSION=0.1.0
+
+printVersion() {
+    echo "$VERSION"
+}
+
+printUsage() {
+    printVersion
+    echo "
+    Shivering-Isles pull tool
+
+    This tool will pull all available base container images from upstream
+
+    Usage of $0:
+        $0 <IMAGE REFERENCE>
+
+    Example:
+        $0 registry.example.com/example/app
+    "
+    exit 1
+}
+
+CONTAINER_CMD=podman
+
+if ! command -v "$CONTAINER_CMD" >/dev/null 2>&1; then
+    CONTAINER_CMD=docker
+fi
+
+if [ "$1" = "--help" ]; then
+    printUsage
+fi
+
+for i in "$@"
+do
+case $i in
+    --container-cmd=*)
+    CONTAINER_CMD="${i#*=}"
+    shift # past argument with no value
+    ;;
+    -v|--version)
+    printVersion
+    exit 0
+    shift # past argument with no value
+    ;;
+    -h|--help)
+    printUsage
+    shift
+    ;;
+    *)
+          # further/unknown options
+    ;;
+esac
+done
+
+CONTAINER_IMAGE_NAME=${1:-invalid}
+
+if [ "$CONTAINER_IMAGE_NAME" = "invalid" ]; then
+    echo "Error: Invalid image name" >&2
+    printUsage
+fi
+
+IMAGE_PULL_SUCCESS=0
+
+# shellcheck disable=SC2015
+podman pull "$CONTAINER_IMAGE_NAME" && \
+  IMAGE_PULL_SUCCESS=1 || \
+  true
+
+# shellcheck disable=SC2015
+podman pull "$CONTAINER_IMAGE_NAME-amd64" && \
+  IMAGE_PULL_SUCCESS=1 || \
+  true
+
+# shellcheck disable=SC2015
+podman pull "$CONTAINER_IMAGE_NAME-arm64" || \
+  podman pull "$CONTAINER_IMAGE_NAME-aarch64" && \
+  IMAGE_PULL_SUCCESS=1 || \
+  true
+
+if [ "$IMAGE_PULL_SUCCESS" == "0" ]; then
+  echo "Error: No images pulled" >&2
+  exit 1
+fi

resources/shell-tools/bin/push.sh

+#!/bin/bash
+
+set -o pipefail
+set -u
+set -e
+
+VERSION=0.1.0
+
+printVersion() {
+    echo "$VERSION"
+}
+
+printUsage() {
+    printVersion
+    echo "
+    Shivering-Isles push tool
+
+    This tool will push all tagged versions of a container image upstream
+
+    Usage of $0:
+        $0 <IMAGE REFERENCE>
+
+    Example:
+        $0 registry.example.com/example/app
+    "
+    exit 1
+}
+
+pushImageOrManifest() {
+    if podman manifest inspect "$1"; then
+        $CONTAINER_CMD manifest push --all --format v2s2 "$1" "docker://$1"
+    else
+        $CONTAINER_CMD push "$1"
+    fi
+}
+
+CONTAINER_CMD=podman
+
+if ! command -v "$CONTAINER_CMD" >/dev/null 2>&1; then
+    CONTAINER_CMD=docker
+fi
+
+if [ "$1" = "--help" ]; then
+    printUsage
+fi
+
+for i in "$@"
+do
+case $i in
+    --container-cmd=*)
+    CONTAINER_CMD="${i#*=}"
+    shift # past argument with no value
+    ;;
+    -v|--version)
+    printVersion
+    exit 0
+    shift # past argument with no value
+    ;;
+    -h|--help)
+    printUsage
+    shift
+    ;;
+    *)
+          # further/unknown options
+    ;;
+esac
+done
+
+CONTAINER_IMAGE_NAME=${1:-invalid}
+
+if [ "$CONTAINER_IMAGE_NAME" = "invalid" ]; then
+    echo "Error: Invalid image name" >&2
+    printUsage
+fi
+
+# shellcheck disable=SC2086
+$CONTAINER_CMD images --format "{{.Repository}}:{{.Tag}}" "$CONTAINER_IMAGE_NAME" | grep "$CONTAINER_IMAGE_NAME" | xargs -L 1 pushImageOrManifest

resources/shell-tools/bin/tagging.sh

+#!/bin/bash
+
+set -o pipefail
+set -u
+set -e
+
+VERSION=0.1.0
+
+printVersion() {
+    echo "$VERSION"
+}
+
+printUsage() {
+    printVersion
+    echo "
+    Shivering-Isles tagging tool
+
+    Usage of $0:
+        $0 <IMAGE REFERENCE> <PREFIX> <VERSION> <SUFFIX>
+
+    Example:
+        $0 myimage:test myApp 2.0.24 alpine
+    "
+    exit 1
+}
+
+createTagOrManifest() {
+    TARGET_IMAGE_REFERENCE=$1
+    if [ "$MULTIARCH_IMAGES_COUNT" -gt "0" ]; then
+        $CONTAINER_CMD manifest create "$TARGET_IMAGE_REFERENCE"
+        for IMAGE_REFERENCE in "${MULTIARCH_IMAGES[@]}"
+        do
+            MANIFEST_OPTIONS=()
+            if [[ "$IMAGE_REFERENCE" =~ "aarch64"|"arm64" ]]; then
+                MANIFEST_OPTIONS+=(--variant v8)
+            fi
+            if [ "$LOCAL" = "1" ]; then
+                $CONTAINER_CMD manifest add "${MANIFEST_OPTIONS[@]}" "$TARGET_IMAGE_REFERENCE" containers-storage:"$IMAGE_REFERENCE"
+            else
+                $CONTAINER_CMD manifest add "${MANIFEST_OPTIONS[@]}" "$TARGET_IMAGE_REFERENCE" "$IMAGE_REFERENCE"
+            fi
+        done
+    else
+        $CONTAINER_CMD tag "$CURRENT_IMAGE_REFERENCE" "$TARGET_IMAGE_REFERENCE"
+    fi
+}
+
+CONTAINER_CMD="podman"
+PREFIX=""
+SUFFIX=""
+LATEST=0
+LOCAL=0
+MULTIARCH_SUPPORTED="amd64|arm64|aarch64"
+
+if ! command -v "$CONTAINER_CMD" >/dev/null 2>&1; then
+    CONTAINER_CMD=docker
+fi
+
+if [ "$1" = "--help" ]; then
+    printUsage
+fi
+
+for i in "$@"
+do
+case $i in
+    -p=*|--prefix=*)
+    PREFIX="${i#*=}"
+    shift
+    ;;
+    -s=*|--suffix=*)
+    SUFFIX="${i#*=}"
+    shift # past argument=value
+    ;;
+    -l|--latest)
+    LATEST=1
+    shift # past argument=value
+    ;;
+    --local)
+    LOCAL=1
+    shift # past argument=value
+    ;;
+    --container-cmd=*)
+    CONTAINER_CMD="${i#*=}"
+    shift # past argument with no value
+    ;;
+    -v|--version)
+    printVersion
+    exit 0
+    shift # past argument with no value
+    ;;
+    -h|--help)
+    printUsage
+    shift
+    ;;
+    *)
+          # further/unknown options
+    ;;
+esac
+done
+
+CURRENT_IMAGE_REFERENCE=${1:-invalid}
+TARGET_IMAGE_NAME=${2:-invalid}
+TARGET_IMAGE_VERSION=${3:-invalid}
+
+if [ "$CURRENT_IMAGE_REFERENCE" = "invalid" ] || [ "$TARGET_IMAGE_NAME" = "invalid" ] || [ "$TARGET_IMAGE_VERSION" = "invalid" ]; then
+    echo "Error: Invalid image name or version" >&2
+    printUsage
+fi
+
+mapfile -t MULTIARCH_IMAGES < <($CONTAINER_CMD images | awk '{print $1 ":" $2}' | grep -Pe "$CURRENT_IMAGE_REFERENCE-($MULTIARCH_SUPPORTED)$")
+MULTIARCH_IMAGES_COUNT=${#MULTIARCH_IMAGES[@]}
+
+counter=1
+new_version="$(echo "$TARGET_IMAGE_VERSION" | cut -d. -f$counter)"
+last_version=""
+while [ "$last_version" != "$new_version" ]; do
+    createTagOrManifest "${TARGET_IMAGE_NAME}:${PREFIX}${new_version}${SUFFIX}"
+    last_version="$new_version"
+    ((counter++))
+    new_version="$(echo "$TARGET_IMAGE_VERSION" | cut -d. -f-"$counter")"
+done
+
+if [ "$LATEST" = "1" ]; then
+    createTagOrManifest "${TARGET_IMAGE_NAME}:latest"
+fi

resources/shell-tools/install.sh

+#!/bin/bash
+
+BASENAME="$(dirname "$0")"
+
+cp "$BASENAME"/./bin/tagging.sh /usr/local/bin/si-tagging
+cp "$BASENAME"/./bin/push.sh /usr/local/bin/si-push
+cp "$BASENAME"/./bin/pull.sh /usr/local/bin/si-pull
+cp "$BASENAME"/./bin/fix-dockerfile.sh /usr/local/bin/si-fix
+chown root:root /usr/local/bin/si-tagging
+chown root:root /usr/local/bin/si-push
+chown root:root /usr/local/bin/si-pull
+chown root:root /usr/local/bin/si-fix
+chmod 0755 /usr/local/bin/si-tagging
+chmod 0755 /usr/local/bin/si-push
+chmod 0755 /usr/local/bin/si-pull
+chmod 0755 /usr/local/bin/si-fix

resources/shell-tools/test/fix-dockerfile-pinning.sh

+#!/bin/bash
+
+set -o pipefail
+set -u
+
+BASEDIR=$(dirname "$0")
+
+TEST_DOCKERFILE="$(mktemp)"
+
+cat >"$TEST_DOCKERFILE" <<EOF
+FROM docker.io/library/haproxy:1.9.15-alpine@sha256:1da27f6d19b52d85c3c31412f2e387be54b4cddba938d68ec6b24775bd89d75c
+FROM docker.io/library/haproxy:1.9.15-alpine
+EOF
+
+"$BASEDIR"/../bin/fix-dockerfile.sh "$TEST_DOCKERFILE"
+
+TEST_VERIFY_DOCKERFILE="$(mktemp)"
+cat >"$TEST_VERIFY_DOCKERFILE" <<EOF
+FROM docker.io/library/haproxy@sha256:1da27f6d19b52d85c3c31412f2e387be54b4cddba938d68ec6b24775bd89d75c
+FROM docker.io/library/haproxy:1.9.15-alpine
+EOF
+
+diff "$TEST_DOCKERFILE" "$TEST_VERIFY_DOCKERFILE"
+
+EXIT_CODE=$?
+
+rm -f "$TEST_DOCKERFILE" "$TEST_VERIFY_DOCKERFILE"
+
+exit $EXIT_CODE

resources/shell-tools/test/latest.sh

+#!/bin/bash
+
+set -o pipefail
+set -u
+
+BASEDIR=$(dirname "$0")
+
+podman build -t mytest-latest:test -f- <<EOF
+FROM scratch
+EOF
+
+TEST_OUTPUT="$(mktemp)"
+TEST_VERIFY_OUTPUT="$(mktemp)"
+
+"$BASEDIR"/../bin/tagging.sh -l mytest-latest:test registry.example.com/latest-test 1.2.3
+
+podman images --format "{{.Repository}}:{{.Tag}}" registry.example.com/latest-test | grep registry.example.com/latest-test > "$TEST_OUTPUT" 2>&1
+podman images --format "{{.Repository}}:{{.Tag}}"  registry.example.com/latest-test | grep registry.example.com/latest-test
+
+cat >"$TEST_VERIFY_OUTPUT" <<EOF
+registry.example.com/latest-test:1
+registry.example.com/latest-test:1.2
+registry.example.com/latest-test:1.2.3
+registry.example.com/latest-test:latest
+EOF
+
+diff "$TEST_OUTPUT" "$TEST_VERIFY_OUTPUT"
+
+EXIT_CODE=$?
+
+rm -f "$TEST_OUTPUT" "$TEST_VERIFY_OUTPUT"
+podman rmi -f "$(podman images -q mytest-latest:test)"
+
+
+exit $EXIT_CODE

resources/shell-tools/test/multiarch.sh

+#!/bin/bash
+
+set -o pipefail
+set -u
+
+BASEDIR=$(dirname "$0")
+
+podman build --format docker -t mytest-latest:test-amd64 -f- <<EOF
+FROM scratch
+EOF
+podman build --format docker -t mytest-latest:test-arm64 -f- <<EOF
+FROM scratch
+EOF
+
+TEST_OUTPUT="$(mktemp)"
+TEST_VERIFY_OUTPUT="$(mktemp)"
+
+"$BASEDIR"/../bin/tagging.sh --local -l mytest-latest:test registry.example.com/latest-test 1.2.3
+
+ERRORS=0
+podman manifest inspect registry.example.com/latest-test:1
+ERRORS=$((ERRORS + "$?"))
+podman manifest inspect registry.example.com/latest-test:1.2
+ERRORS=$((ERRORS + "$?"))
+podman manifest inspect registry.example.com/latest-test:1.2.3
+ERRORS=$((ERRORS + "$?"))
+podman manifest inspect registry.example.com/latest-test:latest
+ERRORS=$((ERRORS + "$?"))
+
+if [ "$ERRORS" = "0" ]; then
+  EXIT_CODE=0
+else
+  EXIT_CODE=1
+fi
+
+rm -f "$TEST_OUTPUT" "$TEST_VERIFY_OUTPUT"
+podman rmi -f registry.example.com/latest-test:1 \
+  registry.example.com/latest-test:1.2 \
+  registry.example.com/latest-test:1.2.3 \
+  registry.example.com/latest-test:latest
+podman rmi -f "$(podman images -q mytest-latest:test-amd64)"
+podman rmi -f "$(podman images -q mytest-latest:test-arm64)"
+
+exit $EXIT_CODE