diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 90535ac95f3459897af93e5891147bdfbe1c1e10..cc3e9a0fd54c6ee1178971b53fde84f83892304e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ include: - - local: /gitlab-ci-template.yml + - local: /gitlab-ci-template-manifest.yml - local: /resources/shell-tools/.gitlab-ci.yml stages: @@ -10,30 +10,28 @@ stages: - tag variables: - CI_REGISTRY_IMAGE_VERSION: "1.3.0" - CI_REGISTRY_BUILD_ARG: "--build-arg SI_TOOLS_VERSION=0.3.1" + CI_REGISTRY_IMAGE_VERSION: "2.0.0" +container-build-x86_64: + extends: .container-build + tags: + - x86_64 + +container-build-aarch64: + extends: .container-build + tags: + - aarch64 container-test: stage: test inherit: default: false - variables: - - CI_REGISTRY - - CI_REGISTRY_USER - - CI_REGISTRY_PASSWORD - - CI_REGISTRY_IMAGE - - CI_REGISTRY_BUILD_ARGS - image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA + variables: true + image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA-x86_64 before_script: - - podman login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - podman login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" script: - podman build --pull $CI_REGISTRY_BUILD_ARGS -t "build-image:test" . - - si-tagging -l "build-image:test" "example.com/tagging-image" "0.1.2.3" - - podman images --format "{{.Repository}}:{{.Tag}}" | grep -Pe "^example.com/tagging-image:0.1.2$" - - podman images --format "{{.Repository}}:{{.Tag}}" | grep "latest" - - podman images --format "{{.Repository}}:{{.Tag}}" | grep -Pe "^example.com/tagging-image:0$" - -container-tagging: +container-manifest: stage: tag diff --git a/gitlab-ci-template-manifest.yml b/gitlab-ci-template-manifest.yml new file mode 100644 index 0000000000000000000000000000000000000000..a11101899ee54cbecbea1670abc0b602f0c49b81 --- /dev/null +++ b/gitlab-ci-template-manifest.yml @@ -0,0 +1,53 @@ +stages: + - build + - tag + +variables: + CI_REGISTRY_BUILD_ARGS: "" + +.container-build: + stage: build + inherit: + default: false + variables: true + image: quay.io/sheogorath/build-ah-engine:2.0.0 + artifacts: + paths: + - logs/ + expire_in: 1 week + before_script: + - podman login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - export VCS_REF="$CI_COMMIT_SHA" + - export VCS_URL="$CI_PROJECT_URL" + - export BUILD_DATE="$(date --rfc-3339 ns)" + script: + - si-fix "${CI_REGISTRY_BUILD_DOCKERFILE:-./Dockerfile}" + - podman build --pull + --label "org.opencontainers.image.source=$CI_PROJECT_URL" + --label "org.opencontainers.image.revision=$CI_COMMIT_SHA" + --label "org.opencontainers.image.created=$(date --rfc-3339 ns)" + --label "org.opencontainers.image.title=$CI_PROJECT_TITLE" + $CI_REGISTRY_BUILD_ARGS + -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA-$(uname -i)" + --format docker + . + - podman push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA-$(uname -i)" + - mkdir logs/ + - echo "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA-$(uname -i)" > "logs/build-$(uname -i).log" + +container-manifest: + stage: tag + inherit: + default: false + variables: true + image: quay.io/sheogorath/build-ah-engine:2.0.0 + before_script: + - podman login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" + - export BUILD_DATE="$(date --rfc-3339 ns)" + script: + - IMAGE_REFERENCES=() + - for log in logs/*.log; do IMAGE_REFERENCES+=("$(cat "$log")"); done + - si-manifest --push -l "$CI_REGISTRY_IMAGE" "${CI_REGISTRY_IMAGE_VERSION}" "${IMAGE_REFERENCES[@]}" + resource_group: latest + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/resources/shell-tools/.gitlab-ci.yml b/resources/shell-tools/.gitlab-ci.yml index e06e7ffe30df70acdad7def51f6919da11cf55ee..1212afcf9189121c08ec5fe86162036744c979c5 100644 --- a/resources/shell-tools/.gitlab-ci.yml +++ b/resources/shell-tools/.gitlab-ci.yml @@ -17,3 +17,7 @@ shell-tools-test: script: - ./resources/shell-tools/test/latest.sh - ./resources/shell-tools/test/fix-dockerfile-pinning.sh + - ./resources/shell-tools/test/manifest.sh + - ./resources/shell-tools/test/manifest-latest.sh + - ./resources/shell-tools/test/manifest-print.sh + - ./resources/shell-tools/test/manifest-push.sh diff --git a/resources/shell-tools/bin/manifest-push.sh b/resources/shell-tools/bin/manifest-push.sh new file mode 100755 index 0000000000000000000000000000000000000000..def9257761918aec18134235572f61f32954f2c8 --- /dev/null +++ b/resources/shell-tools/bin/manifest-push.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -o pipefail +set -u +set -e + +printUsage() { + echo " + Shivering-Isles push tool + + This tool will push all tagged versions of a container image upstream + + Usage of $0: + $0 <IMAGE REFERENCE> + + Example: + $0 registry.example.com/example/app:1 registry.example.com/example/app:1.2 + " + exit 1 +} + +if [ "$1" = "--help" ]; then + printUsage +fi + +DRY=0 + +for i in "$@" +do +case $i in + --dry) + DRY=1 + shift + ;; + -h|--help) + printUsage + shift + ;; + *) + # further/unknown options + ;; +esac +done + +TAGS_TO_PUSH=("$@") + +if [ "$DRY" = "1" ]; then + echo "${TAGS_TO_PUSH[@]}" +else + for IMAGE_REFERENCE in "${TAGS_TO_PUSH[@]}" + do + podman manifest push --all --format v2s2 "$IMAGE_REFERENCE" "docker://$IMAGE_REFERENCE" + done +fi diff --git a/resources/shell-tools/bin/manifest.sh b/resources/shell-tools/bin/manifest.sh new file mode 100755 index 0000000000000000000000000000000000000000..2c9f3e5ab066d8f9783e193fad0fd49e9be68cbb --- /dev/null +++ b/resources/shell-tools/bin/manifest.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +set -o pipefail +set -u +set -e + +printUsage() { + echo " + Shivering-Isles manifest tool + + Usage of $0: + $0 [PARAMS...] <NAME> <VERSION> <IMAGE REFERENCE> [<IMAGE REFERENCE>...] + + Example: + $0 myApp 2.0.24 myimage:test + $0 --suffix=alpine myApp 2.0.24 myimage:test + " + exit 1 +} + +createManifest() { + TARGET_IMAGE_REFERENCE=$1 + podman manifest create "$TARGET_IMAGE_REFERENCE" >/dev/null + MANIFESTS_TO_PUSH+=("$TARGET_IMAGE_REFERENCE") + for IMAGE_REFERENCE in "${IMAGE_REFERENCES[@]}" + do + MANIFEST_OPTIONS=() + if [[ "$IMAGE_REFERENCE" =~ "aarch64"|"arm64" ]]; then + MANIFEST_OPTIONS+=(--variant v8) + fi + if [ "$LOCAL" = "1" ]; then + podman manifest add "${MANIFEST_OPTIONS[@]}" "$TARGET_IMAGE_REFERENCE" containers-storage:localhost/"$IMAGE_REFERENCE" >/dev/null + else + podman manifest add "${MANIFEST_OPTIONS[@]}" "$TARGET_IMAGE_REFERENCE" "docker://$IMAGE_REFERENCE" >/dev/null + fi + done +} + +PREFIX="" +SUFFIX="" +LATEST=0 +LOCAL=0 +PRINT=0 +PUSH=0 + +if [ "$1" = "--help" ]; then + printUsage +fi + +for i in "$@" +do +case $i in + -p=*|--prefix=*) + PREFIX="${i#*=}" + shift + ;; + -s=*|--suffix=*) + SUFFIX="${i#*=}" + shift # past argument=value + ;; + -l|--latest) + LATEST=1 + shift # past argument=value + ;; + --local) + LOCAL=1 + shift # past argument=value + ;; + --print) + PRINT=1 + shift # past argument=value + ;; + --push) + PUSH=1 + shift # past argument=value + ;; + -h|--help) + printUsage + shift + ;; + *) + # further/unknown options + ;; +esac +done + +TARGET_IMAGE_NAME=${1:-invalid} +TARGET_IMAGE_VERSION=${2:-invalid} +FIRST_IMAGE_REFERENCE=${3:-invalid} + +if [ "$TARGET_IMAGE_NAME" = "invalid" ] || [ "$TARGET_IMAGE_VERSION" = "invalid" ] || [ "$FIRST_IMAGE_REFERENCE" = "invalid" ]; then + echo "Error: Invalid image name or version" >&2 + printUsage +fi + +shift 2 + +IMAGE_REFERENCES=($@) +MANIFESTS_TO_PUSH=() +counter=1 +new_version="$(echo "$TARGET_IMAGE_VERSION" | cut -d. -f$counter)" +last_version="" +while [ "$last_version" != "$new_version" ]; do + createManifest "${TARGET_IMAGE_NAME}:${PREFIX}${new_version}${SUFFIX}" + last_version="$new_version" + ((counter++)) + new_version="$(echo "$TARGET_IMAGE_VERSION" | cut -d. -f-"$counter")" +done + +if [ "$LATEST" = "1" ]; then + createManifest "${TARGET_IMAGE_NAME}:latest" +fi + +if [ "$PRINT" = "1" ]; then + echo "${MANIFESTS_TO_PUSH[@]}" +fi + +if [ "$PUSH" = "1" ]; then + if ! command -v "si-manifest-push" >/dev/null 2>&1; then + ./manifest-push.sh "${MANIFESTS_TO_PUSH[@]}" + else + si-manifest-push "${MANIFESTS_TO_PUSH[@]}" + fi +fi diff --git a/resources/shell-tools/install.sh b/resources/shell-tools/install.sh index 6480b760e5f2c1a0fd86bb814e4bc97a367160dc..9a88726c62b6b1d39c201ff6e55add5150d56776 100755 --- a/resources/shell-tools/install.sh +++ b/resources/shell-tools/install.sh @@ -4,10 +4,16 @@ BASENAME="$(dirname "$0")" cp "$BASENAME"/./bin/tagging.sh /usr/local/bin/si-tagging cp "$BASENAME"/./bin/push.sh /usr/local/bin/si-push +cp "$BASENAME"/./bin/manifest.sh /usr/local/bin/si-manifest +cp "$BASENAME"/./bin/manifest-push.sh /usr/local/bin/si-manifest-push cp "$BASENAME"/./bin/fix-dockerfile.sh /usr/local/bin/si-fix chown root:root /usr/local/bin/si-tagging chown root:root /usr/local/bin/si-push +chown root:root /usr/local/bin/si-manifest +chown root:root /usr/local/bin/si-manifest-push chown root:root /usr/local/bin/si-fix chmod 0755 /usr/local/bin/si-tagging chmod 0755 /usr/local/bin/si-push +chmod 0755 /usr/local/bin/si-manifest +chmod 0755 /usr/local/bin/si-manifest-push chmod 0755 /usr/local/bin/si-fix diff --git a/resources/shell-tools/test/manifest-ci.sh b/resources/shell-tools/test/manifest-ci.sh new file mode 100755 index 0000000000000000000000000000000000000000..4282e8f1afba145dbf9bb047ee427acdeb51b83f --- /dev/null +++ b/resources/shell-tools/test/manifest-ci.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -o pipefail +set -u + +BASEDIR=$(dirname "$0") + +podman build --format docker -t mytest-latest:test-amd64 -f- <<EOF +FROM scratch +EOF +podman build --format docker -t mytest-latest:test-arm64 -f- <<EOF +FROM scratch +EOF + +mkdir logs +echo "mytest-latest:test-arm64" > logs/base-aarch64.log +echo "mytest-latest:test-amd64" > logs/base-x86_64.log +IMAGE_REFERENCES=() +for log in logs/*.log; do IMAGE_REFERENCES+=("$(cat "$log")"); done +"$BASEDIR"/../bin/manifest.sh --local registry.example.com/latest-test 1.2.3 "${IMAGE_REFERENCES[@]}" + +EXIT_CODE=1 + + +if podman manifest inspect registry.example.com/latest-test:1.2.3; then + EXIT_CODE=0 +fi + +podman rmi -f registry.example.com/latest-test:1 \ + registry.example.com/latest-test:1.2 \ + registry.example.com/latest-test:1.2.3 +podman rmi -f "$(podman images -q mytest-latest:test-amd64)" +podman rmi -f "$(podman images -q mytest-latest:test-arm64)" +rm -r logs/ + +exit $EXIT_CODE diff --git a/resources/shell-tools/test/manifest-latest.sh b/resources/shell-tools/test/manifest-latest.sh new file mode 100755 index 0000000000000000000000000000000000000000..b87f67c2441a52754c793fc3edc09093547e79ed --- /dev/null +++ b/resources/shell-tools/test/manifest-latest.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -o pipefail +set -u + +BASEDIR=$(dirname "$0") + +podman build --format docker -t mytest-latest:test-amd64 -f- <<EOF +FROM scratch +EOF +podman build --format docker -t mytest-latest:test-arm64 -f- <<EOF +FROM scratch +EOF + +"$BASEDIR"/../bin/manifest.sh --local --latest registry.example.com/latest-test 1.2.3 mytest-latest:test-arm64 mytest-latest:test-amd64 + +ERRORS=0 +podman manifest inspect registry.example.com/latest-test:latest +ERRORS=$((ERRORS + "$?")) + +if [ "$ERRORS" = "0" ]; then + EXIT_CODE=0 +else + EXIT_CODE=1 +fi + +podman rmi -f registry.example.com/latest-test:1 \ + registry.example.com/latest-test:1.2 \ + registry.example.com/latest-test:1.2.3 \ + registry.example.com/latest-test:latest +podman rmi -f "$(podman images -q mytest-latest:test-amd64)" +podman rmi -f "$(podman images -q mytest-latest:test-arm64)" + +exit $EXIT_CODE diff --git a/resources/shell-tools/test/manifest-print.sh b/resources/shell-tools/test/manifest-print.sh new file mode 100755 index 0000000000000000000000000000000000000000..dfd1cadf939ecad97d1662ca8bcc363fc41c3150 --- /dev/null +++ b/resources/shell-tools/test/manifest-print.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -o pipefail +set -u + +BASEDIR=$(dirname "$0") + +podman build --format docker -t mytest-latest:test -f- <<EOF +FROM scratch +EOF + +MANIFESTS_TO_PUSH=$("$BASEDIR"/../bin/manifest.sh --local --print registry.example.com/latest-test 1.2.3 mytest-latest:test) + +EXIT_CODE=1 + +if [ "$MANIFESTS_TO_PUSH" == "registry.example.com/latest-test:1 registry.example.com/latest-test:1.2 registry.example.com/latest-test:1.2.3" ]; then + EXIT_CODE=0 +fi + +podman rmi -f registry.example.com/latest-test:1 \ + registry.example.com/latest-test:1.2 \ + registry.example.com/latest-test:1.2.3 +podman rmi -f "$(podman images -q mytest-latest:test)" + +exit $EXIT_CODE diff --git a/resources/shell-tools/test/manifest-push.sh b/resources/shell-tools/test/manifest-push.sh new file mode 100755 index 0000000000000000000000000000000000000000..b084397c4ee56cd47e2d7576ae55d054b8eab3cd --- /dev/null +++ b/resources/shell-tools/test/manifest-push.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -o pipefail +set -u + +BASEDIR=$(dirname "$0") + +podman build --format docker -t mytest-latest:test-amd64 -f- <<EOF +FROM scratch +EOF +podman build --format docker -t mytest-latest:test-arm64 -f- <<EOF +FROM scratch +EOF + +MANIFESTS_TO_PUSH=$("$BASEDIR"/../bin/manifest.sh --local --print registry.example.com/latest-test 1.2.3 mytest-latest:test-arm64 mytest-latest:test-amd64) + +EXIT_CODE=1 + +PUSHED=$("$BASEDIR"/../bin/manifest-push.sh --dry "$MANIFESTS_TO_PUSH") + +if [ "$PUSHED" == "registry.example.com/latest-test:1 registry.example.com/latest-test:1.2 registry.example.com/latest-test:1.2.3" ]; then + EXIT_CODE=0 +fi + +podman rmi -f registry.example.com/latest-test:1 \ + registry.example.com/latest-test:1.2 \ + registry.example.com/latest-test:1.2.3 +podman rmi -f "$(podman images -q mytest-latest:test-amd64)" +podman rmi -f "$(podman images -q mytest-latest:test-arm64)" + +exit $EXIT_CODE diff --git a/resources/shell-tools/test/manifest.sh b/resources/shell-tools/test/manifest.sh new file mode 100755 index 0000000000000000000000000000000000000000..ce5404be8104bac2f4ceb4d96e6fe7c42129be71 --- /dev/null +++ b/resources/shell-tools/test/manifest.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -o pipefail +set -u + +BASEDIR=$(dirname "$0") + +podman build --format docker -t mytest-latest:test-amd64 -f- <<EOF +FROM scratch +EOF +podman build --format docker -t mytest-latest:test-arm64 -f- <<EOF +FROM scratch +EOF + +"$BASEDIR"/../bin/manifest.sh --local registry.example.com/latest-test 1.2.3 mytest-latest:test-arm64 mytest-latest:test-amd64 + +ERRORS=0 +podman manifest inspect registry.example.com/latest-test:1 +ERRORS=$((ERRORS + "$?")) +podman manifest inspect registry.example.com/latest-test:1.2 +ERRORS=$((ERRORS + "$?")) +podman manifest inspect registry.example.com/latest-test:1.2.3 +ERRORS=$((ERRORS + "$?")) + +if [ "$ERRORS" = "0" ]; then + EXIT_CODE=0 +else + EXIT_CODE=1 +fi + +podman rmi -f registry.example.com/latest-test:1 \ + registry.example.com/latest-test:1.2 \ + registry.example.com/latest-test:1.2.3 +podman rmi -f "$(podman images -q mytest-latest:test-amd64)" +podman rmi -f "$(podman images -q mytest-latest:test-arm64)" + +exit $EXIT_CODE