diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 90535ac95f3459897af93e5891147bdfbe1c1e10..9ce0448e40048d8208ce8ee840541f1262e358f2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,30 +10,19 @@ stages: - tag variables: - CI_REGISTRY_IMAGE_VERSION: "1.3.0" - CI_REGISTRY_BUILD_ARG: "--build-arg SI_TOOLS_VERSION=0.3.1" - + CI_REGISTRY_IMAGE_VERSION: "2.0.0" + CI_CONTAINER_BUILD_ARCHS: "amd64:arm64" container-test: stage: test inherit: default: false - variables: - - CI_REGISTRY - - CI_REGISTRY_USER - - CI_REGISTRY_PASSWORD - - CI_REGISTRY_IMAGE - - CI_REGISTRY_BUILD_ARGS - image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA + variables: true + image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA-amd64 before_script: - podman login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - podman build --pull $CI_REGISTRY_BUILD_ARGS -t "build-image:test" . - - si-tagging -l "build-image:test" "example.com/tagging-image" "0.1.2.3" - - podman images --format "{{.Repository}}:{{.Tag}}" | grep -Pe "^example.com/tagging-image:0.1.2$" - - podman images --format "{{.Repository}}:{{.Tag}}" | grep "latest" - - podman images --format "{{.Repository}}:{{.Tag}}" | grep -Pe "^example.com/tagging-image:0$" - container-tagging: stage: tag diff --git a/gitlab-ci-template.yml b/gitlab-ci-template.yml index 85ee3b704980531845f21a73c8cfee6db4ad752c..01d67b002f3271cbf9bd8d5b1a116c6c78dcea7d 100644 --- a/gitlab-ci-template.yml +++ b/gitlab-ci-template.yml @@ -4,49 +4,72 @@ stages: variables: CI_REGISTRY_BUILD_ARGS: "" + CI_CONTAINER_BUILD_ARCHS: "amd64" -container-build: +.container-build: stage: build variables: CI_REGISTRY_CONTAINER_FORMAT: "docker" inherit: default: false - variables: - - CI_REGISTRY - - CI_REGISTRY_USER - - CI_REGISTRY_PASSWORD - - CI_REGISTRY_IMAGE - - CI_REGISTRY_BUILD_DOCKERFILE - - CI_REGISTRY_BUILD_ARGS - image: quay.io/sheogorath/build-ah-engine:1.3.0 + variables: true + image: quay.io/sheogorath/build-ah-engine:2.0.0 before_script: - podman login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - - export BUILD_DATE=$(date --rfc-3339 ns) - - export VCS_REF=$CI_COMMIT_SHA - - export VCS_URL=$CI_PROJECT_URL + - export VCS_REF="$CI_COMMIT_SHA" + - export VCS_URL="$CI_PROJECT_URL" + - export BUILD_DATE="$(date --rfc-3339 ns)" script: - si-fix "${CI_REGISTRY_BUILD_DOCKERFILE:-./Dockerfile}" - - podman build --pull --format "$CI_REGISTRY_CONTAINER_FORMAT" + - podman build --pull --label "org.opencontainers.image.source=$CI_PROJECT_URL" --label "org.opencontainers.image.revision=$CI_COMMIT_SHA" --label "org.opencontainers.image.created=$(date --rfc-3339 ns)" --label "org.opencontainers.image.title=$CI_PROJECT_TITLE" $CI_REGISTRY_BUILD_ARGS - -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA" + -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA-$CI_REGISTRY_IMAGE_ARCH" + --format docker . - - podman push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA" + - podman push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA-$CI_REGISTRY_IMAGE_ARCH" + +container-build-amd64: + extends: .container-build + variables: + CI_REGISTRY_IMAGE_ARCH: amd64 + tags: + - amd64 + rules: + - if: '$CI_CONTAINER_BUILD_ARCHS =~ /:?amd64:?/ && $CI_PIPELINE_SOURCE != "merge_request_event"' + +container-build-arm64: + extends: .container-build + variables: + CI_REGISTRY_IMAGE_ARCH: arm64 + tags: + - arm64 + rules: + - if: '$CI_CONTAINER_BUILD_ARCHS =~ /:?arm64:?/ && $CI_PIPELINE_SOURCE != "merge_request_event"' + +container-build-aarch64: + extends: .container-build + variables: + CI_REGISTRY_IMAGE_ARCH: aarch64 + tags: + - aaarch64 + rules: + - if: '$CI_CONTAINER_BUILD_ARCHS =~ /:?aarch64:?/ && $CI_PIPELINE_SOURCE != "merge_request_event"' container-tagging: stage: tag inherit: default: false variables: true - image: quay.io/sheogorath/build-ah-engine:1.3.0 + image: quay.io/sheogorath/build-ah-engine:2.0.0 before_script: - podman login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - export BUILD_DATE="$(date --rfc-3339 ns)" script: - - podman pull "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA" + - si-pull "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA" - si-tagging -l "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA" "$CI_REGISTRY_IMAGE" "${CI_REGISTRY_IMAGE_VERSION}" - si-push "$CI_REGISTRY_IMAGE" resource_group: latest diff --git a/resources/shell-tools/.gitlab-ci.yml b/resources/shell-tools/.gitlab-ci.yml index 80cf250ca2977cf20d2d9f1fa33157022a384995..5c5700c3229d098cb7facbc4c334d5d3d3e42787 100644 --- a/resources/shell-tools/.gitlab-ci.yml +++ b/resources/shell-tools/.gitlab-ci.yml @@ -17,3 +17,4 @@ shell-tools-test: script: - ./resources/shell-tools/test/latest.sh - ./resources/shell-tools/test/fix-dockerfile-pinning.sh + - ./resources/shell-tools/test/multiarch.sh diff --git a/resources/shell-tools/bin/pull.sh b/resources/shell-tools/bin/pull.sh new file mode 100755 index 0000000000000000000000000000000000000000..934c9760daecf921f11dfa92b4839115263bf98e --- /dev/null +++ b/resources/shell-tools/bin/pull.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +set -o pipefail +set -u +set -e + +VERSION=0.1.0 + +printVersion() { + echo "$VERSION" +} + +printUsage() { + printVersion + echo " + Shivering-Isles pull tool + + This tool will pull all available base container images from upstream + + Usage of $0: + $0 <IMAGE REFERENCE> + + Example: + $0 registry.example.com/example/app + " + exit 1 +} + +CONTAINER_CMD=podman + +if ! command -v "$CONTAINER_CMD" >/dev/null 2>&1; then + CONTAINER_CMD=docker +fi + +if [ "$1" = "--help" ]; then + printUsage +fi + +for i in "$@" +do +case $i in + --container-cmd=*) + CONTAINER_CMD="${i#*=}" + shift # past argument with no value + ;; + -v|--version) + printVersion + exit 0 + shift # past argument with no value + ;; + -h|--help) + printUsage + shift + ;; + *) + # further/unknown options + ;; +esac +done + +CONTAINER_IMAGE_NAME=${1:-invalid} + +if [ "$CONTAINER_IMAGE_NAME" = "invalid" ]; then + echo "Error: Invalid image name" >&2 + printUsage +fi + +IMAGE_PULL_SUCCESS=0 + +# shellcheck disable=SC2015 +podman pull "$CONTAINER_IMAGE_NAME" && \ + IMAGE_PULL_SUCCESS=1 || \ + true + +# shellcheck disable=SC2015 +podman pull "$CONTAINER_IMAGE_NAME-amd64" && \ + IMAGE_PULL_SUCCESS=1 || \ + true + +# shellcheck disable=SC2015 +podman pull "$CONTAINER_IMAGE_NAME-arm64" || \ + podman pull "$CONTAINER_IMAGE_NAME-aarch64" && \ + IMAGE_PULL_SUCCESS=1 || \ + true + +if [ "$IMAGE_PULL_SUCCESS" == "0" ]; then + echo "Error: No images pulled" >&2 + exit 1 +fi diff --git a/resources/shell-tools/bin/push.sh b/resources/shell-tools/bin/push.sh index d6f7f9ba552d3b5e9814e6d88d2c498450a03523..4bdd00909b7b03e67d988509d7e4c351ead96153 100755 --- a/resources/shell-tools/bin/push.sh +++ b/resources/shell-tools/bin/push.sh @@ -26,8 +26,15 @@ printUsage() { exit 1 } -CONTAINER_CMD=podman +pushImageOrManifest() { + if podman manifest inspect "$1"; then + $CONTAINER_CMD manifest push --all --format v2s2 "$1" "docker://$1" + else + $CONTAINER_CMD push "$1" + fi +} +CONTAINER_CMD=podman if ! command -v "$CONTAINER_CMD" >/dev/null 2>&1; then CONTAINER_CMD=docker @@ -67,4 +74,4 @@ if [ "$CONTAINER_IMAGE_NAME" = "invalid" ]; then fi # shellcheck disable=SC2086 -$CONTAINER_CMD images --format "{{.Repository}}:{{.Tag}}" "$CONTAINER_IMAGE_NAME" | grep "$CONTAINER_IMAGE_NAME" | xargs -L 1 $CONTAINER_CMD push +$CONTAINER_CMD images --format "{{.Repository}}:{{.Tag}}" "$CONTAINER_IMAGE_NAME" | grep "$CONTAINER_IMAGE_NAME" | xargs -L 1 pushImageOrManifest diff --git a/resources/shell-tools/bin/tagging.sh b/resources/shell-tools/bin/tagging.sh index 6e78a847edf35a61a8f4f56122ede6318cac47ce..dc66ad291f92c011c1b80e4d3b635c9978ea8d02 100755 --- a/resources/shell-tools/bin/tagging.sh +++ b/resources/shell-tools/bin/tagging.sh @@ -24,11 +24,33 @@ printUsage() { exit 1 } -CONTAINER_CMD=podman +createTagOrManifest() { + TARGET_IMAGE_REFERENCE=$1 + if [ "$MULTIARCH_IMAGES_COUNT" -gt "0" ]; then + $CONTAINER_CMD manifest create "$TARGET_IMAGE_REFERENCE" + for IMAGE_REFERENCE in "${MULTIARCH_IMAGES[@]}" + do + MANIFEST_OPTIONS=() + if [[ "$IMAGE_REFERENCE" =~ "aarch64"|"arm64" ]]; then + MANIFEST_OPTIONS+=(--variant v8) + fi + if [ "$LOCAL" = "1" ]; then + $CONTAINER_CMD manifest add "${MANIFEST_OPTIONS[@]}" "$TARGET_IMAGE_REFERENCE" containers-storage:"$IMAGE_REFERENCE" + else + $CONTAINER_CMD manifest add "${MANIFEST_OPTIONS[@]}" "$TARGET_IMAGE_REFERENCE" "$IMAGE_REFERENCE" + fi + done + else + $CONTAINER_CMD tag "$CURRENT_IMAGE_REFERENCE" "$TARGET_IMAGE_REFERENCE" + fi +} + +CONTAINER_CMD="podman" PREFIX="" SUFFIX="" LATEST=0 - +LOCAL=0 +MULTIARCH_SUPPORTED="amd64|arm64|aarch64" if ! command -v "$CONTAINER_CMD" >/dev/null 2>&1; then CONTAINER_CMD=docker @@ -53,6 +75,10 @@ case $i in LATEST=1 shift # past argument=value ;; + --local) + LOCAL=1 + shift # past argument=value + ;; --container-cmd=*) CONTAINER_CMD="${i#*=}" shift # past argument with no value @@ -81,16 +107,19 @@ if [ "$CURRENT_IMAGE_REFERENCE" = "invalid" ] || [ "$TARGET_IMAGE_NAME" = "inval printUsage fi +mapfile -t MULTIARCH_IMAGES < <($CONTAINER_CMD images | awk '{print $1 ":" $2}' | grep -Pe "$CURRENT_IMAGE_REFERENCE-($MULTIARCH_SUPPORTED)$") +MULTIARCH_IMAGES_COUNT=${#MULTIARCH_IMAGES[@]} + counter=1 new_version="$(echo "$TARGET_IMAGE_VERSION" | cut -d. -f$counter)" last_version="" while [ "$last_version" != "$new_version" ]; do - $CONTAINER_CMD tag "$CURRENT_IMAGE_REFERENCE" "${TARGET_IMAGE_NAME}:${PREFIX}${new_version}${SUFFIX}" + createTagOrManifest "${TARGET_IMAGE_NAME}:${PREFIX}${new_version}${SUFFIX}" last_version="$new_version" ((counter++)) - new_version="$(echo "$TARGET_IMAGE_VERSION" | cut -d. "-f-$counter")" + new_version="$(echo "$TARGET_IMAGE_VERSION" | cut -d. -f-"$counter")" done if [ "$LATEST" = "1" ]; then - $CONTAINER_CMD tag "$CURRENT_IMAGE_REFERENCE" "${TARGET_IMAGE_NAME}:latest" + createTagOrManifest "${TARGET_IMAGE_NAME}:latest" fi diff --git a/resources/shell-tools/install.sh b/resources/shell-tools/install.sh index 6480b760e5f2c1a0fd86bb814e4bc97a367160dc..4ec0a50cbb9c7dc31aa550f5f451e3179f67700d 100755 --- a/resources/shell-tools/install.sh +++ b/resources/shell-tools/install.sh @@ -4,10 +4,13 @@ BASENAME="$(dirname "$0")" cp "$BASENAME"/./bin/tagging.sh /usr/local/bin/si-tagging cp "$BASENAME"/./bin/push.sh /usr/local/bin/si-push +cp "$BASENAME"/./bin/pull.sh /usr/local/bin/si-pull cp "$BASENAME"/./bin/fix-dockerfile.sh /usr/local/bin/si-fix chown root:root /usr/local/bin/si-tagging chown root:root /usr/local/bin/si-push +chown root:root /usr/local/bin/si-pull chown root:root /usr/local/bin/si-fix chmod 0755 /usr/local/bin/si-tagging chmod 0755 /usr/local/bin/si-push +chmod 0755 /usr/local/bin/si-pull chmod 0755 /usr/local/bin/si-fix diff --git a/resources/shell-tools/test/multiarch.sh b/resources/shell-tools/test/multiarch.sh new file mode 100755 index 0000000000000000000000000000000000000000..3afc623db52acca6e95dabc10df2b40386c245f5 --- /dev/null +++ b/resources/shell-tools/test/multiarch.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -o pipefail +set -u + +BASEDIR=$(dirname "$0") + +podman build --format docker -t mytest-latest:test-amd64 -f- <<EOF +FROM scratch +EOF +podman build --format docker -t mytest-latest:test-arm64 -f- <<EOF +FROM scratch +EOF + +TEST_OUTPUT="$(mktemp)" +TEST_VERIFY_OUTPUT="$(mktemp)" + +"$BASEDIR"/../bin/tagging.sh --local -l mytest-latest:test registry.example.com/latest-test 1.2.3 + +ERRORS=0 +podman manifest inspect registry.example.com/latest-test:1 +ERRORS=$((ERRORS + "$?")) +podman manifest inspect registry.example.com/latest-test:1.2 +ERRORS=$((ERRORS + "$?")) +podman manifest inspect registry.example.com/latest-test:1.2.3 +ERRORS=$((ERRORS + "$?")) +podman manifest inspect registry.example.com/latest-test:latest +ERRORS=$((ERRORS + "$?")) + +if [ "$ERRORS" = "0" ]; then + EXIT_CODE=0 +else + EXIT_CODE=1 +fi + +rm -f "$TEST_OUTPUT" "$TEST_VERIFY_OUTPUT" +podman rmi -f registry.example.com/latest-test:1 \ + registry.example.com/latest-test:1.2 \ + registry.example.com/latest-test:1.2.3 \ + registry.example.com/latest-test:latest +podman rmi -f "$(podman images -q mytest-latest:test-amd64)" +podman rmi -f "$(podman images -q mytest-latest:test-arm64)" + +exit $EXIT_CODE