diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..0a764a4de3a890dbe2a3336c648f7f6d1892c132 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +env diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9b6dbdecba35fdecd11dca412f942dbfaee2922c --- /dev/null +++ b/README.md @@ -0,0 +1,50 @@ +Mirror Bot +=== + +Repository to provide the tooling to mirror repositories in this GitLab instance. + +mirror.sh +--- + +Script to mirror repositories from source to target. Requires the host it's +running on to be setup properly with write access to the target repository. + +``` + Usage: + ./mirror.sh <pull repo> <push repo> + + Example: + ./mirror.sh https://github.com/me/myrepo.git ssh://git@gitlab.example.com/me/myrepo.git + +``` + +setup-github-mirror.sh +--- + +Script to interact with GitLab in order to provide projects setup correctly for +the purpose of mirroring the repository content only. + +Those repositories are setup to: + +- Configure the mirror bot ssh-key as a deploy key with write access to the + project +- Disable issues, merge requests, CI/CD, snippets, and the wiki on this + project +- Setup the organization of user account picture from GitHub as project avatar +- Provides a ready to use script line for `./mirror.sh` which can be run directly or via GitLab-CI + +``` + Usage: + ./setup-github-mirror.sh <repo slug> + Example: + ./setup-github-mirror.sh octocat/Hello-World + +``` + +Preparation to do on GitLab before using this project on your GitLab +--- + +1. Create an group with the path `github-mirror`. The resulting URL should look like this: `https://gitlab.example.com/github-mirror` +2. Create a copy of this repository +3. Modify the .gitlab-ci.yml to represent your needs of mirroring +4. Setup a scheduled CI task in the frequency you prefer your mirrored repositories to be updated. diff --git a/env.example b/env.example new file mode 100644 index 0000000000000000000000000000000000000000..12d4f8cc6acef493b852cc7572239caaddd426a9 --- /dev/null +++ b/env.example @@ -0,0 +1,12 @@ +ACCESS_TOKEN=<your gitlab token> +GITLAB_URL=https://gitlab.example.com + +# ID of the deployment key of your mirror bot +# you can find it using curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/deploy_keys" +MIRROR_KEY_ID=55 +# Group ID of the parent group you use to mirror your repos to. +# You can find it using curl"https://gitlab.example.com/api/v4/groups" +# +# This groups slug is currently hardcoded to github-mirror +MIRROR_GROUP_ID=45 + diff --git a/setup-github-mirror.sh b/setup-github-mirror.sh new file mode 100755 index 0000000000000000000000000000000000000000..397e72314a2f0f219ec90b026f976b99447ff586 --- /dev/null +++ b/setup-github-mirror.sh @@ -0,0 +1,136 @@ +#!/bin/sh + +set -e +if [ "$DEBUG" = "1" ]; then + set -x +fi + +if [ "$1" = "" ]; then + cat >/dev/stderr << EOF + Usage: + $0 <repo slug> + Example: + $0 octocat/Hello-World +EOF + exit 1 +fi +REPO_SLUG="$1" + +# Load configs +SCRIPT_DIR=$(dirname $0) + +if [ ! -e "$SCRIPT_DIR/env" ]; then + cat >/dev/stderr <<EOF + Missing configuration file '$SCRIPT_DIR/env'. + Please copy '$SCRIPT_DIR/env.example' and fill the variables properly. +EOF + exit 1 +fi + +. "$SCRIPT_DIR/env" + +# Collect information from GitHub + +GITHUB_FILE="$(mktemp)" + +curl "https://api.github.com/repos/$REPO_SLUG" > "$GITHUB_FILE" + +GITHUB_NAME=$(jq '.name' "$GITHUB_FILE" | sed -e 's/^"//g' -e 's/"$//g') +GITHUB_AVATAR_URL=$(jq '.owner.avatar_url' "$GITHUB_FILE" | sed -e 's/^"//g' -e 's/"$//g') +GITHUB_DESCRIPTION=$(jq '.description' "$GITHUB_FILE" | sed -e 's/^"//g' -e 's/"$//g') +GITHUB_OWNER=$(jq '.owner.login' "$GITHUB_FILE" | sed -e 's/^"//g' -e 's/"$//g') + +rm -f "$GITHUB_FILE" + + +# Setup subgroup +GITLAB_GROUP_FILE="$(mktemp)" + +GITLAB_GROUP_ID=$(curl "${GITLAB_URL}/api/v4/groups/github-mirror%2F${GITHUB_OWNER}" | jq .id) + +if [ "$GITLAB_GROUP_ID" = "null" ]; then + curl --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" \ + -H "Accept: application/json" \ + -X POST \ + --data-urlencode "name=${GITHUB_OWNER}" \ + --data-urlencode "path=${GITHUB_OWNER}" \ + --data-urlencode "visibility=public" \ + --data-urlencode "parent_id=${MIRROR_GROUP_ID}" \ + "${GITLAB_URL}/api/v4/groups" > "$GITLAB_GROUP_FILE" + GITLAB_GROUP_ID=$(jq '.id' "$GITLAB_GROUP_FILE") +fi + +rm -f "$GITLAB_GROUP_FILE" + +if [ "${GITHUB_OWNER}" = "null" ] || [ "${GITHUB_NAME}" = "null" ]; then + cat >/dev/stderr <<EOF + Repo https://github.com/${REPO_SLUG} error. Owner or repo not found. +EOF + exit 1 +fi + +GITLAB_PROJECT_FILE="$(mktemp)" + +if [ "$(curl "${GITLAB_URL}/api/v4/projects/github-mirror%2F${GITHUB_OWNER}%2F${GITHUB_NAME}" | tee "$GITLAB_PROJECT_FILE" | jq '.id')" = "null" ]; then + curl --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" \ + -H "Accept: application/json" \ + -X POST \ + --data-urlencode "path=${GITHUB_NAME}" \ + --data-urlencode "namespace_id=${GITLAB_GROUP_ID}" \ + --data-urlencode 'visibility=public' \ + "${GITLAB_URL}/api/v4/projects" > "$GITLAB_PROJECT_FILE" +fi + +GITLAB_PROJECT_ID=$(jq .id "$GITLAB_PROJECT_FILE") +GITLAB_PROJECT_URL=$(jq '.ssh_url_to_repo' "$GITLAB_PROJECT_FILE" | sed -e 's/^"//g' -e 's/"$//g') + +rm -f "$GITLAB_PROJECT_FILE" + + +# Setup Avatar for project +GITLAB_IMAGE_FILE=$(mktemp --suffix=.png) + +curl "$GITHUB_AVATAR_URL" > "$GITLAB_IMAGE_FILE" + +curl --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" \ +-H "Accept: application/json" \ +-X PUT \ +-F "avatar=@${GITLAB_IMAGE_FILE}" \ +"${GITLAB_URL}/api/v4/projects/${GITLAB_PROJECT_ID}" > /dev/null + +rm -f "${GITLAB_IMAGE_FILE}" + + +# Setup project permissions and description +GITLAB_DESCRIPTION=" +${GITHUB_DESCRIPTION} + +Mirror of https://github.com/${REPO_SLUG} +" + +curl --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" \ +-H "Accept: application/json" \ +-X PUT \ +--data-urlencode "description=${GITLAB_DESCRIPTION}" \ +--data-urlencode 'issues_access_level=disabled' \ +--data-urlencode 'merge_requests_access_level=disabled' \ +--data-urlencode 'builds_access_level=disabled' \ +--data-urlencode 'wiki_access_level=disabled' \ +--data-urlencode 'snippets_access_level=disabled' \ +"${GITLAB_URL}/api/v4/projects/${GITLAB_PROJECT_ID}" > /dev/null + +# Enable mirror bot deployment key +curl --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" \ +-H "Accept: application/json" \ +-X POST \ +"${GITLAB_URL}/api/v4/projects/${GITLAB_PROJECT_ID}/deploy_keys/${MIRROR_KEY_ID}/enable" > /dev/null + +# Allow mirror bot deployment key to push changes to project +curl --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" \ +-H "Accept: application/json" \ +-X PUT \ +--data-urlencode "can_push=true" \ +"${GITLAB_URL}/api/v4/projects/${GITLAB_PROJECT_ID}/deploy_keys/${MIRROR_KEY_ID}" > /dev/null + +echo "./mirror.sh 'https://github.com/${REPO_SLUG}.git' '${GITLAB_PROJECT_URL}'" +