From 8b20d49fff22902c67543ef34d15e0faf61eccf4 Mon Sep 17 00:00:00 2001 From: Tomas Pecka <tomas.pecka@fit.cvut.cz> Date: Tue, 10 Sep 2019 11:59:59 +0200 Subject: [PATCH] CI/CD: Packaging scripts. --- .gitlab-ci.yml | 81 +++-------- .../CD/distrofiles}/arch/nightly/PKGBUILD | 0 .../CD/distrofiles}/arch/release/PKGBUILD | 0 .../scripts/CD/packaging/package-archlinux.sh | 27 ++++ extra/scripts/CD/packaging/package-debian.sh | 25 ++++ extra/scripts/CD/packaging/package-docker.sh | 50 +++++++ extra/scripts/CD/packaging/package-sources.sh | 14 ++ extra/scripts/CD/packaging/package.sh | 129 ++++++++++++++++++ extra/scripts/build.sh | 19 ++- 9 files changed, 277 insertions(+), 68 deletions(-) rename extra/{distro => scripts/CD/distrofiles}/arch/nightly/PKGBUILD (100%) rename extra/{distro => scripts/CD/distrofiles}/arch/release/PKGBUILD (100%) create mode 100644 extra/scripts/CD/packaging/package-archlinux.sh create mode 100644 extra/scripts/CD/packaging/package-debian.sh create mode 100644 extra/scripts/CD/packaging/package-docker.sh create mode 100644 extra/scripts/CD/packaging/package-sources.sh create mode 100755 extra/scripts/CD/packaging/package.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6ab49fdd16..c4dfc5ea02 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,10 +20,7 @@ stages: dependencies: [] stage: build script: - - mkdir -p build && cd build - - ../CMake/generate.py -wm - - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON .. -GNinja - - ninja -j $(grep -c processor /proc/cpuinfo) + - extra/scripts/build.sh -d build -m Release -n artifacts: paths: - build/ @@ -294,12 +291,15 @@ static-analysis:clang-tidy: .template:package: stage: package dependencies: [] + script: + - DISTRO=$(echo "$CI_JOB_NAME" | cut -d":" -f2) + - MODE=$(echo "$CI_JOB_NAME" | cut -d":" -f3) + - extra/scripts/CD/packaging/package.sh $MODE $DISTRO . .template:release: stage: release dependencies: [] - .releaser: extends: .template:release before_script: @@ -319,35 +319,22 @@ static-analysis:clang-tidy: .package:archlinux: extends: .template:package - image: gitlab.fit.cvut.cz:5000/algorithms-library-toolkit/ci-docker-images/archlinux-makepkg - script: - - pacman -Suy --noconfirm --needed base-devel gnupg - - echo "$GPG_PRIVATE_KEY" | sudo -u builder -- gpg --no-tty --batch --import - - - KEYGRIP=$(sudo -u builder -- gpg --with-keygrip -k $GPG_KEYID | grep Keygrip | cut -d= -f2 | tr -d ' ') - - sudo -u builder -- /usr/lib/gnupg/gpg-preset-passphrase --preset --passphrase "$GPG_PASSPHRASE" "$KEYGRIP" - - sudo -u builder -- makepkg --syncdeps --noconfirm --sign --key $GPG_KEYID + image: archlinux/base:latest artifacts: paths: - "*.pkg.tar.xz*" expire_in: 1 day variables: + MAKEFLAGS: "-j3" PACKAGER: "Tomáš Pecka <peckato1@fit.cvut.cz>" -# ----------------------------------------------------------------------------- - package:archlinux:nightly: <<: *release_nightly extends: .package:archlinux - before_script: - - cp extra/distro/arch/nightly/PKGBUILD . - - chown builder:builder PKGBUILD package:archlinux:release: <<: *release_tag extends: .package:archlinux - before_script: - - cp extra/distro/arch/release/PKGBUILD . - - chown builder:builder PKGBUILD # ----------------------------------------------------------------------------- .release:archlinux: @@ -385,13 +372,6 @@ release:archlinux:release: variables: DOCKER_DRIVER: overlay2 DOCKER_HOST: docker - # --------- - IMAGE_CLI: $CI_REGISTRY_IMAGE/cli - IMAGE_GUI: $CI_REGISTRY_IMAGE/gui - IMAGE_ALL: $CI_REGISTRY_IMAGE - TAG_NIGHTLY: nightly - TAG_RELEASE: $CI_COMMIT_REF_NAME - TAG_LATEST: latest before_script: - apk add --no-cache docker - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY @@ -408,28 +388,11 @@ package:docker:nightly: <<: *release_nightly <<: *docker extends: .package:docker - script: - - docker build --target=deploy-cli -f extra/docker/Dockerfile.master -t "$IMAGE_CLI:$TAG_NIGHTLY" . - - docker build --target=deploy-all -f extra/docker/Dockerfile.master -t "$IMAGE_ALL:$TAG_NIGHTLY" . - - docker tag "$IMAGE_ALL:$TAG_NIGHTLY" "$IMAGE_GUI:$TAG_NIGHTLY" # gui-version is now only a symlink to all-version - - docker run "$IMAGE_CLI:$TAG_NIGHTLY" /usr/bin/aql2 --help - - docker run "$IMAGE_GUI:$TAG_NIGHTLY" /usr/bin/aql2 --help - - docker run "$IMAGE_ALL:$TAG_NIGHTLY" /usr/bin/aql2 --help - - docker save -o images.tar "$IMAGE_CLI:$TAG_NIGHTLY" "$IMAGE_GUI:$TAG_NIGHTLY" "$IMAGE_ALL:$TAG_NIGHTLY" package:docker:release: <<: *release_tag <<: *docker extends: .package:docker - script: - - docker build --target=deploy-cli -f extra/docker/Dockerfile.master -t "$IMAGE_CLI:$TAG_NIGHTLY" . - - docker build --target=deploy-all -f extra/docker/Dockerfile.master -t "$IMAGE_ALL:$TAG_NIGHTLY" . - - docker tag "$IMAGE_ALL:$TAG_RELEASE" "$IMAGE_GUI:$TAG_RELEASE" # gui-version is now only a symlink to all-version - - docker tag "$IMAGE_ALL:$TAG_LATEST" "$IMAGE_GUI:$TAG_LATEST" # gui-version is now only a symlink to all-version - - docker run "$IMAGE_ALL:$TAG_RELEASE" /usr/bin/aql2 --help - - docker run "$IMAGE_CLI:$TAG_RELEASE" /usr/bin/aql2 --help - - docker run "$IMAGE_GUI:$TAG_RELEASE" /usr/bin/aql2 --help - - docker save -o images.tar "$IMAGE_CLI:$TAG_RELEASE" "$IMAGE_GUI:$TAG_RELEASE" "$IMAGE_ALL:$TAG_RELEASE" # ----------------------------------------------------------------------------- @@ -442,9 +405,9 @@ release:docker:nightly: url: https://gitlab.fit.cvut.cz/algorithms-library-toolkit/automata-library/container_registry script: - docker load -i images.tar - - docker push "$IMAGE_CLI:$TAG_NIGHTLY" - - docker push "$IMAGE_GUI:$TAG_NIGHTLY" - - docker push "$IMAGE_ALL:$TAG_NIGHTLY" + - docker push $CI_REGISTRY_IMAGE/cli:nightly + - docker push $CI_REGISTRY_IMAGE/gui:nightly + - docker push $CI_REGISTRY_IMAGE:nightly dependencies: - package:docker:nightly @@ -457,9 +420,12 @@ release:docker:release: url: https://gitlab.fit.cvut.cz/algorithms-library-toolkit/automata-library/container_registry script: - docker load -i images.tar - - docker push "$IMAGE_CLI:$TAG_RELEASE" - - docker push "$IMAGE_GUI:$TAG_RELEASE" - - docker push "$IMAGE_ALL:$TAG_RELEASE" + - docker push $CI_REGISTRY_IMAGE/cli:latest + - docker push $CI_REGISTRY_IMAGE/gui:latest + - docker push $CI_REGISTRY_IMAGE:latest + - docker push $CI_REGISTRY_IMAGE/cli:$CI_COMMIT_REF_NAME + - docker push $CI_REGISTRY_IMAGE/gui:$CI_COMMIT_REF_NAME + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME dependencies: - package:docker:release @@ -469,13 +435,6 @@ release:docker:release: .package:sources: extends: .template:package - before_script: - - apk add --no-cache git gnupg - - mkdir -p ~/.gnupg && echo -ne 'allow-preset-passphrase\nmax-cache-ttl 60480000\ndefault-cache-ttl 60480000\n' > ~/.gnupg/gpg-agent.conf - - echo "$GPG_PRIVATE_KEY" | gpg --no-tty --batch --import - - - KEYGRIP=$(gpg --with-keygrip -k $GPG_KEYID | grep Keygrip | cut -d= -f2 | tr -d ' ') - - /usr/libexec/gpg-preset-passphrase --preset --passphrase "$GPG_PASSPHRASE" "$KEYGRIP" - artifacts: paths: - "*.tar.gz*" @@ -484,18 +443,10 @@ release:docker:release: package:sources:nightly: <<: *release_nightly extends: .package:sources - script: - - FILENAME=algorithms-library-$(git describe --long --tags --match="v*" | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g').tar.gz - - git archive --format=tar.gz --prefix="algorithms-library/" HEAD -o $FILENAME - - gpg --armor --detach-sig -o $FILENAME.sig $FILENAME package:sources:release: <<: *release_tag extends: .package:sources - script: - - FILENAME=algorithms-library-$(git describe --tags --match="v*" --abbrev=0 | sed 's/^v//g').tar.gz - - git archive --format=tar.gz --prefix="algorithms-library/" HEAD -o $FILENAME - - gpg --armor --detach-sig -o $FILENAME.sig $FILENAME # ----------------------------------------------------------------------------- diff --git a/extra/distro/arch/nightly/PKGBUILD b/extra/scripts/CD/distrofiles/arch/nightly/PKGBUILD similarity index 100% rename from extra/distro/arch/nightly/PKGBUILD rename to extra/scripts/CD/distrofiles/arch/nightly/PKGBUILD diff --git a/extra/distro/arch/release/PKGBUILD b/extra/scripts/CD/distrofiles/arch/release/PKGBUILD similarity index 100% rename from extra/distro/arch/release/PKGBUILD rename to extra/scripts/CD/distrofiles/arch/release/PKGBUILD diff --git a/extra/scripts/CD/packaging/package-archlinux.sh b/extra/scripts/CD/packaging/package-archlinux.sh new file mode 100644 index 0000000000..4cfb58a23c --- /dev/null +++ b/extra/scripts/CD/packaging/package-archlinux.sh @@ -0,0 +1,27 @@ +# $GIT_ROOT +# $TYPE (nightly/release) +# $CWD + +before () { + # prepare base-devel. gpg, git + pacman -Suy --noconfirm --needed git gnupg sudo base-devel namcap + useradd builder -m && passwd -d builder && printf 'builder ALL=(ALL) ALL\n' | tee -a /etc/sudoers + + export -f gpg_import + su builder -c '/bin/sh -c gpg_import' +} + +package () { + set -e + + # prepare PKGBUILD + cp $GIT_ROOT/extra/scripts/CD/distrofiles/arch/$TYPE/PKGBUILD . + chown builder:builder PKGBUILD + + # makepkg Outputs: + # - FOO.pkg.tar.xz + # - FOO.pkg.tar.xz.sig + sudo -u builder -- makepkg --syncdeps --noconfirm --sign --key $GPG_KEYID + + set +e +} diff --git a/extra/scripts/CD/packaging/package-debian.sh b/extra/scripts/CD/packaging/package-debian.sh new file mode 100644 index 0000000000..c4f229484f --- /dev/null +++ b/extra/scripts/CD/packaging/package-debian.sh @@ -0,0 +1,25 @@ +package () { + set -e + # apt-get install dh-make debsigs devscripts build-essentials + + export LOGNAME="peckato1" + export DEBFULLNAME="Tomáš Pecka" + export DEBEMAIL="peckato1@fit.cvut.cz" + + # create orig file + cp $ARCHIVE ${ARCHIVE%.tar.gz}.orig.tar.gz + + # extract sources and build + tar xzvf $ARCHIVE + pushd $ARCHIVE_DIR + cp -r $GIT_ROOT/extra/scripts/CD/distrofiles/deb/debian . + debuild + + popd + + set +e +} + +publish () { + return 0 +} diff --git a/extra/scripts/CD/packaging/package-docker.sh b/extra/scripts/CD/packaging/package-docker.sh new file mode 100644 index 0000000000..96d04f3ce1 --- /dev/null +++ b/extra/scripts/CD/packaging/package-docker.sh @@ -0,0 +1,50 @@ +before () { + :; +} + +package () { + set -e + + IMAGE_CLI=$CI_REGISTRY_IMAGE/cli + IMAGE_GUI=$CI_REGISTRY_IMAGE/gui + IMAGE_ALL=$CI_REGISTRY_IMAGE + TAG_NIGHTLY=nightly + TAG_RELEASE=$CI_COMMIT_REF_NAME + TAG_LATEST=latest + + BUILD_ALL=all_build + BUILD_CLI=cli_build + + docker build --target=deploy-cli -f $GIT_ROOT/extra/docker/Dockerfile.master -t $BUILD_CLI . + docker build --target=deploy-all -f $GIT_ROOT/extra/docker/Dockerfile.master -t $BUILD_ALL . + + docker run $BUILD_CLI /usr/bin/aql2 --help + docker run $BUILD_ALL /usr/bin/aql2 --help + + if [[ $TYPE == "nightly" ]]; then + docker tag "$BUILD_CLI" "$IMAGE_CLI:$TAG_NIGHTLY" + docker tag "$BUILD_ALL" "$IMAGE_GUI:$TAG_NIGHTLY" # gui-version is now only a symlink to all-version + docker tag "$BUILD_ALL" "$IMAGE_ALL:$TAG_NIGHTLY" + + docker save -o images.tar \ + "$IMAGE_CLI:$TAG_NIGHTLY" \ + "$IMAGE_GUI:$TAG_NIGHTLY" \ + "$IMAGE_ALL:$TAG_NIGHTLY" + + elif [[ $TYPE == "release" ]]; then + docker tag "$BUILD_CLI" "$IMAGE_CLI:$TAG_RELEASE" + docker tag "$BUILD_CLI" "$IMAGE_CLI:$TAG_LATEST" + docker tag "$BUILD_ALL" "$IMAGE_GUI:$TAG_RELEASE" # gui-version is now only a symlink to all-version + docker tag "$BUILD_ALL" "$IMAGE_GUI:$TAG_LATEST" # gui-version is now only a symlink to all-version + docker tag "$BUILD_ALL" "$IMAGE_ALL:$TAG_RELEASE" + docker tag "$BUILD_ALL" "$IMAGE_ALL:$TAG_LATEST" + + docker save -o images.tar \ + "$IMAGE_CLI:$TAG_RELEASE" "$IMAGE_CLI:$TAG_LATEST" \ + "$IMAGE_GUI:$TAG_RELEASE" "$IMAGE_GUI:$TAG_LATEST" \ + "$IMAGE_ALL:$TAG_RELEASE" "$IMAGE_ALL:$TAG_LATEST" + fi + + set +e +} + diff --git a/extra/scripts/CD/packaging/package-sources.sh b/extra/scripts/CD/packaging/package-sources.sh new file mode 100644 index 0000000000..b43af279e7 --- /dev/null +++ b/extra/scripts/CD/packaging/package-sources.sh @@ -0,0 +1,14 @@ +before () { + apk add --no-cache git gnupg + + gpg_import +} + +package () { + set -e + + # just sign + gpg --armor --detach-sig -o $ARCHIVE.sig $ARCHIVE + + set +e +} diff --git a/extra/scripts/CD/packaging/package.sh b/extra/scripts/CD/packaging/package.sh new file mode 100755 index 0000000000..e93ab046d2 --- /dev/null +++ b/extra/scripts/CD/packaging/package.sh @@ -0,0 +1,129 @@ +#!/bin/sh + +usage_and_exit () { + cat >&2 << EOF +Usage: $0 TYPE TARGET CWD + +CD package script wrapper. + +Arguments: + TYPE Either 'nightly' or 'release'. + TARGET Target distro. Determines script to be called. + CWD Working directory. + +EOF + exit 1 +} + +usage_and_exit_with_msg () { + echo Error: "$1" >&2 + echo "" >&2 + usage_and_exit +} + +exit_with_msg () { + echo Error: "$1" >&2 + exit $2 +} + +tolower () { + echo "$1" | tr '[:upper:]' '[:lower:]' +} + +function_exists () { + type "$1" >/dev/null 2>/dev/null +} + +# --------------------------------------------------------------------------------------------------------------------- + +gpg_import () { + if [[ -x /usr/libexec/gpg-preset-passphrase ]]; then + GPGPP=/usr/libexec/gpg-preset-passphrase + elif [[ -x /usr/lib/gnupg/gpg-preset-passphrase ]]; then + GPGPP=/usr/lib/gnupg/gpg-preset-passphrase + else + echo "gpg-preset-passphrase not found" + exit 1 + fi + + # initialize agent + mkdir -p ~/.gnupg + echo -ne 'allow-preset-passphrase\nmax-cache-ttl 60480000\ndefault-cache-ttl 60480000\n' > ~/.gnupg/gpg-agent.conf + gpg-connect-agent reloadagent /bye + + # import key + echo "$GPG_PRIVATE_KEY" | gpg --no-tty --batch --import - + KEYGRIP=$(gpg --with-keygrip -k $GPG_KEYID | grep Keygrip | cut -d= -f2 | tr -d ' ') + $GPGPP --preset --passphrase "$GPG_PASSPHRASE" "$KEYGRIP" +} + +# --------------------------------------------------------------------------------------------------------------------- + +SCRIPT_ROOT=$(realpath $(dirname $0)) +GIT_ROOT=$(realpath "$SCRIPT_ROOT"/../../../../) +TYPE= +TARGET= +CWD= + +# --------------------------------------------------------------------------------------------------------------------- +# check input +[[ $# -ne 3 ]] && usage_and_exit + +TYPE=$(tolower "$1") +[[ $TYPE != "nightly" && $TYPE != "release" ]] && usage_and_exit_with_msg "Invalid type." + +TARGET=$(tolower "$2") +DISTROSCRIPT="$SCRIPT_ROOT/package-$TARGET.sh" +[[ ! -e "$DISTROSCRIPT" ]] && usage_and_exit_with_msg "Invalid distro. File $(basename $DISTROSCRIPT) not found." + +CWD=$(realpath "$3") +[[ ! -d "$3" ]] && usage_and_exit_with_msg "Directory $3 does not exist". + +# --------------------------------------------------------------------------------------------------------------------- +# source distroscript + +echo "Sourcing '$DISTROSCRIPT'" +source "$DISTROSCRIPT" + +function_exists "package" || exit_with_msg "Function package() not found in package-$TARGET.sh" 2 +function_exists "before" || exit_with_msg "Function before() not found in package-$TARGET.sh" 2 + +# --------------------------------------------------------------------------------------------------------------------- +# run before... + +before + +# --------------------------------------------------------------------------------------------------------------------- +# create archive inside $CWD + +cd $GIT_ROOT + +if [[ $TYPE == "nightly" ]]; then + ARCHIVE_DIR=algorithms-library_$(git describe --long --tags --match="v*" | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g') + ARCHIVE=$ARCHIVE_DIR.tar.gz + echo "Creating archive $ARCHIVE" + + git archive --format=tar.gz --prefix="$ARCHIVE_DIR/" HEAD -o $CWD/$ARCHIVE +elif [[ $TYPE == "release" ]]; then + ARCHIVE_DIR=algorithms-library_$(git describe --tags --match="v*" --abbrev=0 | sed 's/^v//g').tar.gz + ARCHIVE=$ARCHIVE_DIR.tar.gz + echo "Creating archive $ARCHIVE" + + git archive --format=tar.gz --prefix="$ARCHIVE_DIR/" $(git describe --tags --match="v*" --abbrev=0) -o $CWD/$ARCHIVE +fi + +# --------------------------------------------------------------------------------------------------------------------- +# output what is going on + +echo "Start packaging..." +echo " - \$TARGET : $TARGET" +echo " - \$TYPE : $TYPE" +echo " - \$GIT_ROOT: $GIT_ROOT" +echo " - \$CWD : $CWD" +echo " - \$ARCHIVE : $ARCHIVE" + +# change directory and run +cd $CWD +package + +exit $? diff --git a/extra/scripts/build.sh b/extra/scripts/build.sh index 9bea31a158..aa8254b4a4 100755 --- a/extra/scripts/build.sh +++ b/extra/scripts/build.sh @@ -4,6 +4,7 @@ BUILD_THREADS="$(grep -c processor /proc/cpuinfo)" BUILD_DIR= BUILD_MODE= BUILD_SYSTEM= +RUN_TESTS=1 is_installed() { FOUND=0 @@ -46,6 +47,7 @@ Arguments: -d DIRECTORY Build directory (relative to ALT root) -m MODE 'Release' or 'Debug' -j THREADS Number of build threads. Default: Number of CPU cores ($(grep -c processor /proc/cpuinfo)) + -n Do not run tests EOF exit 1 @@ -71,7 +73,7 @@ if [[ $# -ge 1 ]] && [[ $1 == "release" || $1 == "debug" ]]; then shift fi -while getopts "d:m:j:" arg; do +while getopts "d:m:j:n" arg; do case "$arg" in d) [[ ! -z $ARGS_SHORTCUT ]] && usage_and_exit @@ -100,6 +102,12 @@ while getopts "d:m:j:" arg; do fi ;; + n) + [[ ! -z $ARGS_SHORTCUT ]] && usage_and_exit + + RUN_TESTS=0 + ;; + ?) usage_and_exit ;; @@ -126,6 +134,7 @@ Building Algorithms Library Core from $ROOT: - Into: $ROOT/$BUILD_DIR $BUILD_DIR_SYMLINK - Threads: $BUILD_THREADS - Builder: $BUILD_SYSTEM + - Testing: $RUN_TESTS ------------------------------------------------------------------------------ EOF # generate cmakefiles @@ -157,9 +166,13 @@ cmake $CMAKE_FLAGS ${ROOT} # build and test if [[ $BUILD_SYSTEM == "ninja" ]]; then ninja -j${BUILD_THREADS} || exit 1 - ctest . --output-on-failure -j ${BUILD_THREADS} || exit 1 + if [[ $RUN_TESTS -ne 0 ]]; then + ctest . --output-on-failure -j ${BUILD_THREADS} || exit 1 + fi else make -j${BUILD_THREADS} || exit 1 - make test ARGS="-j${BUILD_THREADS} --output-on-failure" || exit 1 + if [[ $RUN_TESTS -ne 0 ]]; then + make test ARGS="-j${BUILD_THREADS} --output-on-failure" || exit 1 + fi fi -- GitLab