From 50cb23a4c740cf462dc409e1ed560db5cbde38ef Mon Sep 17 00:00:00 2001 From: Alec Turner Date: Mon, 11 Mar 2024 16:13:56 -0700 Subject: [PATCH] Packaging improvements - compress images with xz by default (and decompress on install) - add help menu and command line argument parsing to package.sh - display packaging status if `pv` command available --- README.md | 15 +------ install.sh | 4 ++ package.sh | 118 ++++++++++++++++++++++++++++++++++++++++------------- 3 files changed, 95 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 28ef1d74..78322149 100644 --- a/README.md +++ b/README.md @@ -7,20 +7,7 @@ Packaging and installation tools for Streamline's "Streamline EMR" application. 2. run `package.sh` to generate a new application package 3. package will be stored to `build-packages/` -### Alternative Builds -Some variables can be overridden to support alternative builds. For example, an option exists to package the application WITHOUT docker images. This can be done by setting the appropriate environment variables (e.g. `VAR=VALUE ./package.sh`). - -Options that can be overridden are: -- `PACKAGE_NAME`: base name of the output package. (default: `streamline-emr`) -- `OUTPUT_DIR`: directory where packages will be stored. (default: `$PROJECT_ROOT/build-packages`) -- `PACKAGE_IMAGES`: if set to a value other than 1, do NOT package images. (default: `1`) -- `IMAGE_TAG`: if set, used to specify a source image tag. (default: uses git-based version name) -- `IMAGES_LIST`: if set, specifies an alternate image list file. (default: `$PROJECT_ROOT/images.csv`) - -For example, to build using the absolute latest images available: -```bash -IMAGE_TAG=latest package.sh -``` +Several defaults can be overridden with command line options or by setting appropriate environment variables. A help prompt can be accessed using `package.sh -h`. For example, `package.sh -t latest -p linux/arm64` will build only the `linux/arm64` platform and use the `latest` tag instead of the current repo version. ## Installing 1. extract the generated archive file in a temporary folder on the target system diff --git a/install.sh b/install.sh index a6fddf34..1027c9c4 100755 --- a/install.sh +++ b/install.sh @@ -41,6 +41,10 @@ for f in $(find $LOCATION/images -name "*.img" 2>/dev/null);do echo "importing docker image: $f" docker load -i $f done +for f in $(find $LOCATION/images -name "*.img.xz" 2>/dev/null);do + echo "importing docker image: $f" + xz -d -c $f | docker load +done # install systemd service unless running in a docker environment if [ ! -f /.dockerenv ]; then diff --git a/package.sh b/package.sh index 16070624..afa0dcb5 100755 --- a/package.sh +++ b/package.sh @@ -31,13 +31,80 @@ fi PACKAGE_NAME=${PACKAGE_NAME-"streamline-emr"} OUTPUT_DIR=${OUTPUT_DIR-"$PROJECT_ROOT/build-packages"} PACKAGE_IMAGES=${PACKAGE_IMAGES:-1} +COMPRESS_IMAGES=${COMPRESS_IMAGES:-1} IMAGE_TAG=${IMAGE_TAG:-${VERSION_NAME}} IMAGES_LIST=${IMAGES_LIST:-${PROJECT_ROOT}/images.csv} + +function print_usage { + echo "Kenya EMR packaging utility" + echo + echo "Usage: package.sh [ OPTIONS ]" + echo + echo "Options:" + echo " -h Print this help prompt and exit" + echo " -p PLATFORM Specify a docker platform to target, may be passed multiple" + echo " times. If none specified, all platforms built. Supported" + echo " platforms: 'linux/amd64', 'linux/arm64'" + echo " -t IMAGE_TAG Specify an image tag. It not specified, the tag will be" + echo " inferred from the current git version ('${IMAGE_TAG}')" + echo " -n PACKAGE_NAME Specify the package base name. Default: ${PACKAGE_NAME}" + echo " -o OUTPUT_DIR Specify a directory to save packages. Default: ${OUTPUT_DIR}" + echo " -l IMAGES_LIST Specify a file containing a list of images. Default: ${IMAGES_LIST}" + echo " -I Do not package images" + echo " -C Do not compress images" + echo +} + +# collect user args +PLATFORMS=() +while getopts ":p:t:n:o:l:ICh" options; do + case "${options}" in + p) + PLATFORMS+=(${OPTARG}) + ;; + t) + IMAGE_TAG=${OPTARG} + ;; + n) + PACKAGE_NAME=${OPTARG} + ;; + o) + OUTPUT_DIR=${OPTARG} + ;; + l) + IMAGES_LIST=${OPTARG} + ;; + I) + PACKAGE_IMAGES=0 + ;; + C) + COMPRESS_IMAGES=0 + ;; + h) + print_usage + exit 0 + ;; + *) + print_usage + exit 1 + ;; + esac +done +shift $((OPTIND -1)) + +# set defaults +if [ ${#PLATFORMS} -eq 0 ]; then + PLATFORMS=("linux/amd64" "linux/arm64") +fi + echo "packaging $PACKAGE_NAME" echo " output dir: $OUTPUT_DIR" echo " package images: $PACKAGE_IMAGES" +echo " compress images: $COMPRESS_IMAGES" echo " image tag: $IMAGE_TAG" +echo " image list: $IMAGES_LIST" +echo " platforms: ${PLATFORMS[@]}" # work from the project root pushd $PROJECT_ROOT > /dev/null @@ -46,28 +113,8 @@ pushd $PROJECT_ROOT > /dev/null set -e trap 'echo "Error: Last command failed with exit code $?."' ERR -# generate version name -# use ci tag name if available (e.g. "v1.2") -if [ $CI_COMMIT_TAG ]; then - VERSION_NAME="$CI_COMMIT_TAG" - -# otherwise if ci, use BRANCH-COMMIT -# Note: commit count is harder to get to from CI - can't reproduce the git -# describe functionality as easily -elif [ $CI_COMMIT_BRANCH ]; then - VERSION_NAME="$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA" - -# if not ci, try to use git -elif command -v git &> /dev/null; then - VERSION_NAME=$(git describe --always --tags --dirty) - -# fallback -else - VERSION_NAME="unknown" -fi - # build for each relevant platform -for platform in "linux/amd64" "linux/arm64"; do +for platform in ${PLATFORMS[@]}; do echo "preparing package for platform $platform" if [ "$platform" == "linux/amd64" ];then pkg_arch="x86_64" @@ -101,9 +148,8 @@ for platform in "linux/amd64" "linux/arm64"; do cp -a ${PROJECT_ROOT}/root ${PACKAGE_ROOT}/ echo "${VERSION_NAME}" > ${PACKAGE_ROOT}/root/var/signalytic/clientapps/streamline/streamline-emr/version - if [ "${PACKAGE_IMAGES}" = "1" ]; then + if [ "${PACKAGE_IMAGES}" = 1 ]; then # add images - i=0 # read image list from file (e.g. 'images-aarch64') # images are listed as SAVED_TAG=SOURCE_IMAGE for image in $(cat ${IMAGES_LIST});do @@ -112,13 +158,29 @@ for platform in "linux/amd64" "linux/arm64"; do if [ "${img_platform}" = "${platform}" ]; then echo "pulling image '${source}' (${platform})" docker pull --platform ${platform} ${source}:${IMAGE_TAG} + # re-tag the image with a more generic name echo "tagging image as '${dest}:latest'" docker tag ${source}:${IMAGE_TAG} ${dest}:latest - i=$((i+1)) - name="image${i}.img" - path="${IMAGES_DIR}/${name}" - echo "saving to '${path}'" - docker save -o ${path} ${dest}:latest + # save the image to file + if [ "${COMPRESS_IMAGES}" = 1 ]; then + path="${IMAGES_DIR}/${dest}.img.xz" + echo "saving to '${path}'" + # if installed, use pv to show progress + if command -v pv &> /dev/null; then + docker save ${dest}:latest | xz -9 - -c | pv -t -r -b -N "saving compressed image..." > ${path} + else + docker save ${dest}:latest | xz -9 - -c > ${path} + fi + else + path="${IMAGES_DIR}/${dest}.img" + echo "saving to '${path}'" + # if installed, use pv to show progress + if command -v pv &> /dev/null; then + docker save ${dest}:latest | pv -t -r -b -N "saving raw image..." > ${path} + else + docker save -o ${path} ${dest}:latest + fi + fi fi done else