OCI container creation at the LPC

containerize/Dockerfile

@@ -0,0 +1,56 @@
+ARG BUILDIMAGE
+FROM ${BUILDIMAGE} as builder
+
+ARG BUILD_DATE
+ARG ANALYSIS_NAME=analysis
+LABEL org.label-schema.build-date=$BUILD_DATE \
+      org.label-schema.name="$ANALYSIS_NAME Docker image" \
+      org.label-schema.description="Provides a runnable CMSSW image with $ANALYSIS_NAME pre-installed." \
+      org.label-schema.vendor="FNAL"
+
+ARG USER_BUILD=cmsusr
+ARG USERDIR_BUILD=/home/${USER_BUILD}
+USER ${USER_BUILD}
+WORKDIR ${USERDIR_BUILD}
+
+ARG CMSSW_VERSION
+ARG TAR
+
+COPY ${TAR} ${USERDIR_BUILD}/${CMSSW_VERSION}.tar.gz
+RUN pwd && \
+    ls -alh ./ && \
+    source /cvmfs/cms.cern.ch/cmsset_default.sh && \
+    echo "Unpacking ${CMSSW_VERSION}.tar.gz into ${PWD} ..." && \
+    tar -xzf ${CMSSW_VERSION}.tar.gz && \
+    rm ${CMSSW_VERSION}.tar.gz && \
+    ls -alh ./ && \
+    cd ${CMSSW_VERSION}/src/ && \
+    scram b ProjectRename && \
+    pwd && \
+    ls -alh ./ && \
+    echo "Setting the CMSSW environment ..." && \
+    eval `scramv1 runtime -sh` && \
+    echo "The CMSSW_BASE is ${CMSSW_BASE}" && \
+    cd ${HOME}
+
+# ---
+
+ARG BASEIMAGE
+FROM ${BASEIMAGE}
+
+ARG BUILD_DATE
+ARG ANALYSIS_NAME
+LABEL org.label-schema.build-date=$BUILD_DATE \
+      org.label-schema.name="$ANALYSIS_NAME Docker image" \
+      org.label-schema.description="Provides a runnable CMSSW image with $ANALYSIS_NAME pre-installed." \
+      org.label-schema.vendor="FNAL"
+
+ARG CMSSW_VERSION
+ARG USER_BUILD=cmsusr
+ARG NONPRIVILEGED_USER=cmsusr
+ARG USERDIR_BUILD=/home/${USER_BUILD}
+ARG USERDIR=/home/${NONPRIVILEGED_USER}
+WORKDIR ${USERDIR}
+
+COPY --from=builder --chown=cmsusr:cmsusr ${USERDIR_BUILD}/${CMSSW_VERSION} ${USERDIR}/${CMSSW_VERSION}
+RUN pwd && ls -alh ./

containerize/cache.json

@@ -0,0 +1,12 @@
+{
+	"Directories" : [
+		{
+			"Path" : "${CMSSW_BASE}/test",
+			"Cache" : 1
+		},
+		{
+			"Path" : "${CMSSW_BASE}/tmp",
+			"Cache" : 1
+		}
+	]
+}

containerize/containerize.sh

@@ -0,0 +1,180 @@
+#!/bin/bash -e
+
+CWD=${PWD}
+SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+BASE=
+KEEPCACHE="--exclude-caches-all"
+CLEAN="false"
+DIR="/home/cmsusr"
+FILE="Dockerfile"
+JSON="${SCRIPTPATH}/cache.json"
+ROOT="/scratch/containers/`whoami`/"
+TAG="analysis"
+TAR=""
+USER="cmsusr"
+VCS="--exclude-vcs"
+
+usage(){
+    EXIT=$1
+
+    echo "containerize.sh [options]"
+    echo ""
+    echo "-b [base]           the base image to use (default = ${BASE})"
+    echo "-c                  keep the cached files when making the tarball (default = False)"
+    echo "-C                  cleanup the temporary files when finished making the image (default = ${CLEAN})"
+    echo "-d [dir]            project installation area inside the container (default = ${DIR})"
+    echo "-f [file]           the Dockerfile to use to build the image (default = ${FILE})"
+    echo "-j [json]           path to the json file containing the path to cache (default = ${JSON})"
+    echo "-r [root]           change the 'root' and 'runroot' locations for buildah (default = ${ROOT})"
+    echo "-t [tag]            the tag to use for the image (default = ${TAG})"
+    echo "-T [tar]            path to an existing tarball to use (default = ${TAR})"
+    echo "-u [user]           override the default username in the container (default = ${USER})"
+    echo "-v                  include the vcs directories (default = False)"
+    echo "-h                  display this message and exit"
+    echo
+    echo "Examples:"
+    echo "./containerize.sh -t <name>:<tag> -b docker://docker.io/cmscloud/cc7-cms"
+    echo "podman run --rm -it -v /cvmfs/cms.cern.ch/:/cvmfs/cms.cern.ch/:ro <name>:<tag>"
+    echo "./containerize.sh -t <name>:<tag> -b docker://docker.io/aperloff/cms-cvmfs-docker:light -C -v"
+    echo "podman run --rm -it <name>:<tag>"
+
+    exit ${EXIT}
+}
+
+# process options
+while getopts "b:cCd:f:j:r:t:T:u:vh" opt; do
+    case "$opt" in
+    b) BASE=$OPTARG
+    ;;
+    c) KEEPCACHE=""
+    ;;
+    C) CLEAN="true"
+    ;;
+    d) DIR=$OPTARG
+    ;;
+    f) FILE=$OPTARG
+    ;;
+    j) JSON=$OPTARG
+    ;;
+    r) ROOT=$OPTARG
+    ;;
+    t) TAG=$OPTARG
+    ;;
+    T) TAR=$OPTARG
+    ;;
+    u) USER=$OPTARG
+    ;;
+    v) VCS=""
+    ;;
+    h) usage 0
+    ;;
+    esac
+done
+
+dependency_check() {
+    subuid=$(cat /etc/subuid | grep "^`id -u`:")
+    subgid=$(cat /etc/subgid | grep "^`id -u`:")
+    if [ ! command -v buildah &> /dev/null ]; then
+        EXIT=$?
+        echo "Buildah could not be found!"
+        exit ${EXIT}
+    elif [ ! command -v jq &> /dev/null ];then
+        EXIT=$?
+        echo "jq could not be found!"
+        exit ${EXIT}
+    elif [[ ! -d ${ROOT} ]]; then
+        EXIT=$?
+        echo "The directory ${ROOT} does not exist and cannot be used for buildah's 'root' or 'runroot' directory."
+        exit ${EXIT}
+    elif [[ -z "${subuid}" ]] || [[ -z "${subgid}" ]]; then
+        echo "Unable to find a subuid or subgid for id=`id -u` in /etc/subuid and /etc/subgid."
+        echo "Contact user support or your sysadmin for further assistance."
+    elif [[ ! -d /cvmfs/cms.cern.ch ]] || [ ! cvmfs_config probe cms.cern.ch &> /dev/null ]; then
+        EXIT=$?
+        echo "/cvmfs/cms.cern.ch must be mounted on the host to proceed."
+        exit ${EXIT}
+    fi
+
+    if [ ! command -v podman &> /dev/null ]; then
+        echo "Podman could not be found. While this is not strictly necessary, you will not be able to create a container from the resulting image."
+    fi
+}
+dependency_check
+
+# cache unneeded files
+CACHEDIR=" \
+Signature: 8a477f597d28d172789f06886806bc55\n
+# This file is a cache directory tag.\n
+# For information about cache directory tags, see:\n
+#       http://www.brynosaurus.com/cachedir/
+"
+IFS=$'\n'
+list_of_cache_files=()
+# https://stackoverflow.com/questions/1955505/parsing-json-with-unix-tools
+for dir in $(jq -r '.Directories[] | .Path + " " + (.Cache|tostring)' ${JSON}); do
+    IFS=' '
+    dirarray=($dir)
+    path=${dirarray[0]}
+    path=`eval echo ${path}`
+    cache=${dirarray[1]}
+    cache_file=${path}/CACHEDIR.TAG
+    if [[ "${cache}" == "1" ]] && [[ ! -f ${cache_file} ]]; then
+        echo -e "Cache ${path}"
+        echo ${CACHEDIR} > ${cache_file}
+        list_of_cache_files=(${list_of_cache_files[@]} ${cache_file})
+    elif [[ "${cache}" == "1" ]] && [[ -f ${cache_file} ]]; then
+        echo -e "Already cached ${path}"
+    elif [[ "${cache}" == "0" ]] && [[ -f ${cache_file} ]]; then
+        echo -e "Uncache ${path}"
+        rm ${cache_file}
+    elif [[ "${cache}" == "0" ]] && [[ ! -f ${cache_file} ]]; then
+        echo "Already uncached ${path}"
+    fi
+done
+
+# tarball of CMSSW area
+if [ -z "${TAR}" ]; then
+    echo -e "Making the ${CMSSW_VERSION} tarball ... "
+    cd ${CMSSW_BASE}/..
+    tar ${KEEPCACHE} ${VCS} -zcf ${CMSSW_VERSION}.tar.gz -C ${CMSSW_BASE}/.. ${CMSSW_VERSION}
+    TAR="${CMSSW_VERSION}.tar.gz"
+fi
+
+# show the tarball
+if [ -e "${TAR}" ]; then
+    ls -lth ${TAR}
+fi
+
+# select the correct build image based on the SCRAM_ARCH of the CMSSW release
+ARCH=${SCRAM_ARCH%%_*}
+ARCH_VER=${ARCH: -1}
+BUILDIMAGE=docker://docker.io/cmscloud/
+if [[ "${ARCH_VER}" == "5" ]] || [[ "${ARCH_VER}" == "6" ]]; then
+    BUILDIMAGE=${BUILDIMAGE}slc${ARCH_VER}-cms
+elif [[ "${ARCH_VER}" == "7" ]]; then
+    BUILDIMAGE=${BUILDIMAGE}cc${ARCH_VER}-cms
+else
+    echo -e "Unknown CMSSW architecture version (${SCRAM_ARCH} -- > ${ARCH} -- > ${ARCH_VER}). Defaulting to cc7."
+    BUILDIMAGE=${BUILDIMAGE}cc${ARCH_VER}-cms
+fi
+
+# build the image
+echo -e "Building the image ..."
+buildah --root ${ROOT} --runroot ${ROOT} bud -f ${FILE} -t ${TAG} -v /cvmfs/cms.cern.ch:/cvmfs/cms.cern.ch \
+        --build-arg BUILDIMAGE=${BUILDIMAGE} --build-arg BASEIMAGE=${BASE} --build-arg BUILD_DATE=`date -u +%Y-%m-%d` --build-arg ANALYSIS_NAME=${NAME} \
+        --build-arg CMSSW_VERSION=${CMSSW_VERSION} --build-arg TAR=`realpath --relative-to="${PWD}" "${TAR}"` --build-arg NONPRIVILEGED_USER=${USER}
+
+# Cleanup the temporary files
+if [[ "${CLEAN}" == "true" ]]; then
+    echo -e "Cleaning the temporary files made while building the images ..."
+    echo -e "\tRemoving the tarball ${TAR} ..."
+    rm ${TAR}
+    for f in "${list_of_cache_files[@]}"; do
+        echo -e "\tRemoving the cache file ${f} ..."
+        rm ${f}
+    done
+fi
+
+# Return to the original working directory
+cd ${CWD}