From dfc0ecf9665a9cce778e2027505bd7507c049d77 Mon Sep 17 00:00:00 2001 From: "kebler.net" Date: Tue, 17 Jan 2023 15:20:30 -0800 Subject: [PATCH] convert scripts to functions and create library of secondary functions. Enable calling those directly form build script/function more image scripts to /src folder --- rebuild => aliases | 1 + build | 103 ++++--- distros/alpine/init.sh | 6 - distros/ubuntu/packages | 1 - image-arch | 3 - image-distro | 4 - image-info | 12 - distros/alpine/packages => install | 0 lib/file.mod | 291 ++++++++++++++++++ lib/load.sh | 4 + lib/src/01-helpers | 30 ++ make-tag => lib/src/02-make-tag | 11 +- lib/src/image-info | 28 ++ push => lib/src/image-push | 11 +- lib/src/image-tag | 34 ++ lib/src/try | 96 ++++++ lib/src/usage | 18 ++ readme.md | 16 +- src/alpine/init.sh | 8 + src/alpine/packages | 1 + src/common/adduser.sh | 0 {distros => src}/common/info.sh | 3 +- .../common/common.sh => src/common/init.sh | 8 +- .../common-packages => src/common/packages | 0 {distros => src}/common/ucishell.sh | 0 src/common/user.sh | 170 ++++++++++ {distros => src}/debian/init.sh | 0 src/init.sh | 23 ++ {distros => src}/ubuntu/add-ppa.sh | 0 {distros => src}/ubuntu/init.sh | 2 +- src/ubuntu/packages | 2 + tag | 28 -- try | 62 ---- try.yml | 8 - 34 files changed, 804 insertions(+), 180 deletions(-) rename rebuild => aliases (82%) delete mode 100644 distros/alpine/init.sh delete mode 100644 distros/ubuntu/packages delete mode 100755 image-arch delete mode 100755 image-distro delete mode 100755 image-info rename distros/alpine/packages => install (100%) create mode 100644 lib/file.mod create mode 100755 lib/load.sh create mode 100755 lib/src/01-helpers rename make-tag => lib/src/02-make-tag (85%) create mode 100755 lib/src/image-info rename push => lib/src/image-push (93%) create mode 100755 lib/src/image-tag create mode 100755 lib/src/try create mode 100644 lib/src/usage create mode 100644 src/alpine/init.sh create mode 100644 src/alpine/packages create mode 100644 src/common/adduser.sh rename {distros => src}/common/info.sh (65%) rename distros/common/common.sh => src/common/init.sh (54%) rename distros/common/common-packages => src/common/packages (100%) rename {distros => src}/common/ucishell.sh (100%) create mode 100755 src/common/user.sh rename {distros => src}/debian/init.sh (100%) create mode 100644 src/init.sh rename {distros => src}/ubuntu/add-ppa.sh (100%) rename {distros => src}/ubuntu/init.sh (73%) create mode 100644 src/ubuntu/packages delete mode 100755 tag delete mode 100755 try delete mode 100644 try.yml diff --git a/rebuild b/aliases similarity index 82% rename from rebuild rename to aliases index 9c5608d..770cfdc 100755 --- a/rebuild +++ b/aliases @@ -6,6 +6,7 @@ # with -n option (prefered) ./build -n "$@" +alias rebuild="build -nfunction_list" # as export #export NO_CACHE=true diff --git a/build b/build index 9fddc33..aba2dde 100755 --- a/build +++ b/build @@ -1,46 +1,54 @@ #!/bin/bash -declare targets=(dev arm amd deploy private multi) + +docker_image_build () { + +local targets=(dev arm amd deploy private multi) +local verbose declare OPTION; declare OPTARG; declare OPTIND SDIR=$(pwd) -BDIR=$(dirname "$(realpath "$0")") +BDIR=$(dirname "$(realpath "$BASH_SOURCE")") pushd $BDIR > /dev/null -if [ $1 == "try" ]; then - ./try $2 $SDIR - exit -fi +source $BDIR/lib/load.sh -usage() { # Function: Print a help message. - echo "Image Build Script: Creates one or more images using a target in the docker-bake.hcl file" - echo "USAGE: $0 buildtarget " - echo "valid targets: ${targets[*]}; default: dev" - echo no argument options: - echo "-c try out the image by starting a container terminal therein, for dev target this is the default;" - echo "-x exclude distro from image name;" - echo "-n for --no_cache" - echo "-p push to repository; after build push to repository default is hub.docker.common (not need for deploy target)" - echo required argument options: - echo "-d supported: alpine, debian, ubuntu, default: alpine; if base image set distro will be determined" - echo "-t tag following : in output image name (i.e. REPO/USER/NAME:TAG), default: latest" - echo "-u ; repository user prefix in output image name (i.e. REPO/USER/NAME:TAG)" - echo "-r ; private repo name, do not use for hub.docker.com (docker.io)" - echo "-b ; used in FROM in Dockerfile, default is official distro image (e.g. alpine:latest)" - echo "-w ; set a custom WORKDIR in Dockerfile, default is /opt/build" -} +case "$1" in + try) + shift 1 + try_container "$@" + return $? + ;; + tag) + shift 1 + image_tag "$@" + return $? + ;; + info) + shift 1 + [[ $1 == "arch" ]] && { shift 1; image_arch "$@"; return $?; } + [[ $1 == "exists" ]] && { shift 1; image_exists "$@"; return $?; } + [[ $1 == "id" ]] && { shift 1; image_id "$@"; return $?; } + image_info "$@"; return $? + ;; +esac exit_abnormal() { # Function: Exit with error. usage - exit ${1:-1} + return ${1:-1} } -source $BDIR/helpers.lib - scripts_dir=$SDIR/src +[[ -z "$PS1" ]] || no_prompt=true -while getopts ':b:d:t:ncr:u:pxhs:w:ak' OPTION; do +while getopts ':b:d:t:ncr:u:pxhs:w:akvo' OPTION; do # echo processing: option:$OPTION argument:$OPTARG index:$OPTIND remaining:${@:$OPTIND} case "$OPTION" in + o) + overwrite=true + ;; + v) + verbose=true + ;; a) # automated - script is to be run without prompt (non-interactive) no_prompt=true @@ -105,15 +113,15 @@ done shift $((OPTIND - 1)) - - target=${1:-dev} LINUX_DISTRO=${LINUX_DISTRO:-alpine} name=$2 RUSER=${3:-$RUSER} - IMAGE_NAME=$([[ $RUSER ]] && echo ${RUSER}/)${name}$([[ (! $exclude_distro) && $name ]] && echo "-")$([[ ! $exclude_distro ]] && echo ${LINUX_DISTRO}) +# TODO writing to existing tag untags existing image so write a new tag to that image then continue +# [[ $(image_exists $IMAGE_NAME) ]] && [[ ! $overwrite ]] && { echo local image \'$(make_tag $IMAGE_NAME)\' exists use -o to overwrite; return 1; } + if [[ $BASE_IMAGE ]]; then echo determining DISTRO of base image: $BASE_IMAGE LINUX_DISTRO=$(get_distro -d $BASE_IMAGE) @@ -132,18 +140,23 @@ export LINUX_DISTRO export SCRIPTS export KEEP -echo " ******************************************" -echo "Building with base image: $BASE_IMAGE, output image name => $IMAGE_NAME" +echo "******************************************" +echo "Building with base image: $BASE_IMAGE" +echo "Outputing to image name => $IMAGE_NAME" echo "Linux Distro: $LINUX_DISTRO" echo "Using build target: $target" -docker buildx bake --print $target -echo -e "\n---------------------------------" -echo "build scripts at $scripts_dir to be copied to ${SCRIPTS:-/opt/build} in container ***** " -ls -la $scripts_dir -echo -e "\n----- base init script init.sh ------" -cat $scripts_dir/init.sh -echo -e "\n---------------------------------" +if [[ $verbose ]]; then + echo -e "\n---------------------------------" + docker buildx bake --print $target + echo -e "\n---------------------------------" + echo "build scripts at $scripts_dir to be copied to ${SCRIPTS:-/opt/build} in container ***** " + ls -la $scripts_dir + echo -e "\n----- base init script init.sh ------" + cat $scripts_dir/init.sh + echo -e "\n---------------------------------" +fi echo -e "\n***************************************" + if [[ ! $no_prompt ]]; then read -n 1 -p "do you want to continue [y]=>" REPLY [[ $REPLY != "y" ]] && echo -e "\n" && exit 0 @@ -186,7 +199,15 @@ if [[ $target == "private" ]]; then fi fi -[[ ($try || $target == "dev") ]] && ./try $([[ $target == "deploy" ]] && echo -p) $IMAGE_NAME +if [[ ($try || $target == "dev") ]] && [[ ! $no_prompt ]]; then + echo trying newly built image in a container + try_container $([[ $target == "deploy" ]] && echo -p) $IMAGE_NAME +fi popd > /dev/null -#echo reset to calling directory $PWD \ No newline at end of file +#echo reset to calling directory $PWD + +} + +# if script was executed then call the function +(return 0 2>/dev/null) || docker_image_build $@ \ No newline at end of file diff --git a/distros/alpine/init.sh b/distros/alpine/init.sh deleted file mode 100644 index 78fd398..0000000 --- a/distros/alpine/init.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -echo alpine distro uci script -apk update; apk upgrade; -echo ">>>> installing packages => $(cat packages) $(cat common-packages)" -apk add --no-cache bash bash-completion $(cat packages) $(cat common-packages) -# apk add bindfs --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing diff --git a/distros/ubuntu/packages b/distros/ubuntu/packages deleted file mode 100644 index a7506d6..0000000 --- a/distros/ubuntu/packages +++ /dev/null @@ -1 +0,0 @@ -gpg \ No newline at end of file diff --git a/image-arch b/image-arch deleted file mode 100755 index 226b873..0000000 --- a/image-arch +++ /dev/null @@ -1,3 +0,0 @@ -tag=$(./make-tag $@) -info=$(docker image inspect $tag) -echo $info | jq '.[] | .Architecture' diff --git a/image-distro b/image-distro deleted file mode 100755 index 9ce1728..0000000 --- a/image-distro +++ /dev/null @@ -1,4 +0,0 @@ -docker run -it $1 /bin/sh -c "cat /etc/os-release" > /tmp/container-os.tmp 2> /dev/null -. /tmp/container-os.tmp -echo $ID -rm /tmp/container-os.tmp diff --git a/image-info b/image-info deleted file mode 100755 index 18797a3..0000000 --- a/image-info +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -[[ $1 == "-k" ]] && key=$2 && shift 2 -tag=$(./make-tag $@) -info=$(docker image inspect $tag 2> /dev/null) || info=$(docker image inspect $1) || -exit -if [[ $key ]]; then -# echo image: $tag, key:$key -echo $info | jq --arg k "$key" '.[] | .[$k]' -else -# quote to preserve newlines -echo "$info" -fi diff --git a/distros/alpine/packages b/install similarity index 100% rename from distros/alpine/packages rename to install diff --git a/lib/file.mod b/lib/file.mod new file mode 100644 index 0000000..26da0dd --- /dev/null +++ b/lib/file.mod @@ -0,0 +1,291 @@ +#!/bin/bash +# TODO allow debug to have managment flags and levels +function debug () { +[[ $BASH_DEBUG ]] && echo -e "#### DEBUG ####\n $@ \n#####" >&2 +} + +# alias debug_on="sudo -i uncomment BASH_DEBUG /etc/bash.bashrc" +# alias debug_off="sudo -i comment BASH_DEBUG /etc/bash.bashrc" + + +# *************** DEBUGGING *********************** +# module_load debug +# if [[ $? -ne 0 ]]; then +# echo "unable to load a 'debug' module using a noop for debug function" +# # noop +# function debug () { +# : +# } +# fi + +# [[ ! $- == *i* ]] && exec >> ~/logs/load.log && exec 2>&1 +# export BASH_DEBUG=true + +# uncomment for debugging +# echo $USER running load script in $DIR +# echo callers +# caller +# echo $(ps -o comm= $PPID) +# echo ----- +# echo $BASH_SHELL_DIRS +# ******************END DEBUGGING ******************************* + +# ***************** LOGGING ***************** +# function nilog () { +# [[ ! $- == *i* ]] && echo -e "-----\n $USER $*" &>> ~/logs/load.log +# } + +# nilog "called load.sh $(date)" +# nilog caller: $(caller) +# nilog pid: $(ps -o comm= $PPID) + +# * +#!/bin/bash +iecho () { +[[ ! -z "$PS1" ]] && echo $1 +} + +# no way to tell if sourced vs subshell +# # [[ ! -z "$PS1" ]] && [[ $SHLVL -eq 1 ]] && echo $1 +# # echo [[ -z $PS1 ]] +# echo $SHLVL +# echo $$ +# echo $BASHPID +# if [[ -t 1 ]]; then +# echo "Terminal" +# else +# echo "Not-a-terminal" +# fi +#!/bin/bash + +# export BASH_DEBUG=true + +isFile() { + if [[ -f $1 ]] + then + echo "true" + return 0 + else + return 1 + fi +} + +build_file () { + # local file + # echo build file $1 $2 + # todo merge no cr + [[ $1 == "-f" ]] && file=true && shift 1 + [[ -f "$2" ]] || { iecho "output file $2 does not exist"; return 1; } + if [[ -f "$1" ]]; then + # echo adding file $1 to $2 + [[ $file ]] && echo -e "\n####### ADDED $1 ########" >> $2 + # remove comment lines, remove blank last line + cat "$1" | sed '/^\s*#/d' | sed '/^$/{:a;N;s/\n$//;ta}' >> $2 + [[ ! $file ]] && echo -e "\n" >> $2 + else iecho "no such file $1 to append to $2" + fi +} + +# TODO need to test +function lines_2_str () { + [[ ! -f "$1" ]] && return 1 + local str='' + # echo the lines: $lines >&2 + # lines="$(cat "$1")" + while read line; do + [[ "$line" =~ ^[[:space:]]*# ]] && continue + # echo line: "${line}" >&2 + str="$str ${line}" + # echo str: $str >&2 + done < $1 + # for line in $lines ; do + # str+='"'$line'" ' + # done + echo $str +} + +# find, superceeds find use `command find` to get the super +function _find () { +# USAGE +# all option arguments that contain globs/wildcards must be quoted to avoid expansion +# f sets path and file excludes from a supplied file path +# all lines ending in / will be treated as directory names to ignore, otherwise files +# p option explictly excludes paths(directories) +# d option sets the directory depth which is current directy by default, 0 is all +# x excitly excludes file globs as a string +# n inclucdes only file globs otherwise it's all except .files +# if no directory is given it will attempt to source the present working directory +# example: +# source_dir -p "archive" -x '"*.off" "*.md"' -d 0 # $DIR/$SUBDIR +local EXCLUDE_FILE +local PATHS +local NAMES +local ENAMES +local DEPTH=1 +local HIDDEN + +declare OPTION +declare OPTARG +declare OPTIND +while getopts 't:p:d:e:n:f:h' OPTION; do +case "$OPTION" in + t) + TYPE=$OPTARG + # echo "TYPE $TYPE" + ;; + f) + EXCLUDE_FILE=$OPTARG + # echo EXCLUDE FILE $EXCLUDE_FILE >&2 + ;; + p) + # PATHS=("$OPTARG") + IFS=',' read -r -a PATHS <<< "$OPTARG" + # echo EXCLUDING THESE PATHS ${PATHS[*]} + ;; + e) + IFS=',' read -r -a ENAMES <<< "${OPTARG}" + # echo EXCLUDING THESE FILE NAMES ${ENAMES[*]} + ;; + n) + # NAMES=("$OPTARG") + IFS=',' read -r -a NAMES <<< "${OPTARG}" + # NAMES=$OPTARG + ;; + d) + DEPTH=$OPTARG + # echo "SOURCING TO DEPTH (0=any)" "$DEPTH" + ;; + h) + HIDDEN=true + # echo "SOURCING TO DEPTH (0=any)" "$DEPTH" + ;; + *) + echo unknown option $OPTION + ;; +esac +done + +shift $(( OPTIND - 1 )) + +local DIR +DIR="$*" +if [ ! "$DIR" ]; then + if [ -v PS1 ]; then + echo no directory provided to search + echo searching present working directory $(pwd) + read -p "Do you want to continue? " -n 1 -r + [[ $REPLY =~ ^[Yy]$ ]] && DIR=$(pwd) || return 1 + else + return 1 + fi +fi + +[ ! -d "$DIR" ] && echo " directory $DIR does not exist, aborting" && return 1 + +# echo dir $DIR + +local FIND +FIND="command find $DIR" +FIND+=$([ ! $DEPTH == 0 ] && echo " -maxdepth $DEPTH ") +# FIND+=" -type $([ $TYPE ] && echo "$TYPE" || echo "f")" +TYPE=${TYPE:-f} +FIND+=" -type $TYPE " +# include HIDDEN files and directories IS FALSE BY DEFULT +[[ ! $HIDDEN ]] && FIND+="! -path \"*/.*/*\" ! -name \".*\" " + +local name +local path + +if [[ -f $EXCLUDE_FILE ]]; then + local ignores=$(lines_2_str "$EXCLUDE_FILE") + # echo ignores: $ignores >&2 + for exclude in $ignores ; do + # echo exclude: ${exclude} >&2 + [[ "$exclude" == */ ]] && PATHS+=("${exclude::-1}") || ENAMES+=("$exclude") + done +fi + +# echo paths ${PATHS[@]} >&2 +# echo exclude names ${ENAMES[@]} >&2 + +set -o noglob + +if [[ ${PATHS[0]} ]]; then + for path in ${PATHS[@]}; do + # echo excluding $path + FIND+=$(echo ' ! -path "*/'$path'/*"') + done +fi + +if [[ ${ENAMES[0]} ]]; then + for name in ${ENAMES[@]}; do + debug excluding name "$name" + FIND+=$(echo ' ! -name '$name'') + done +fi + +debug "INCLUDING ONLY THESE FILE NAMES ${NAMES[*]}" +if [[ ${NAMES[0]} ]]; then + for name in "${NAMES[@]}"; do + debug only finding $name + FIND+=$(echo " -name '${name}'") + done +fi + +# if [[ $NAMES ]]; then +# debug names for find command $NAMES +# for name in $NAMES; do +# debug "xxonly finding '$name'" +# FIND+=$(set -o noglob;echo " -name '${name}'") +# done +# fi + + +# echo +# echo find dir: $DIR >&2 +debug "find command: $FIND" +set +o noglob +local FILES +FILES=$(eval $FIND | sort) +[[ $FILES ]] && echo $FILES +return 0 +} + +source_dir () { + # echo passed: $* + debug function: source_dir + local FILES + FILES=$(_find "$@") # find function + # echo $FILES >&2 + [[ $? -ne 0 ]] && return 1 + for f in $FILES; do + # echo sourcing: $f >&2 + source "$f" + done + +} + +prepend_file () { +# ---------------------------------------------------------------------------------------------------------------------- +# usage prepend_file +# ---------------------------------------------------------------------------------------------------------------------- +# Prepend the contents of [$1], to [$2], leaving the result in [$2]. +# insert a newline at the end of [$1] if necessary +# ---------------------------------------------------------------------------------------------------------------------- + +# check # echo $1 $2 +[[ -f $1 ]] || return 1 +[[ -f $2 ]] || return 2 +# init +tmp_fn=$( mktemp -t TEMP_FILE_prepend.XXXXXXXX ) +chmod 600 "$tmp_fn" +\cp $1 $tmp_fn +sed -i '$a\' $tmp_fn +cat $2 >> $tmp_fn +\mv "$tmp_fn" "$2" +# cleanup +rm -f "$tmp_fn" +return 0 + +# [End] +} diff --git a/lib/load.sh b/lib/load.sh new file mode 100755 index 0000000..2f6bbb3 --- /dev/null +++ b/lib/load.sh @@ -0,0 +1,4 @@ +declare libdir +libdir=$(dirname "$(realpath "$BASH_SOURCE")") +source $libdir/file.mod +source_dir $libdir/src \ No newline at end of file diff --git a/lib/src/01-helpers b/lib/src/01-helpers new file mode 100755 index 0000000..763508c --- /dev/null +++ b/lib/src/01-helpers @@ -0,0 +1,30 @@ +#!/bin/bash + +function get_distro() { +/bin/cp /etc/os-release /tmp/os-release.tmp +if [[ $1 == "-d" ]]; then +shift 1 +# docker run -it --name get_container_os --rm --entrypoint cat $1 /etc/os-release +docker create --name dummy $1 > /dev/null +docker cp -L dummy:/etc/os-release /tmp/os-release.tmp +docker rm -f dummy > /dev/null +# docker run -it --name get_container_os --rm --entrypoint cat $1 /etc/os-release > /tmp/container-os.tmp 2> /dev/null +shift 1 +fi +source /tmp/os-release.tmp +declare valid=${@:-"alpine debian ubuntu"} +# echo $ID $ID_LIKE +[[ "${valid}" =~ $ID ]] && echo $ID && return 0 +[[ "${valid}" =~ $ID_LIKE ]] && echo $ID_LIKE && return 0 +return 1 +} + +isAbsPath() { + if [[ "${1:0:1}" == / || "${1:0:2}" == ~[/a-z] ]] + then + echo "true" + return 0 + else + return 1 + fi +} \ No newline at end of file diff --git a/make-tag b/lib/src/02-make-tag similarity index 85% rename from make-tag rename to lib/src/02-make-tag index 9454040..68828f7 100755 --- a/make-tag +++ b/lib/src/02-make-tag @@ -1,8 +1,11 @@ #!/bin/bash + +make_tag () { + +local DIR DIR=$(cd "$(dirname "$BASH_SOURCE")" >/dev/null 2>&1 ; pwd -P ) -#tags an image and pushes it to a repository -# if not reposity is given will use docker.io and push to hub.docker.com +# generate a full image name with tag # $1 name, $2 user(or repo), $3 repo [[ $# -lt 1 ]] && echo "image base name required" && exit @@ -44,3 +47,7 @@ tag=$([[ $repo ]] && echo ${repo}/)$([[ $user ]] && echo ${user}/)$name$([[ $arm echo $tag +} + +# if script was executed then call the function +(return 0 2>/dev/null) || make_tag $@ \ No newline at end of file diff --git a/lib/src/image-info b/lib/src/image-info new file mode 100755 index 0000000..c44f125 --- /dev/null +++ b/lib/src/image-info @@ -0,0 +1,28 @@ +#!/bin/bash +image_info () { +[[ $1 == "-k" ]] && key=$2 && shift 2 +tag=$(make_tag "$@") +info=$(docker image inspect $tag 2> /dev/null) || info=$(docker image inspect $1 2> /dev/null) || return 1 +if [[ $key ]]; then +# echo image: $tag, key:$key +echo $info | jq --arg k "$key" '.[] | .[$k]' +else +# quote to preserve newlines +echo "$info" +fi +} + +image_exists () { + image_info -k RepoTags "$@" +} + +image_arch () { + image_info -k Architecture "$@" +} + +image_id () { + image_info -k Id "$@" | sed 's/.*\://' | sed 's/"//' +} + +# if script was executed then call the function +(return 0 2>/dev/null) || image_info $@ \ No newline at end of file diff --git a/push b/lib/src/image-push similarity index 93% rename from push rename to lib/src/image-push index 4547dc7..322c125 100755 --- a/push +++ b/lib/src/image-push @@ -1,4 +1,8 @@ #!/bin/bash + +image_push () { + +local DIR DIR=$(cd "$(dirname "$BASH_SOURCE")" >/dev/null 2>&1 ; pwd -P ) #tags an image and pushes it to a repository @@ -82,4 +86,9 @@ docker image rm $source2 > /dev/null 2>&1 fi echo removing tag $target -docker image rm $target > /dev/null 2>&1 \ No newline at end of file +docker image rm $target > /dev/null 2>&1 + +} + +# if script was executed then call the function +(return 0 2>/dev/null) || image_push $@ \ No newline at end of file diff --git a/lib/src/image-tag b/lib/src/image-tag new file mode 100755 index 0000000..26798b7 --- /dev/null +++ b/lib/src/image-tag @@ -0,0 +1,34 @@ +#!/bin/bash + +image_tag () { + +local DIR +DIR=$(cd "$(dirname "$BASH_SOURCE")" >/dev/null 2>&1 ; pwd -P ) + +# tags an image +# -d -f + +[[ $# -lt 1 ]] && echo "image base name required" && exit + + +[[ $1 == "-d" ]] && delete=true && shift 1 +[[ $1 == "-f" ]] && force=true && shift 1 +[[ $1 == "-i" ]] && { shift 1; id=$1; } || id=$(image_id $1) + +[[ ! $id ]] && { echo "no image with id $id $(make_tag "@")"; return 1; } + + +[[ $force ]] && docker rmi -f $id && return 0 + +if [[ $delete ]];then + docker rmi $id +else +docker tag $id $(make_tag $2) +fi +echo tags after operation for image $id +image_info -k RepoTags $id +} + + + + diff --git a/lib/src/try b/lib/src/try new file mode 100755 index 0000000..99c3b3c --- /dev/null +++ b/lib/src/try @@ -0,0 +1,96 @@ +#!/bin/bash + +# starts a trail container with passed image with a bash prompt +# $1 image name, $2 user +# user can be also prepended by using u option +# added tag is "latest" by default, use t option for alternate tag +# if p flag is used script will scrub any local image and attempt to download a deployed to docker image + +try_container () { + + declare -A arch=( ["x86_64"]="" ["aarch64"]="-arm64") + + [[ $# -lt 1 ]] && echo "image name required to try" && return 1 + + declare OPTION; declare OPTARG; declare OPTIND + OPTIND=0 + while getopts 'pt:u:' OPTION; do + # echo processing: option:$OPTION argument:$OPTARG index:$OPTIND remaining:${@:$OPTIND} + case "$OPTION" in + u) + ruser=$OPTARG + ;; + t) + TAG=$OPTARG + ;; + p) + PROD=$OPTARG + ;; + *) echo unknown run option -$OPTARG + echo "USAGE: try " + echo "available options: -t custom tag " + ;; + esac + done + + shift $((OPTIND - 1)) + + user=${2:-$ruser} + image=$([[ $user ]] && echo ${user}/)$1 + + # :${TAG:-latest} + + if [[ $PROD ]]; then + echo removing any local copy of image $image + docker image rm $image + host=prod + else + host=local + # TODO change this + image=${image/:/${arch[$(uname -p)]}:} + fi + + name=${image//\//-} + image=$image:${TAG:-latest} + + bmount () { + [[ $1 == "-u" ]] && umount=true && shift + declare mp=${1:-opt} + declare dir=${2:-${PWD}/_$mp} + vname=try-$name-$mp + + if [[ $umount ]]; then + echo removing volume .... + docker volume rm $vname + echo "volume $vname was removed" + echo "Do you want to also delete the bound volume directory $dir?" + read -n 1 -p "Warning: ALL changes in $mp directory in container will be lost! [y]=>" REPLY + [[ $REPLY != "y" ]] && echo -e "\n" && exit 0 + sudo rm -rf $dir + else + mkdir -p $dir + docker volume create --driver local \ + --opt type=none \ + --opt device=$dir \ + --opt o=bind \ + $vname + # echo "type=volume,dst=/$mp,volume-driver=local,volume-opt=type=none,volume-opt=o=bind,volume-opt=device=$dir" + fi + + } + + echo starting container with image: $image, and name $name + echo at container prompt type \'exit\' to exit from shell and remove trial container + docker rm try-$name > /dev/null 2>&1 + docker run -i -t --rm --entrypoint /usr/bin/env --name try-$name --hostname try-$host-$name \ + -v "$(bmount)":/opt \ + $image \ + /bin/bash -l + echo "done with session, removing containter try-$name" + + bmount -u + + } + +# if script was executed then call the function +(return 0 2>/dev/null) || try_container $@ \ No newline at end of file diff --git a/lib/src/usage b/lib/src/usage new file mode 100644 index 0000000..f4971c2 --- /dev/null +++ b/lib/src/usage @@ -0,0 +1,18 @@ +#!/bin/bash +usage() { # Function: Print a help message. + echo "Image Build Script: Creates one or more images using a target in the docker-bake.hcl file" + echo "USAGE: $0 buildtarget " + echo "valid targets: ${targets[*]}; default: dev" + echo no argument options: + echo "-c try out the image by starting a container terminal therein, for dev target this is the default;" + echo "-x exclude distro from image name;" + echo "-n for --no_cache" + echo "-p push to repository; after build push to repository default is hub.docker.common (not need for deploy target)" + echo required argument options: + echo "-d supported: alpine, debian, ubuntu, default: alpine; if base image set distro will be determined" + echo "-t tag following : in output image name (i.e. REPO/USER/NAME:TAG), default: latest" + echo "-u ; repository user prefix in output image name (i.e. REPO/USER/NAME:TAG)" + echo "-r ; private repo name, do not use for hub.docker.com (docker.io)" + echo "-b ; used in FROM in Dockerfile, default is official distro image (e.g. alpine:latest)" + echo "-w ; set a custom WORKDIR in Dockerfile, default is /opt/build" +} \ No newline at end of file diff --git a/readme.md b/readme.md index c641153..c5ff07c 100644 --- a/readme.md +++ b/readme.md @@ -1,8 +1,8 @@ # UCI Docker Image Builder -A set of scripts to facilitate building docker linux (amd64/arm64) images using any of three distros (alpine,debian,ubuntu) +A build script plus associated library to facilitate building docker linux images using any of three distros (alpine,debian,ubuntu) and either architecture (amd64/arm64) -The Dockerfile is minimal calling a set of distro specific scripts and common scripts in order to build the image +The Dockerfile is minimal, calling a set of distro specific scripts and common scripts in order to build the image The build environment makes user of docker's "buildx bake" commands and a docker-bake.hcl file @@ -10,11 +10,17 @@ The master branch is configured to build base images from the docker hub distro The main script is "build" At the very minimum run as just `./build` it will build an alpine image from the docker hub official alpine latest image with a minimal set of packages installed (e.g. git) and a custom uci shell environment. -By using the -b flag you can set an alternative FROM image. In this way you can make your own script that builds multiple tiers of images +The repo also supports (with scripts) pushing to alternate private repositories packages (like a self hosted gitea or github) -The repo also supports (with scripts) pushing to alternate private repositories (like self hosted gitea) +One can make decendent images in one of two way. + +1. Make your base images on the master branch then create a new branch and edit the script files in the src folder +3. User the installer script (./install) to install to link the build script into your system path. Then clone the template branch + + +the build script requires an init.sh file in a source directory for the scripts it will run while building a new image +by default that is `/init.sh`. by default that source directory is `src/` relative to the current directory -starting a new branch is a good way to work on a new image by editing the script files in the src folder diff --git a/src/alpine/init.sh b/src/alpine/init.sh new file mode 100644 index 0000000..a05b39d --- /dev/null +++ b/src/alpine/init.sh @@ -0,0 +1,8 @@ +#!/bin/sh +echo alpine distro init script +apk update; apk upgrade; +pwd; ls -la +echo ">>>> installing packages => alpine: $(cat ./packages); common: $(cat ../common/packages)" +apk add --no-cache bash bash-completion $(cat ./packages) $(cat ../common/packages) +# apk add bindfs --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing + diff --git a/src/alpine/packages b/src/alpine/packages new file mode 100644 index 0000000..f4d7fa5 --- /dev/null +++ b/src/alpine/packages @@ -0,0 +1 @@ +bash \ No newline at end of file diff --git a/src/common/adduser.sh b/src/common/adduser.sh new file mode 100644 index 0000000..e69de29 diff --git a/distros/common/info.sh b/src/common/info.sh similarity index 65% rename from distros/common/info.sh rename to src/common/info.sh index a2267b5..ac6891b 100644 --- a/distros/common/info.sh +++ b/src/common/info.sh @@ -1,4 +1,5 @@ #!/bin/bash wget -O /opt/scripts/info https://git.io/vaHfR; chmod +x /opt/scripts/info -ln -s /opt/scripts/info /opt/bin \ No newline at end of file +ln -s /opt/scripts/info /opt/bin +/opt/bin/info \ No newline at end of file diff --git a/distros/common/common.sh b/src/common/init.sh similarity index 54% rename from distros/common/common.sh rename to src/common/init.sh index bec86aa..5d4e421 100644 --- a/distros/common/common.sh +++ b/src/common/init.sh @@ -1,12 +1,10 @@ #!/bin/bash echo -e "\n##################################" -mkdir -p /opt/scripts /opt/bin /shell /data +mkdir -p /opt/scripts /opt/bin /shell /opt/conf /data +# this is the default host user +chown -R 1000:1000 /opt /shell /data git clone https://git.kebler.net/bash/shell-base.git /shell/base cp ucishell.sh /shell/base/setup /shell/base/setup/ucishell.sh -if [[ -f post_common.sh ]]; then -echo "running distro specific commands after common install in post_common.sh" -./post_common.sh -fi # install and display distro info ./info.sh \ No newline at end of file diff --git a/distros/common/common-packages b/src/common/packages similarity index 100% rename from distros/common/common-packages rename to src/common/packages diff --git a/distros/common/ucishell.sh b/src/common/ucishell.sh similarity index 100% rename from distros/common/ucishell.sh rename to src/common/ucishell.sh diff --git a/src/common/user.sh b/src/common/user.sh new file mode 100755 index 0000000..a1c42be --- /dev/null +++ b/src/common/user.sh @@ -0,0 +1,170 @@ +#!/bin/bash + +# clone a user + +# adduserid name id# +# add a suer with specific id number +# adduserid () { +# sudo groupadd -g $2 $1 +# sudo useradd -d ${/home/$1} -s /bin/bash -u $2 -g $1 $1 +# } + +adduserid() { + +[[ "$#" -lt 1 ]] && echo a user name is requied, aborting && return 1 +name=$1 +uid=${2:-1000} +gid=${3:-$uid} +echo $name, $uid, $gid + +[[ $(getent group $gid) ]] && echo group id $gid already exists, aborting && return 3 +[[ $(getent group $name) ]] && echo group name $name already exists, aborting && return 3 +[[ $(getent passwd $uid) ]] && echo user id $uid already exists, aborting && return 2 +[[ $(getent passwd $name) ]] && echo user name $name already exists, aborting && return 2 + +echo addgroup --gid $gid $name +echo adduser -u $uid -G $name -g "" -D -H $name +# cat /etc/group | grep $name && cat /etc/passwd | grep $name + +} + +function clone_user_ () { + + module_load confirm + + echo "=============" + echo "this script will create a new user" + echo "based on an existing user's data" + echo + echo "You will be shown a list of users who can currently log on" + echo "Remember which user you would like to clone." + echo "You will be asked for the new user's name, their password" + echo "and the old user to clone". + echo "=============" + echo + + echo -n "New user's name: " + read newuser + + echo -n "New user's password: " + read newpassword + + echo + + echo "Current users you can clone:" + echo "----" + awk -F'[/:]' '{if ($3 >= 1000 && $3 != 65534) print $1}' /etc/passwd + echo + + echo -n "Old user to clone: " + read olduser + echo "olduser uid is $(id -u $olduser)" + + echo + echo "You have selected: " + echo "----" + echo "new user: $newuser" + echo "new user password: $newpassword" + echo "old user: $olduser" + echo + + olduser_GROUPS="$(id -Gn ${olduser} | sed "s/${olduser} //g" | sed "s/ ${olduser}//g" | sed "s/ /,/g"),$olduser" + olduser_SHELL=$(awk -F : -v name=${olduser} '(name == $1) { print $7 }' /etc/passwd) + + echo "old user groups: " + echo "----" + echo $olduser_GROUPS + echo "olduser shell: " + echo $olduser_SHELL + + confirm "ready to clone user, begin?" || return 1 + + useradd --groups $olduser_GROUPS --shell $olduser_SHELL $newuser + + echo $newuser:$newpassword | chpasswd + + read -rsp $'ready to make home direcoty -- ctrl-c to exit...\n' -n1 key + + mkdir /home/$newuser + chown -R $newuser:$newuser /home/$newuser + + echo + echo "Script should be done now." + echo + echo "Do you see your new users name below?" + echo + awk -F'[/:]' '{if ($3 >= 1000 && $3 != 65534) print $1}' /etc/passwd + + echo + echo "We are now going to copy the old user's home folder to the new user" + echo "then change ownership to the new user" + echo + read -rsp $'Ready to copy home folder --- or ctrl-c to exit...\n' -n1 key + + rsync -aPv --exclude 'Downloads' /home/$olduser/. /home/$newuser/ + chown -R --from=$olduser $newuser:$newuser /home/$newuser + + echo + echo "Now we are going to change the names of files and folders to the new user" + echo + + grep -rlI $olduser /home/$newuser/ . | sudo xargs sed -i 's/$olduser/$newuser/g' + + echo + echo "Done now." + echo + read -rsp $'Press any key to exit...\n' -n1 key + echo + echo + +} + +clone_user () { + if [[ $EUID != 0 ]]; then + + clone=$(declare -f clone_user_) + module_load confirm + confirm=$(declare -f confirm) + sudo bash -c "$confirm; $clone; clone_user_" + else + echo run as regular user with sudo privliges and it will elevate + fi +} + +rename_user () { + + module_load confirm + local force; local name; local newname; local newhome + + [[ $1 = "-f" ]] && { force=true; shift 1; } + [[ $1 = "-h" ]] && { newhome=$2; shift 2; } + # usage: < -f, -h newhome > name newname + # default new user home is /home/newname + name=$1 + newname=$2 + if [[ ! ($name && $newname) ]]; then + echo "=============" + echo "this script will rename an existing user" + echo "user running this script must to root or have sudo priviledges to run" + echo "---- Available Users to Rename ---" + awk -F'[/:]' '{if ($3 >= 1000 && $3 != 65534) print $1}' /etc/passwd + echo "=============" + read -r -p "Enter an available user name: " name + read -r -p "Enter users new name: " newname + read -r -p "Enter users new home directory : " newhome + fi + + newhome=${newhome:-/home/$newname} + + if [[ ! $force ]]; then + echo "Changing $name to $newname with home $newhome" + echo sudo usermod -l $newname $name + echo sudo groupmod -n $newname $name + echo sudo usermod --d $newhome --m $name + confirm -s "These are the commands that will be run. Do you want to continue?" || return 1 + fi + + sudo usermod -l $newname $name + echo sudo groupmod -n $newname $name + echo sudo usermod --d $newhome --m $name +} diff --git a/distros/debian/init.sh b/src/debian/init.sh similarity index 100% rename from distros/debian/init.sh rename to src/debian/init.sh diff --git a/src/init.sh b/src/init.sh new file mode 100644 index 0000000..d81487b --- /dev/null +++ b/src/init.sh @@ -0,0 +1,23 @@ +#!/bin/sh + echo "entry init.sh script in $PWD" + # remove other distro files + find $PWD -maxdepth 1 -type d ! -path $PWD ! -name ${LINUX_DISTRO} ! -name common -exec rm -rf {} + + cd ${LINUX_DISTRO} || exit 1 + echo " ----- Running ${LINUX_DISTRO} specific init script ----- "; + ./init.sh + cd .. + cd common || exit 1 + echo " ***** Running common initialzation script *****" + ls -la; + ./init.sh; + cd ${LINUX_DISTRO} || exit 1 + echo " ----- Running ${LINUX_DISTRO} specific post commont script ----- "; + if [[ -f post_common.sh ]]; then + echo "running distro specific commands after common install in post_common.sh" + ./post_common.sh + fi + echo returned from ${LINUX_DISTRO} post common script + if [ -z $KEEP ]; then + echo removing $SCRIPTS directory used for build + cd /opt && rm -rf $SCRIPTS + fi \ No newline at end of file diff --git a/distros/ubuntu/add-ppa.sh b/src/ubuntu/add-ppa.sh similarity index 100% rename from distros/ubuntu/add-ppa.sh rename to src/ubuntu/add-ppa.sh diff --git a/distros/ubuntu/init.sh b/src/ubuntu/init.sh similarity index 73% rename from distros/ubuntu/init.sh rename to src/ubuntu/init.sh index ddae3e0..6663e9a 100644 --- a/distros/ubuntu/init.sh +++ b/src/ubuntu/init.sh @@ -2,5 +2,5 @@ echo ubuntu distro uci script apt-get update echo ">>>> installing packages => $(cat packages) $(cat common-packages)" -apt-get install $(cat packages) $(cat common-packages) -y +apt-get install $(cat packages) $(cat ../common/packages) -y ./add-ppa.sh -i -p git git-core/ppa diff --git a/src/ubuntu/packages b/src/ubuntu/packages new file mode 100644 index 0000000..a8bfc18 --- /dev/null +++ b/src/ubuntu/packages @@ -0,0 +1,2 @@ +ca-certificates +gpg \ No newline at end of file diff --git a/tag b/tag deleted file mode 100755 index 22f5956..0000000 --- a/tag +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -DIR=$(cd "$(dirname "$BASH_SOURCE")" >/dev/null 2>&1 ; pwd -P ) - -#tags an image and pushes it to a repository -# if not reposity is given will use docker.io and push to hub.docker.com -# $1 name, $2 user(or repo), $3 repo - -[[ $# -lt 1 ]] && echo "image base name required" && exit - -[[ $1 == "-d" ]] && delete=true && shift 1 -[[ $1 == "-f" ]] && force=true && shift 1 - - -[[ $force ]] && docker rmi -f $(docker images -q $(./make-tag $@)) && exit 0 -if [[ $delete ]];then - id=$(docker images -q $(./make-tag $@)) - docker rmi $(./make-tag $@) -else -docker tag $(./make-tag $1) $(./make-tag $2) -id=$(docker images -q $(./make-tag $2)) -fi -echo tags after operation for image $id -./image-info -k RepoTags $id - - - - - diff --git a/try b/try deleted file mode 100755 index b6f60e5..0000000 --- a/try +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -file=try.yml -DIR=$(cd "$(dirname "$BASH_SOURCE")" >/dev/null 2>&1 ; pwd -P ) - -#tags an images and pushes it to a repository -# if not reposity is given will use docker.io and push to hub.docker.com -# $1 name, $2 user - -declare -A arch=( ["x86_64"]="" ["aarch64"]="-arm64") - -[[ $# -lt 1 ]] && echo "image name required to try" && exit - -declare OPTION; declare OPTARG; declare OPTIND -while getopts 'pt:u:' OPTION; do -# echo processing: option:$OPTION argument:$OPTARG index:$OPTIND remaining:${@:$OPTIND} -case "$OPTION" in -u) - RUSER=$OPTARG -;; -t) - TAG=$OPTARG - ;; -p) - PROD=$OPTARG - ;; -*) echo unknown run option -$OPTARG - echo "USAGE: try " - echo "available options: -t custom tag " -;; -esac -done - -shift $((OPTIND - 1)) - -# build up package tag -name=$1 -user=${2:-$RUSER} - -image=$([[ $user ]] && echo ${user}/)$name:${TAG:-latest} - -if [[ $PROD ]]; then - echo removing any local copy of image $image - docker image rm $source - HOST=prod - else - HOST=local - image=${image/:/${arch[$(uname -p)]}:} - fi - -echo starting container with image $image -export IMAGE=$image -export HOST -shopt -s extglob -export NAME=${image//+([^A-Za-z0-9])/-} - -docker-compose -f $file down; -docker-compose -f $file up -d; -docker exec -it try-$NAME /bin/bash -l -docker-compose -f $file down; - -# return to starting directory -# popd \ No newline at end of file diff --git a/try.yml b/try.yml deleted file mode 100644 index 5043e5d..0000000 --- a/try.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: "3" - -services: - uci: - image: $IMAGE - container_name: try-$NAME - hostname: try-$HOST-$NAME - command: ["sleep", "infinity"]