From 70bbfd5c0b87b1365d010be459cbe2e6d76f37ed Mon Sep 17 00:00:00 2001 From: David Kebler Date: Sun, 15 Sep 2024 18:47:22 -0700 Subject: [PATCH] improved determining distro and distro image --- Dockerfile.d/Dockerfile.tpl | 9 +- Dockerfile.d/core.tpl | 2 + Dockerfile.d/create | 1 - Dockerfile.d/entrypoint.tpl | 1 - Dockerfile.d/volume.tpl | 1 - build | 26 +- core/packages.sh | 1 - core/packages/arch | 1 + core/rootfs/opt/env/run.env | 1 + core/rootfs/opt/lib/distros.csv | 8 +- docker-bake.hcl | 6 +- lib/alter-image.py | 940 -------------------------------- lib/bash-tpl | 83 ++- lib/bash-tpl.update | 4 + lib/build.lib | 135 +++-- readme.md | 6 +- test/build | 2 +- test/test.env | 9 +- test/try | 2 +- 19 files changed, 207 insertions(+), 1031 deletions(-) delete mode 100644 Dockerfile.d/entrypoint.tpl delete mode 100644 Dockerfile.d/volume.tpl create mode 100644 core/rootfs/opt/env/run.env delete mode 100755 lib/alter-image.py create mode 100755 lib/bash-tpl.update diff --git a/Dockerfile.d/Dockerfile.tpl b/Dockerfile.d/Dockerfile.tpl index e8a954d..220db2b 100644 --- a/Dockerfile.d/Dockerfile.tpl +++ b/Dockerfile.d/Dockerfile.tpl @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:latest -ARG BASE_IMAGE +ARG BASE_IMAGE=alpine ARG LINUX_DISTRO=alpine % if [[ "$BASE_IMAGE_COPY" ]]; then FROM <% $LINUX_DISTRO %> @@ -8,10 +8,12 @@ ARG LINUX_DISTRO=alpine FROM $BASE_IMAGE % fi +# repeat these so they are available for rest of dockerfile ARG BASE_IMAGE +ARG LINUX_DISTRO +# ARG VERBOSE ARG REBUILD -ARG LINUX_DISTRO=alpine WORKDIR /build # CORE @@ -28,13 +30,12 @@ eot COPY .src/rootfs/ / % fi - % if [[ ( -f "$BUILD_SRC/init/init.sh" && ! $BUILD_SRC = "_core_" ) ]]; then .INCLUDE init.run % fi # appends any additional custom Dockerfile code in source -.INCLUDE "$BDIR/.src/Dockerfile" +.INCLUDE? "$BDIR/.src/Dockerfile" % if [[ $VOLUME_DIRS ]]; then VOLUME <% $VOLUME_DIRS %> diff --git a/Dockerfile.d/core.tpl b/Dockerfile.d/core.tpl index 7311b84..fd5a575 100644 --- a/Dockerfile.d/core.tpl +++ b/Dockerfile.d/core.tpl @@ -10,10 +10,12 @@ if ! { [ "$VERBOSE" = "core" ] || [ "$VERBOSE" = "all" ]; }; then unset VERBOSE; echo "**************************************" echo "****** Building UCI Image Core ******" + echo copying core rootfs to image /bin/cp -R -f -p rootfs/. / . /opt/lib/verbose.lib +quiet env quiet echo core build directory quiet pwd quiet ls -la diff --git a/Dockerfile.d/create b/Dockerfile.d/create index b2378f0..d39dd1d 100755 --- a/Dockerfile.d/create +++ b/Dockerfile.d/create @@ -2,7 +2,6 @@ echo "------------ creating Dockfile from template in Dockerfile.d -------------" mkdir -p $BDIR/.src -[[ ! -f $BDIR/.src/Dockerfile ]] && echo "#dummy file" > $BDIR/.src/Dockerfile [[ -f $APPEND_BUILD_ENV ]] && source "$APPEND_BUILD_ENV" && echo using $APPEND_BUILD_ENV when building Dockerfile && cat $APPEND_BUILD_ENV && echo -e "\n-----" diff --git a/Dockerfile.d/entrypoint.tpl b/Dockerfile.d/entrypoint.tpl deleted file mode 100644 index 10f0e51..0000000 --- a/Dockerfile.d/entrypoint.tpl +++ /dev/null @@ -1 +0,0 @@ -ENTRYPOINT [ ] \ No newline at end of file diff --git a/Dockerfile.d/volume.tpl b/Dockerfile.d/volume.tpl deleted file mode 100644 index 10f0e51..0000000 --- a/Dockerfile.d/volume.tpl +++ /dev/null @@ -1 +0,0 @@ -ENTRYPOINT [ ] \ No newline at end of file diff --git a/build b/build index 8d99758..7f2a89d 100755 --- a/build +++ b/build @@ -158,9 +158,13 @@ done shift $((OPTIND - 1)) - [[ ! $BUILD_EFILE ]] && source_env_file +if ! validate_distro; then + >&2 echo "FATAL: unable to validate the BASE_IMAGE ($BASE_IMAGE) and it's LINUX_DISTRO ($LINUX_DISTRO), aborting build" + return 2 +fi + if ! get_build_src > /dev/null ; then if [[ $no_prompt ]] ; then echo aborting the build... @@ -178,9 +182,6 @@ fi TARGET=${TARGET:-default} [[ ! "${targets[@]}" =~ $TARGET ]] && echo $TARGET is not a valid target && echo valid targets are: ${targets[@]} && exit 4 -LINUX_DISTRO=${LINUX_DISTRO:-alpine} -if ! get_base_image; then return $?; fi - IMAGE_NAME=$(make_image_name $@) # TODO writing to existing tag untags existing image so write a new tag to that image then continue @@ -210,11 +211,23 @@ export ARCH export VERBOSE export REBUILD +if [[ $VERBOSE ]]; then +echo BASE_IMAGE=$BASE_IMAGE +echo TAG=$TAG +echo IMAGE_NAME=$IMAGE_NAME +echo LINUX_DISTRO=$LINUX_DISTRO +echo BUILD_SRC=$BUILD_SRC +echo ARCH=$ARCH +echo VERBOSE=$VERBOSE +echo REBUILD=$REBUILD +fi + build_info if [[ ! $no_prompt ]]; then read -n 1 -p "do you want to continue [y]=>" REPLY [[ $REPLY != "y" ]] && echo -e "\n" && return 4 + echo -e "********** starting build ****************\n" fi # cat $BDIR/Dockerfile | grep -b5 -a5 ENTRY @@ -247,12 +260,8 @@ if [[ ! $BUILD_SRC = "_core_" ]]; then /bin/cp -a $BDIR/.src/rootfs/opt/env/. $BDIR/core/rootfs/opt/env > /dev/null 2>&1 fi ls -la $BDIR/.src/rootfs - ls -la $BDIR/.src/rootfs/root fi -echo run environment directory copied to core at $BDIR/core/$_env_dir -ls -la $BDIR/core/$_env_dir - # create Dockerfile from template if ! source $BDIR/Dockerfile.d/create; then echo unable to create Dockerfile from template, aborting build @@ -288,7 +297,6 @@ pushd "$BDIR" > /dev/null || return 3 export BUILDING=true -echo -e "\n\e[1;31m######### RUNNING THE DOCKER BUILD COMMAND ######################" echo running build command: docker buildx --builder ${builder} bake ${nocache} ${TARGET} echo -e "#################################################################\e[1;37m" docker buildx --builder ${builder} bake ${nocache} ${TARGET} 2>&1 | tee "$log_dir/${IMAGE_NAME//\//-}build.log" diff --git a/core/packages.sh b/core/packages.sh index 91ff120..22a4bce 100644 --- a/core/packages.sh +++ b/core/packages.sh @@ -13,7 +13,6 @@ if [ -f ./packages/$LINUX_DISTRO ]; then echo "DONE INSTALLING $LINUX_DISTRO SPECIFIC PACKAGES" fi echo INSTALLING COMMON PACKAGES FOR ANY DISTRO -quiet this is a test of quiet _pkgs=$(cat ./packages/common) echo $_pkgs echo .... diff --git a/core/packages/arch b/core/packages/arch index e69de29..1421196 100644 --- a/core/packages/arch +++ b/core/packages/arch @@ -0,0 +1 @@ +which \ No newline at end of file diff --git a/core/rootfs/opt/env/run.env b/core/rootfs/opt/env/run.env new file mode 100644 index 0000000..9c10621 --- /dev/null +++ b/core/rootfs/opt/env/run.env @@ -0,0 +1 @@ +export DEFAULT_DIR=/opt/bin \ No newline at end of file diff --git a/core/rootfs/opt/lib/distros.csv b/core/rootfs/opt/lib/distros.csv index 3848ff6..99db7ea 100644 --- a/core/rootfs/opt/lib/distros.csv +++ b/core/rootfs/opt/lib/distros.csv @@ -1,7 +1,7 @@ # valid distros list # the distro must be the name used in /etc/os-release -# ,,, -alpine,alpine, apk add --no-cache, apk update -debian,debian, apt-get install -y, apt-get update -arch, archlinux,pacman -S --noconfirm --needed, pacman -Syu +# ,,, +alpine, alpine, apk add --no-cache , apk update +debian, debian, apt-get install -y, apt-get update +arch, archlinux, pacman -S --noconfirm --needed, pacman -Syu ubuntu, ubuntu, apt-get install -y, apt-get update \ No newline at end of file diff --git a/docker-bake.hcl b/docker-bake.hcl index 29b36f8..dbe7777 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -3,13 +3,13 @@ variable "TAG" { default = "latest" } variable "LINUX_DISTRO" { - // default = "alpine" + default = "alpine" } variable "IMAGE_NAME" { - // default = "alpine" + default = "alpine" } variable "BASE_IMAGE" { - // default = "alpine" + default = "alpine" } variable "VERBOSE" { default = "" diff --git a/lib/alter-image.py b/lib/alter-image.py deleted file mode 100755 index 2eac874..0000000 --- a/lib/alter-image.py +++ /dev/null @@ -1,940 +0,0 @@ -#! /usr/bin/env python3 -from __future__ import print_function - -__copyright__ = "(C) 2017-2023 Guido U. Draheim, licensed under the EUPL" -__version__ = "1.4.6097" - -import subprocess -import collections -import sys -import os -import re -import json -import copy -import shutil -import hashlib -import datetime -import logging -from fnmatch import fnmatchcase as fnmatch - -logg = logging.getLogger("edit") - -if sys.version[0] != '2': - xrange = range - -MAX_PATH = 1024 # on Win32 = 260 / Linux PATH_MAX = 4096 / Mac = 1024 -MAX_NAME = 253 -MAX_PART = 63 -MAX_VERSION = 127 -MAX_COLLISIONS = 100 - -TMPDIR = "load.tmp" -DOCKER = "docker" -KEEPDIR = 0 -KEEPDATADIR = False -KEEPSAVEFILE = False -KEEPINPUTFILE = False -KEEPOUTPUTFILE = False -OK = True -NULL = "NULL" - -StringConfigs = {"user": "User", "domainname": "Domainname", - "workingdir": "WorkingDir", "workdir": "WorkingDir", "hostname": "Hostname"} -StringMeta = {"author": "author", "os": "os", "architecture": "architecture", "arch": "architecture", "variant": "variant"} -StringCmd = {"cmd": "Cmd", "entrypoint": "Entrypoint"} - -ShellResult = collections.namedtuple("ShellResult", ["returncode", "stdout", "stderr"]) - -def sh(cmd=":", shell=True, check=True, ok=None, default=""): - if ok is None: ok = OK # a parameter "ok = OK" does not work in python - if not ok: - logg.info("skip %s", cmd) - return ShellResult(0, default, "") - run = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - run.wait() - assert run.stdout is not None and run.stderr is not None - result = ShellResult(run.returncode, run.stdout.read(), run.stderr.read()) - if check and result.returncode: - logg.error("CMD %s", cmd) - logg.error("EXIT %s", result.returncode) - logg.error("STDOUT %s", result.stdout) - logg.error("STDERR %s", result.stderr) - raise Exception("shell command failed") - return result - -def portprot(arg): - port, prot = arg, "" - if "/" in arg: - port, prot = arg.rsplit("/", 1) - if port and port[0] in "0123456789": - pass - else: - import socket - if prot: - portnum = socket.getservbyname(port, prot) - else: - portnum = socket.getservbyname(port) - port = str(portnum) - if not prot: - prot = "tcp" - return port, prot - -def podman(): - return "podman" in DOCKER -def cleans(text): - if podman(): - return text.replace('": ', '":').replace(', "', ',"').replace(', {', ',{') - return text -def os_jsonfile(filename): - if podman(): - os.chmod(filename, 0o644) - os.utime(filename, (0, 0)) - -class ImageName: - def __init__(self, image): - self.registry = None - self.image = image - self.version = None - self.parse(image) - def parse(self, image): - parsing = image - parts = image.split("/") - if ":" in parts[-1] or "@" in parts[-1]: - colon = parts[-1].find(":") - atref = parts[-1].find("@") - if colon >= 0 and atref >= 0: - first = min(colon, atref) - else: - first = max(colon, atref) - version = parts[-1][first:] - parts[-1] = parts[-1][:first] - self.version = version - self.image = "/".join(parts) - if len(parts) > 1 and ":" in parts[0]: - registry = parts[0] - parts = parts[1:] - self.registry = registry - self.image = "/".join(parts) - logg.debug("image parsing = %s", parsing) - logg.debug(".registry = %s", self.registry) - logg.debug(".image = %s", self.image) - logg.debug(".version = %s", self.version) - def __str__(self): - image = self.image - if self.registry: - image = "/".join([self.registry, image]) - if self.version: - image += self.version - return image - def tag(self): - image = self.image - if self.registry: - image = "/".join([self.registry, image]) - if self.version: - image += self.version - else: - image += ":latest" - return image - def local(self): - if not self.registry: return True - if "." not in self.registry: return True - if "localhost" in self.registry: return True - return False - def valid(self): - return not list(self.problems()) - def problems(self): - # https://docs.docker.com/engine/reference/commandline/tag/ - # https://github.com/docker/distribution/blob/master/reference/regexp.go - if self.registry and self.registry.startswith("["): - if len(self.registry) > MAX_NAME: - yield "registry name: full name may not be longer than %i characters" % MAX_NAME - yield "registry name= " + self.registry - x = self.registry.find("]") - if not x: - yield "registry name: invalid ipv6 number (missing bracket)" - yield "registry name= " + self.registry - port = self.registry[x + 1:] - if port: - m = re.match("^:[A-Za-z0-9]+$", port) - if not m: - yield 'registry name: invalid ipv6 port (only alnum)' - yield "registry name= " + port - base = self.registry[:x] - if not base: - yield "registry name: invalid ipv6 number (empty)" - else: - m = re.match("^[0-9abcdefABCDEF:]*$", base) - if not m: - yield "registry name: invalid ipv6 number (only hexnum+colon)" - yield "registry name= " + base - elif self.registry: - if len(self.registry) > MAX_NAME: - yield "registry name: full name may not be longer than %i characters" % MAX_NAME - yield "registry name= " + self.registry - registry = self.registry - if registry.count(":") > 1: - yield "a colon may only be used to designate the port number" - yield "registry name= " + registry - elif registry.count(":") == 1: - registry, port = registry.split(":", 1) - m = re.match("^[A-Za-z0-9]+$", port) - if not m: - yield 'registry name: invalid ipv4 port (only alnum)' - yield "registry name= " + registry - parts = registry.split(".") - if "" in parts: - yield "no double dots '..' allowed in registry names" - yield "registry name= " + registry - for part in parts: - if len(part) > MAX_PART: - yield "registry name: dot-separated parts may only have %i characters" % MAX_PART - yield "registry name= " + part - m = re.match("^[A-Za-z0-9-]*$", part) - if not m: - yield "registry name: dns names may only have alnum+dots+dash" - yield "registry name= " + part - if part.startswith("-"): - yield "registry name: dns name parts may not start with a dash" - yield "registry name= " + part - if part.endswith("-") and len(part) > 1: - yield "registry name: dns name parts may not end with a dash" - yield "registry name= " + part - if self.image: - if len(self.image) > MAX_NAME: - yield "image name: should not be longer than %i characters (min path_max)" % MAX_NAME - yield "image name= " + self.image - if len(self.image) > MAX_PATH: - yield "image name: can not be longer than %i characters (limit path_max)" % MAX_PATH - yield "image name= " + self.image - parts = self.image.split("/") - for part in parts: - if not part: - yield "image name: double slashes are not a good idea" - yield "image name= " + part - continue - if len(part) > MAX_NAME: - yield "image name: slash-separated parts should only have %i characters" % MAX_NAME - yield "image name= " + part - separators = "._-" - m = re.match("^[a-z0-9._-]*$", part) - if not m: - yield "image name: only lowercase+digits+dots+dash+underscore" - yield "image name= " + part - if part[0] in separators: - yield "image name: components may not start with a separator (%s)" % part[0] - yield "image name= " + part - if part[-1] in separators and len(part) > 1: - yield "image name: components may not end with a separator (%s)" % part[-1] - yield "image name= " + part - elems = part.split(".") - if "" in elems: - yield "image name: only single dots are allowed, not even double" - yield "image name= " + part - elems = part.split("_") - if len(elems) > 2: - for x in xrange(len(elems) - 1): - if not elems[x] and not elems[x + 1]: - yield "image name: only single or double underscores are allowed" - yield "image name= " + part - if self.version: - if len(self.version) > MAX_VERSION: - yield "image version: may not be longer than %i characters" % MAX_VERSION - yield "image version= " + self.version - if self.version[0] not in ":@": - yield "image version: must either be :version or @digest" - yield "image version= " + self.version - if len(self.version) > 1 and self.version[1] in "-.": - yield "image version: may not start with dots or dash" - yield "image version= " + self.version - version = self.version[1:] - if not version: - yield "image version: no name provided after '%s'" % self.version[0] - yield "image version= " + self.version - m = re.match("^[A-Za-z0-9_.-]*$", version) - if not m: - yield 'image version: only alnum+undescore+dots+dash are allowed' - yield "image version= " + self.version - -def edit_image(inp, out, edits): - if True: - if not inp: - logg.error("no FROM value provided") - return False - if not out: - logg.error("no INTO value provided") - return False - inp_name = ImageName(inp) - out_name = ImageName(out) - for problem in inp_name.problems(): - logg.warning("FROM value: %s", problem) - for problem in out_name.problems(): - logg.warning("INTO value: %s", problem) - if not out_name.local(): - logg.warning("output image is not local for the 'docker load' step") - else: - logg.warning("output image is local (%s)", out_name.registry) - inp_tag = inp - out_tag = out_name.tag() - # - tmpdir = TMPDIR - if not os.path.isdir(tmpdir): - logg.debug("mkdir %s", tmpdir) - if OK: os.makedirs(tmpdir) - datadir = os.path.join(tmpdir, "data") - if not os.path.isdir(datadir): - logg.debug("mkdir %s", datadir) - if OK: os.makedirs(datadir) - inputfile = os.path.join(tmpdir, "saved.tar") - outputfile = os.path.join(tmpdir, "ready.tar") - inputfile_hints = "" - outputfile_hints = "" - # - docker = DOCKER - if KEEPSAVEFILE: - if os.path.exists(inputfile): - os.remove(inputfile) - cmd = "{docker} save {inp} -o {inputfile}" - sh(cmd.format(**locals())) - cmd = "tar xf {inputfile} -C {datadir}" - sh(cmd.format(**locals())) - logg.info("new {datadir} from {inputfile}".format(**locals())) - else: - cmd = "{docker} save {inp} | tar x -f - -C {datadir}" - sh(cmd.format(**locals())) - logg.info("new {datadir} from {docker} save".format(**locals())) - inputfile_hints += " (not created)" - run = sh("ls -l {tmpdir}".format(**locals())) - logg.debug(run.stdout) - # - if OK: - changed = edit_datadir(datadir, out_tag, edits) - if changed: - outfile = os.path.realpath(outputfile) - cmd = "cd {datadir} && tar cf {outfile} ." - sh(cmd.format(**locals())) - cmd = "{docker} load -i {outputfile}" - sh(cmd.format(**locals())) - else: - logg.warning("unchanged image from %s", inp_tag) - outputfile_hints += " (not created)" - if inp != out: - cmd = "{docker} tag {inp_tag} {out_tag}" - sh(cmd.format(**locals())) - logg.warning(" tagged old image as %s", out_tag) - # - if KEEPDATADIR: - logg.warning("keeping %s", datadir) - else: - if os.path.exists(datadir): - shutil.rmtree(datadir) - if KEEPINPUTFILE: - logg.warning("keeping %s%s", inputfile, inputfile_hints) - else: - if os.path.exists(inputfile): - os.remove(inputfile) - if KEEPOUTPUTFILE: - logg.warning("keeping %s%s", outputfile, outputfile_hints) - else: - if os.path.exists(outputfile): - os.remove(outputfile) - return True - -def edit_datadir(datadir, out, edits): - if True: - manifest_file = "manifest.json" - manifest_filename = os.path.join(datadir, manifest_file) - with open(manifest_filename) as _manifest_file: - manifest = json.load(_manifest_file) - replaced = {} - for item in xrange(len(manifest)): - config_file = manifest[item]["Config"] - config_filename = os.path.join(datadir, config_file) - replaced[config_filename] = None - # - for item in xrange(len(manifest)): - config_file = manifest[item]["Config"] - config_filename = os.path.join(datadir, config_file) - with open(config_filename) as _config_file: - config = json.load(_config_file) - old_config_text = cleans(json.dumps(config)) # to compare later - # - for CONFIG in ['config', 'Config', 'container_config']: - if CONFIG not in config: - logg.debug("no section '%s' in config", CONFIG) - continue - logg.debug("with %s: %s", CONFIG, config[CONFIG]) - for action, target, arg in edits: - if action in ["remove", "rm"] and target in ["volume", "volumes"]: - key = 'Volumes' - if not arg: - logg.error("can not do edit %s %s without arg: <%s>", action, target, arg) - continue - elif target in ["volumes"] and arg in ["*", "%"]: - args = [] - try: - if key in config[CONFIG] and config[CONFIG][key] is not None: - del config[CONFIG][key] - logg.warning("done actual config %s %s '%s'", action, target, arg) - except KeyError as e: - logg.warning("there was no '%s' in %s", key, config_filename) - elif target in ["volumes"]: - pattern = arg.replace("%", "*") - args = [] - if key in config[CONFIG] and config[CONFIG][key] is not None: - for entry in config[CONFIG][key]: - if fnmatch(entry, pattern): - args += [entry] - logg.debug("volume pattern %s -> %s", pattern, args) - if not args: - logg.warning("%s pattern '%s' did not match anything", target, pattern) - elif arg.startswith("/"): - args = [arg] - else: - logg.error("can not do edit %s %s %s", action, target, arg) - continue - # - for arg in args: - entry = os.path.normpath(arg) - try: - if config[CONFIG][key] is None: - raise KeyError("null section " + key) - del config[CONFIG][key][entry] - except KeyError as e: - logg.warning("there was no '%s' in '%s' of %s", entry, key, config_filename) - if action in ["remove", "rm"] and target in ["port", "ports"]: - key = 'ExposedPorts' - if not arg: - logg.error("can not do edit %s %s without arg: <%s>", action, target, arg) - continue - elif target in ["ports"] and arg in ["*", "%"]: - args = [] - try: - if key in config[CONFIG] and config[CONFIG][key] is not None: - del config[CONFIG][key] - logg.warning("done actual config %s %s %s", action, target, arg) - except KeyError as e: - logg.warning("there were no '%s' in %s", key, config_filename) - elif target in ["ports"]: - pattern = arg.replace("%", "*") - args = [] - if key in config[CONFIG] and config[CONFIG][key] is not None: - for entry in config[CONFIG][key]: - if fnmatch(entry, pattern): - args += [entry] - logg.debug("ports pattern %s -> %s", pattern, args) - if not args: - logg.warning("%s pattern '%s' did not match anything", target, pattern) - else: - args = [arg] - # - for arg in args: - port, prot = portprot(arg) - if not port: - logg.error("can not do edit %s %s %s", action, target, arg) - return False - entry = u"%s/%s" % (port, prot) - try: - if config[CONFIG][key] is None: - raise KeyError("null section " + key) - del config[CONFIG][key][entry] - logg.info("done rm-port '%s' from '%s'", entry, key) - except KeyError as e: - logg.warning("there was no '%s' in '%s' of %s", entry, key, config_filename) - if action in ["append", "add"] and target in ["volume"]: - if not arg: - logg.error("can not do edit %s %s without arg: <%s>", action, target, arg) - continue - key = 'Volumes' - entry = os.path.normpath(arg) - if config[CONFIG].get(key) is None: - config[CONFIG][key] = {} - if arg not in config[CONFIG][key]: - config[CONFIG][key][entry] = {} - logg.info("added %s to %s", entry, key) - if action in ["append", "add"] and target in ["port"]: - if not arg: - logg.error("can not do edit %s %s without arg: <%s>", action, target, arg) - continue - key = 'ExposedPorts' - port, prot = portprot(arg) - entry = "%s/%s" % (port, prot) - if key not in config[CONFIG]: - config[CONFIG][key] = {} - if arg not in config[CONFIG][key]: - config[CONFIG][key][entry] = {} - logg.info("added %s to %s", entry, key) - if action in ["set", "set-shell"] and target in ["entrypoint"]: - key = 'Entrypoint' - try: - if not arg: - running = None - elif action in ["set-shell"]: - running = ["/bin/sh", "-c", arg] - elif arg.startswith("["): - running = json.loads(arg) - else: - running = [arg] - config[CONFIG][key] = running - logg.warning("done edit %s %s", action, arg) - except KeyError as e: - logg.warning("there was no '%s' in %s", key, config_filename) - if action in ["set", "set-shell"] and target in ["cmd"]: - key = 'Cmd' - try: - if not arg: - running = None - elif action in ["set-shell"]: - running = ["/bin/sh", "-c", arg] - logg.info("%s %s", action, running) - elif arg.startswith("["): - running = json.loads(arg) - else: - running = [arg] - config[CONFIG][key] = running - logg.warning("done edit %s %s", action, arg) - except KeyError as e: - logg.warning("there was no '%s' in %s", key, config_filename) - if action in ["set"] and target in StringConfigs: - key = StringConfigs[target] - try: - if not arg: - value = u'' - else: - value = arg - if key in config[CONFIG]: - if config[CONFIG][key] == value: - logg.warning("unchanged config '%s' %s", key, value) - else: - config[CONFIG][key] = value - logg.warning("done edit config '%s' %s", key, value) - else: - config[CONFIG][key] = value - logg.warning("done new config '%s' %s", key, value) - except KeyError as e: - logg.warning("there was no config %s in %s", target, config_filename) - if action in ["set"] and target in StringMeta: - key = StringMeta[target] - try: - if not arg: - value = u'' - else: - value = arg - if key in config: - if config[key] == value: - logg.warning("unchanged meta '%s' %s", key, value) - else: - config[key] = value - logg.warning("done edit meta '%s' %s", key, value) - else: - config[key] = value - logg.warning("done new meta '%s' %s", key, value) - except KeyError as e: - logg.warning("there was no meta %s in %s", target, config_filename) - if action in ["set-label"]: - key = "Labels" - try: - value = arg or u'' - if key not in config[CONFIG]: - config[CONFIG][key] = {} - if target in config[CONFIG][key]: - if config[CONFIG][key][target] == value: - logg.warning("unchanged label '%s' %s", target, value) - else: - config[CONFIG][key][target] = value - logg.warning("done edit label '%s' %s", target, value) - else: - config[CONFIG][key][target] = value - logg.warning("done new label '%s' %s", target, value) - except KeyError as e: - logg.warning("there was no config %s in %s", target, config_filename) - if action in ["remove-label", "rm-label"]: - if not target: - logg.error("can not do edit %s without arg: <%s>", action, target) - continue - key = "Labels" - try: - if key in config[CONFIG]: - if config[CONFIG][key] is None: - raise KeyError("null section " + key) - del config[CONFIG][key][target] - logg.warning("done actual %s %s ", action, target) - except KeyError as e: - logg.warning("there was no label %s in %s", target, config_filename) - if action in ["remove-labels", "rm-labels"]: - if not target: - logg.error("can not do edit %s without arg: <%s>", action, target) - continue - key = "Labels" - try: - pattern = target.replace("%", "*") - args = [] - if key in config[CONFIG] and config[CONFIG][key] is not None: - for entry in config[CONFIG][key]: - if fnmatch(entry, pattern): - args += [entry] - for arg in args: - del config[CONFIG][key][arg] - logg.warning("done actual %s %s (%s)", action, target, arg) - except KeyError as e: - logg.warning("there was no label %s in %s", target, config_filename) - if action in ["remove-envs", "rm-envs"]: - if not target: - logg.error("can not do edit %s without arg: <%s>", action, target) - continue - key = "Env" - try: - pattern = target.strip() + "=*" - pattern = pattern.replace("%", "*") - found = [] - if key in config[CONFIG] and config[CONFIG][key] is not None: - for n, entry in enumerate(config[CONFIG][key]): - if fnmatch(entry, pattern): - found += [n] - for n in reversed(found): - del config[CONFIG][key][n] - logg.warning("done actual %s %s (%s)", action, target, n) - except KeyError as e: - logg.warning("there was no label %s in %s", target, config_filename) - if action in ["remove-env", "rm-env"]: - if not target: - logg.error("can not do edit %s without arg: <%s>", action, target) - continue - key = "Env" - try: - if "=" in target: - pattern = target.strip() - else: - pattern = target.strip() + "=*" - found = [] - if key in config[CONFIG] and config[CONFIG][key] is not None: - for n, entry in enumerate(config[CONFIG][key]): - if fnmatch(entry, pattern): - found += [n] - for n in reversed(found): - del config[CONFIG][key][n] - logg.warning("done actual %s %s (%s)", action, target, n) - except KeyError as e: - logg.warning("there was no label %s in %s", target, config_filename) - if action in ["remove-healthcheck", "rm-healthcheck"]: - key = "Healthcheck" - try: - del config[CONFIG][key] - logg.warning("done actual %s %s", action, target) - except KeyError as e: - logg.warning("there was no %s in %s", key, config_filename) - if action in ["set-envs"]: - if not target: - logg.error("can not do edit %s without arg: <%s>", action, target) - continue - key = "Env" - try: - if "=" in target: - pattern = target.strip().replace("%", "*") - else: - pattern = target.strip().replace("%", "*") + "=*" - if key not in config[CONFIG]: - config[key] = {} - found = [] - for n, entry in enumerate(config[CONFIG][key]): - if fnmatch(entry, pattern): - found += [n] - if found: - for n in reversed(found): - oldvalue = config[CONFIG][key][n] - varname = oldvalue.split("=", 1)[0] - newvalue = varname + "=" + (arg or u'') - if config[CONFIG][key][n] == newvalue: - logg.warning("unchanged var '%s' %s", target, newvalue) - else: - config[CONFIG][key][n] = newvalue - logg.warning("done edit var '%s' %s", target, newvalue) - elif "=" in target or "*" in target or "%" in target or "?" in target or "[" in target: - logg.info("non-existing var pattern '%s'", target) - else: - value = target.strip() + "=" + (arg or u'') - config[CONFIG][key] += [pattern + value] - logg.warning("done new var '%s' %s", target, value) - except KeyError as e: - logg.warning("there was no config %s in %s", target, config_filename) - if action in ["set-env"]: - if not target: - logg.error("can not do edit %s without arg: <%s>", action, target) - continue - key = "Env" - try: - pattern = target.strip() + "=" - if key not in config[CONFIG]: - config[key] = {} - found = [] - for n, entry in enumerate(config[CONFIG][key]): - if entry.startswith(pattern): - found += [n] - if found: - for n in reversed(found): - oldvalue = config[CONFIG][key][n] - varname = oldvalue.split("=", 1)[0] - newvalue = varname + "=" + (arg or u'') - if config[CONFIG][key][n] == newvalue: - logg.warning("unchanged var '%s' %s", target, newvalue) - else: - config[CONFIG][key][n] = newvalue - logg.warning("done edit var '%s' %s", target, newvalue) - elif "=" in target or "*" in target or "%" in target or "?" in target or "[" in target: - logg.info("may not use pattern characters in env variable '%s'", target) - else: - value = target.strip() + "=" + (arg or u'') - config[CONFIG][key] += [pattern + value] - logg.warning("done new var '%s' %s", target, value) - except KeyError as e: - logg.warning("there was no config %s in %s", target, config_filename) - logg.debug("done %s: %s", CONFIG, config[CONFIG]) - new_config_text = cleans(json.dumps(config)) - if new_config_text != old_config_text: - for CONFIG in ['history']: - if CONFIG in config: - myself = os.path.basename(sys.argv[0]) - config[CONFIG] += [{"empty_layer": True, - "created_by": "%s #(%s)" % (myself, __version__), - "created": datetime.datetime.utcnow().isoformat() + "Z"}] - new_config_text = cleans(json.dumps(config)) - new_config_md = hashlib.sha256() - new_config_md.update(new_config_text.encode("utf-8")) - for collision in xrange(1, MAX_COLLISIONS): - new_config_hash = new_config_md.hexdigest() - new_config_file = "%s.json" % new_config_hash - new_config_filename = os.path.join(datadir, new_config_file) - if new_config_filename in replaced.keys() or new_config_filename in replaced.values(): - logg.info("collision %s %s", collision, new_config_filename) - new_config_md.update(" ".encode("utf-8")) - continue - break - with open(new_config_filename, "wb") as fp: - fp.write(new_config_text.encode("utf-8")) - logg.info("written new %s", new_config_filename) - logg.info("removed old %s", config_filename) - os_jsonfile(new_config_filename) - # - manifest[item]["Config"] = new_config_file - replaced[config_filename] = new_config_filename - else: - logg.info(" unchanged %s", config_filename) - # - if manifest[item]["RepoTags"]: - manifest[item]["RepoTags"] = [out] - manifest_text = cleans(json.dumps(manifest)) - manifest_filename = os.path.join(datadir, manifest_file) - # report the result - with open(manifest_filename + ".tmp", "wb") as fp: - fp.write(manifest_text.encode("utf-8")) - if podman(): - if os.path.isfile(manifest_filename + ".old"): - os.remove(manifest_filename + ".old") - os_jsonfile(manifest_filename) - os.rename(manifest_filename, manifest_filename + ".old") - os.rename(manifest_filename + ".tmp", manifest_filename) - changed = 0 - for a, b in replaced.items(): - if b: - changed += 1 - logg.debug("replaced\n\t old %s\n\t new %s", a, b) - else: - logg.debug("unchanged\n\t old %s", a) - logg.debug("updated\n\t --> %s", manifest_filename) - logg.debug("changed %s layer metadata", changed) - return changed - -def parsing(args): - inp = None - out = None - action = None - target = None - commands = [] - known_set_targets = list(StringCmd.keys()) + list(StringConfigs.keys()) + list(StringMeta.keys()) - for n in xrange(len(args)): - arg = args[n] - if target is not None: - if target.lower() in ["all"]: - # remove all ports => remove ports * - commands.append((action, arg.lower(), "*")) - elif action in ["set", "set-shell"] and target.lower() in ["null", "no"]: - # set null cmd => set cmd - if arg.lower() not in known_set_targets: - logg.error("bad edit command: %s %s %s", action, target, arg) - commands.append((action, arg.lower(), None)) - elif action in ["set", "set-shell"] and target.lower() in known_set_targets: - # set cmd null => set cmd - if arg.lower() in [NULL.lower(), NULL.upper()]: - logg.info("do not use '%s %s %s' - use 'set null %s'", action, target, arg, target.lower()) - commands.append((action, target.lower(), None)) - elif arg.lower() in ['']: - logg.error("do not use '%s %s %s' - use 'set null %s'", action, target, '""', target.lower()) - logg.warning("we assume here but that will change in the future") - commands.append((action, target.lower(), None)) - else: - commands.append((action, target.lower(), arg)) - else: - commands.append((action, target, arg)) - action, target = None, None - continue - if action is None: - if arg in ["and", "+", ",", "/"]: - continue - action = arg.lower() - continue - rm_labels = ["rm-label", "remove-label", "rm-labels", "remove-labels"] - rm_vars = ["rm-var", "remove-var", "rm-vars", "remove-vars"] - rm_envs = ["rm-env", "remove-env", "rm-envs", "remove-envs"] - if action in (rm_labels + rm_vars + rm_envs): - target = arg - commands.append((action, target, None)) - action, target = None, None - continue - # - if action in ["set"] and arg.lower() in ["shell", "label", "labels", "var", "vars", "env", "envs"]: - action = "%s-%s" % (action, arg.lower()) - continue - if action in ["rm", "remove"] and arg.lower() in ["label", "labels", "var", "vars", "env", "envs"]: - action = "%s-%s" % (action, arg.lower()) - continue - if action in ["rm", "remove"] and arg.lower() in ["healthcheck"]: - action = "%s-%s" % (action, arg.lower()) - commands.append((action, None, None)) - action, target = None, None - continue - if action in ["from"]: - inp = arg - action = None - continue - elif action in ["into"]: - out = arg - action = None - continue - elif action in ["remove", "rm"]: - if arg.lower() in ["volume", "port", "all", "volumes", "ports"]: - target = arg.lower() - continue - logg.error("unknown edit command starting with %s %s", action, arg) - return None, None, [] - elif action in ["append", "add"]: - if arg.lower() in ["volume", "port"]: - target = arg.lower() - continue - logg.error("unknown edit command starting with %s %s", action, arg) - return None, None, [] - elif action in ["set", "override"]: - if arg.lower() in known_set_targets: - target = arg.lower() - continue - if arg.lower() in ["null", "no"]: - target = arg.lower() - continue # handled in "all" / "no" case - logg.error("unknown edit command starting with %s %s", action, arg) - return None, None, [] - elif action in ["set-shell"]: - if arg.lower() in StringCmd: - target = arg.lower() - continue - logg.error("unknown edit command starting with %s %s", action, arg) - return None, None, [] - elif action in ["set-label", "set-var", "set-env", "set-envs"]: - target = arg - continue - else: - logg.error("unknown edit command starting with %s", action) - return None, None, [] - if not inp: - logg.error("no input image given - use 'FROM image-name'") - return None, None, [] - if not out: - logg.error("no output image given - use 'INTO image-name'") - return None, None, [] - return inp, out, commands - -def docker_tag(inp, out): - docker = DOCKER - if inp and out and inp != out: - cmd = "{docker} tag {inp} {out}" - logg.info("%s", cmd) - sh("{docker} tag {inp} {out}".format(**locals()), check=False) - - -if __name__ == "__main__": - from optparse import OptionParser - cmdline = OptionParser("%prog input-image output-image [commands...]") - cmdline.add_option("-T", "--tmpdir", metavar="DIR", default=TMPDIR, - help="use this base temp dir %s [%default]") - cmdline.add_option("-D", "--docker", metavar="DIR", default=DOCKER, - help="use another docker container tool %s [%default]") - cmdline.add_option("-k", "--keepdir", action="count", default=KEEPDIR, - help="keep the unpacked dirs [%default]") - cmdline.add_option("-v", "--verbose", action="count", default=0, - help="increase logging level [%default]") - cmdline.add_option("-z", "--dryrun", action="store_true", default=not OK, - help="only run logic, do not change anything [%default]") - cmdline.add_option("--with-null", metavar="name", default=NULL, - help="specify the special value for disable [%default]") - cmdline.add_option("-c", "--config", metavar="NAME=VAL", action="append", default=[], - help="..override internal variables (MAX_PATH) {%default}") - opt, args = cmdline.parse_args() - logging.basicConfig(level=max(0, logging.ERROR - 10 * opt.verbose)) - TMPDIR = opt.tmpdir - DOCKER = opt.docker - KEEPDIR = opt.keepdir - OK = not opt.dryrun - NULL = opt.with_null - if KEEPDIR >= 1: - KEEPDATADIR = True - if KEEPDIR >= 2: - KEEPSAVEFILE = True - if KEEPDIR >= 3: - KEEPINPUTFILE = True - if KEEPDIR >= 4: - KEEPOUTPUTFILE = True - ######################################## - for setting in opt.config: - nam, val = setting, "1" - if "=" in setting: - nam, val = setting.split("=", 1) - elif nam.startswith("no-") or nam.startswith("NO-"): - nam, val = nam[3:], "0" - elif nam.startswith("No") or nam.startswith("NO"): - nam, val = nam[2:], "0" - if nam in globals(): - old = globals()[nam] - if old is False or old is True: - logg.debug("yes %s=%s", nam, val) - globals()[nam] = (val in ("true", "True", "TRUE", "yes", "y", "Y", "YES", "1")) - elif isinstance(old, float): - logg.debug("num %s=%s", nam, val) - globals()[nam] = float(val) - elif isinstance(old, int): - logg.debug("int %s=%s", nam, val) - globals()[nam] = int(val) - elif isinstance(old, str): - logg.debug("str %s=%s", nam, val) - globals()[nam] = val.strip() - else: - logg.warning("(ignored) unknown target type -c '%s' : %s", nam, type(old)) - else: - logg.warning("(ignored) unknown target config -c '%s' : no such variable", nam) - ######################################## - if len(args) < 2: - logg.error("not enough arguments, use --help") - else: - inp, out, commands = parsing(args) - if not commands: - logg.warning("nothing to do for %s", out) - docker_tag(inp, out) - else: - if opt.dryrun: - oldlevel = logg.level - logg.level = logging.INFO - logg.info(" | from %s into %s", inp, out) - for action, target, arg in commands: - if arg is None: - arg = "" - else: - arg = "'%s'" % arg - logg.info(" | %s %s %s", action, target, arg) - logg.level = oldlevel - edit_image(inp, out, commands) \ No newline at end of file diff --git a/lib/bash-tpl b/lib/bash-tpl index 4e9bf11..4eb8718 100755 --- a/lib/bash-tpl +++ b/lib/bash-tpl @@ -7,7 +7,7 @@ # See the accompanying LICENSE file, if present, or visit: # https://opensource.org/licenses/MIT ####################################################################### -VERSION="v0.7.1" +VERSION="v0.8.0" ####################################################################### # Bash-TPL: A Smart, Lightweight shell script templating engine # @@ -272,7 +272,7 @@ function reset_template_regexes() { d="${DIRECTIVE_DELIM}" escape_regex d - DIRECTIVE_REGEX="^([[:blank:]]*)${d}([a-zA-Z_-]+)(.*)\$" + DIRECTIVE_REGEX="^([[:blank:]]*)${d}([a-zA-Z_-]+\??)(.*)\$" d="${COMMENT_DELIM}" escape_regex d @@ -343,6 +343,26 @@ function reset_template_regexes() { function trim() { read -r "$1" <<< "${!1}"$'\n' } +## +# escape_string +# Escapes value compatible with posix `printf %b` +# usage: escape_string varname +# +function escape_string { + # Escape '\' first since we'll be adding more later + local e="${!1//$'\\'/\\}" + # Some man pages mention \0NNN but in practice it seems \NNN is also works. + e="${e//\\/\\\\}" + e="${e//\'/\\0047}" + e="${e//$'\a'/\\a}" + e="${e//$'\b'/\\b}" + e="${e//$'\f'/\\f}" + e="${e//$'\n'/\\n}" + e="${e//$'\r'/\\r}" + e="${e//$'\t'/\\t}" + e="${e//$'\v'/\\v}" + printf -v "${1}" "'%s'" "${e}" +} ## # escape_regex @@ -517,26 +537,32 @@ function print_text() { # $2 = full line of text to process # function process_tags() { - local stmt_indent line args arg quoted + local stmt_indent line formats args arg quoted stmt_indent="${1}" line="${2}" - args="" + formats="" + args=() while [ -n "${line}" ]; do # echo "# LINE @ START: $(declare -p line)" >&2 if [[ "${line}" =~ $TAG_TEXT_REGEX ]]; then # echo "# TEXT TAG MATCH: $(declare -p BASH_REMATCH)" >&2 - printf -v quoted "%q" "${BASH_REMATCH[1]}" - args="${args}${quoted}" + quoted="${BASH_REMATCH[1]}" + escape_string quoted + #printf -v quoted "%q" "${BASH_REMATCH[1]}" + formats="${formats}%b" + args+=("${quoted}") line="${BASH_REMATCH[2]}" elif [[ "${line}" =~ $TAG_QUOTE_REGEX ]]; then # echo "# QUOTE TAG MATCH: $(declare -p BASH_REMATCH)" >&2 - args="${args}\"${BASH_REMATCH[1]}\"" + formats="${formats}%s" + args+=("\"${BASH_REMATCH[1]}\"") line="${BASH_REMATCH[6]}" elif [[ "${line}" =~ $TAG_STATEMENT_REGEX ]]; then # echo "# STMT TAG MATCH: $(declare -p BASH_REMATCH)" >&2 arg="${BASH_REMATCH[1]}" trim arg - args="${args}\"\$(${arg})\"" + formats="${formats}%s" + args+=("\"\$(${arg})\"") line="${BASH_REMATCH[5]}" # Check standard regex last as it's a super-set of quote and stmt regex # @@ -544,21 +570,25 @@ function process_tags() { # echo "# STD TAG MATCH: $(declare -p BASH_REMATCH)" >&2 arg="${BASH_REMATCH[1]}" trim arg - args="${args}\"${arg}\"" + formats="${formats}%s" + args+=("\"${arg}\"") line="${BASH_REMATCH[5]}" # Assume next character is TEXT - extract and process remainder # elif [[ "${line}" =~ (.)(.*) ]]; then - # echo "# DEFAULT: Assuming first char is TEXT: $(declare -p line)" - printf -v quoted "%q" "${BASH_REMATCH[1]}" - args="${args}${quoted}" + # echo "# DEFAULT: Assuming first char is TEXT: $(declare -p line)" >&2 + quoted="${BASH_REMATCH[1]}" + escape_string quoted + #printf -v quoted "%q" "${BASH_REMATCH[1]}" + formats="${formats}%b" + args+=("${quoted}") line="${BASH_REMATCH[2]}" fi # echo "# LINE @ END: $(declare -p line)" >&2 done local stmt - if [ -n "${args}" ]; then - printf -v stmt "printf \"%%s\\\\n\" %s" "${args}" + if [[ ${#args[@]} -gt 0 ]]; then + printf -v stmt "printf \"%s\\\\n\" %s" "${formats}" "${args[*]}" else printf -v stmt "printf \"\\\\n\"" fi @@ -587,7 +617,11 @@ function process_directive() { directive="${2}" normalize_directive directive case "${directive}" in - INCLUDE) + INCLUDE | INCLUDE\?) + local file_not_found_ok="" + if [ "${directive}" = "INCLUDE?" ]; then + file_not_found_ok="1" + fi local indent="${1}" if [[ "${indent}" == "${BLOCK_INDENT}"* ]]; then indent="${indent/#$BLOCK_INDENT/}" @@ -608,6 +642,7 @@ function process_directive() { --txt-delim "${TEXT_DELIM}" \ --dir-delim "${DIRECTIVE_DELIM}" \ --cmt-delim "${COMMENT_DELIM}" \ + ${file_not_found_ok:+'--file-not-found-ok'} \ "${args_arr[@]}" ;; DELIMS) @@ -1216,6 +1251,10 @@ function parse_args() { BASE_BLOCK_INDENT="${2}" shift 2 ;; + --file-not-found-ok) # internal flag to support .INCLUDE? + FILE_NOT_FOUND_OK=1 + shift + ;; -) __ARGS+=("$1") shift @@ -1250,6 +1289,11 @@ function main() { reset_delims parse_env_delims + FILE_NOT_FOUND_OK="" + if [ -n "${BASH_TPL_FILE_NOT_FOUND_OK}" ]; then + FILE_NOT_FOUND_OK="1" + fi + parse_args "$@" set -- "${__ARGS[@]}" unset __ARGS @@ -1272,8 +1316,13 @@ function main() { # File argument points to non-existing/readable file # if [[ ! -r "${1}" ]]; then - echo "File not found: '${1}'" >&2 - exit 1 + if [ -z "${FILE_NOT_FOUND_OK}" ]; then + echo "File not found: '${1}'" >&2 + exit 1 + else + # Fail silently, no message, no error code + exit 0 + fi fi # File argument is good, re-route it to stdin # diff --git a/lib/bash-tpl.update b/lib/bash-tpl.update new file mode 100755 index 0000000..ac19dff --- /dev/null +++ b/lib/bash-tpl.update @@ -0,0 +1,4 @@ +#!/bin/bash +# /opt/python/apps/bin/lastversion --assets --output /shell/base/modules/scripting/btpl.lib download bash-tpl +wget -O bash-tpl https://raw.githubusercontent.com/TekWizely/bash-tpl/main/bash-tpl +# sed -i 's/\bmain\b/btpl/g' /shell/base/modules/scripting/btpl.lib \ No newline at end of file diff --git a/lib/build.lib b/lib/build.lib index 97b0ad1..a437d61 100755 --- a/lib/build.lib +++ b/lib/build.lib @@ -127,60 +127,101 @@ source_env_file () { } load_csv () { - # add newline, remove comments, remove empty lines, remove extra whitespace around , + # add newline, remove comments, remove empty lines, remove leading whitepace, remove extra whitespace around , if [[ -f $1 ]]; then sed -e '$a\' "$1" | \ sed -e '/\s*#.*$/d' | \ sed -e '/^\s*$/d' | \ + sed 's/^\s*//g' | \ sed 's/\s*,\s*/,/g' else return 1 fi } -get_default_distro_image () { -local distro -distro="$(echo "$(load_csv $BDIR/distros.csv)" | grep $LINUX_DISTRO)" -echo $distro | cut -d',' -f2 +get_distro_core_image_name () { + local distro; local imagename + distro=${1:-$LINUX_DISTRO} + if [[ $distro ]]; then + imagename=$(echo "$(load_csv $BDIR/distros.csv)" | grep "^${distro}," | cut -f2 -d, | sed "s/\s/|/g") + [[ $imagename ]] && echo $imagename || return 1 + else + return 2 + fi } -validate_image_distro() { -local temp=/tmp/os-release.tmp -local distro; local distros -if docker create --name dummy $1 > /dev/null; then - if docker cp -L dummy:/etc/os-release $temp > /dev/null; then - docker rm -f dummy > /dev/null - # echo $(load_csv $BDIR/distros.csv) - distros=$(echo $(echo "$(load_csv $BDIR/distros.csv)" | grep -Eo "^[^,]+") | sed "s/\s/|/g") - distro=$(cat $temp | tr [:upper:] [:lower:] | grep -Eio -m 1 $distros) - rm $temp - [[ ! $distro ]] && echo "image $1 is not a valid distro ($distros)" && return 1 - [[ ! "$distro" == "${2:-$LINUX_DISTRO}" ]] && echo "image ${1}'s distro ($distro) is NOT build distro (${2:-$LINUX_DISTRO})" && return 1 - quiet echo "base image $1 distro ($distro) has been validated" - else - echo "unable to retreive /etc/os-release from image $1, unable to determine image distro" +get_distro_from_image () { + local temp=/tmp/os-release.tmp + local distro + local keep + [[ $1 == "-k" ]] && (keep=true;shift) + [[ ! $1 ]] && return 1 + if docker create --name dummy $1 > /dev/null 2>&1; then + if docker cp -L dummy:${2:-/etc/os-release} $temp > /dev/null 2>&1; then + docker rm -f dummy > /dev/null + distro=$(cat $temp | grep "ID_LIKE=" | cut -f2 -d=) + if [[ ! "$distro" ]]; then + distro=$(cat $temp | grep "^ID=" | cut -f2 -d=) + fi + [[ "$distro" ]] && echo $distro || return 2 + fi + else + return 1 + fi + [[ ! $keep ]] && docker image rm $1 > /dev/null 2>&1 +} + +validate_distro() { +# only valid distros are ones in distros.csv +local distro; local set_distro; +distro=${1:-$LINUX_DISTRO} +if [[ $distro ]]; then + if [[ $BASE_IMAGE ]]; then + >&2 echo "FATAL: cannot specifiy both BASE_IMAGE ($BASE_IMAGE) and a LINUX_DISTRO ($LINUX_DISTRO), aborting build" + return 2 fi else - echo "there is no image $1 locally or at docker hub, can't set the base image" - return 1 -fi -} - -get_base_image() { - -[[ ! $BASE_IMAGE ]] && BASE_IMAGE=$(get_default_distro_image) -if [[ $BASE_IMAGE ]]; then - quiet echo determining DISTRO of base image: $BASE_IMAGE - if ! validate_image_distro $BASE_IMAGE; then - echo "unable to get or use base image: $BASE_IMAGE, aborting build" && return 5 + if [[ $BASE_IMAGE ]]; then + echo validate from base image + if ! distro=$(get_distro_from_image $BASE_IMAGE); then + >&2 echo "ERROR: unable to get distro from BASE_IMAGE $BASE_IMAGE, can't validate" + return 2 + fi + set_distro=true + else + echo "WARNING: neither LINUX_DISTRO nor BASE_IMAGE image specified" + echo "Setting LINUX_DISTRO to default (alpine)" + set_distro=true + distro=alpine fi - quiet echo $BASE_IMAGE is built from distro $LINUX_DISTRO - else - echo unable to determine a base image, aborting build - return 6 -fi +fi + +distros=$(echo $(echo "$(load_csv $BDIR/distros.csv)" | grep -Eo "^[^,]+") | sed "s/\s/|/g" | tr '[:upper:]' '[:lower:]') +if [[ ! "$distros" == *"${distro}"* ]]; then + >&2 echo "distro $distro is not a valid uci-docker-build distro ($distros)" + return 1 +fi +[[ ! $BASE_NAME ]] && BASE_IMAGE=$(get_distro_core_image_name $distro) +if [[ $set_distro ]] && [[ ! "$1" ]]; then LINUX_DISTRO=$distro; fi + } +# delete +# get_base_image() { + +# [[ ! $BASE_IMAGE ]] && BASE_IMAGE=$(get_default_distro_image) +# if [[ $BASE_IMAGE ]]; then +# quiet echo determining DISTRO of base image: $BASE_IMAGE +# if ! validate_image_distro $BASE_IMAGE; then +# echo "unable to get or use base image: $BASE_IMAGE, aborting build" && return 5 +# fi +# quiet echo $BASE_IMAGE is built from distro $LINUX_DISTRO +# else +# echo unable to determine a base image, aborting build +# return 6 +# fi +# } + make_image_name () { local arch @@ -269,13 +310,19 @@ if [[ $VERBOSE ]]; then pushd "$BDIR" > /dev/null || return 3 docker buildx bake --print $TARGET popd > /dev/null || return 4 - echo -e "\n---------------------------------" - echo "build source at $BUILD_SRC to be mounted to /build in container ***** " - ls -la $BUILD_SRC - echo -e "\n----- base init script init.sh ------\n" - cat $BUILD_SRC/init/init.sh - echo -e "\n----- end base init script init.sh ------" - echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + if [[ $BUILD_SRC == "_core_" ]]; then + echo building only core + cat $BDIR/core/core.sh + ls -la $BDIR/core + else + echo -e "\n---------------------------------" + echo "build source at $BUILD_SRC to be mounted to /build in container ***** " + ls -la $BUILD_SRC + echo -e "\n----- base init script init.sh ------\n" + cat $BUILD_SRC/init/init.sh + echo -e "\n----- end base init script init.sh ------" + echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + fi fi echo -e "\e[1;37m**************BUILD PARAMETERS *******************************" diff --git a/readme.md b/readme.md index 10cc6eb..b6e5592 100644 --- a/readme.md +++ b/readme.md @@ -31,4 +31,8 @@ One can make images in one of two ways. It is recommended to do the later. -Supported distros are found in distros.csv in the root of the repository. Do NOT delete this file. It is possible to add other distros. This file links itself into lib/ and core/opt/lib \ No newline at end of file +Supported distros are found in distros.csv in the root of the repository. Do NOT delete this file. It is possible to add other distros. This file links itself into lib/ and core/opt/lib + +TODO need more details (docs folder?) + +to update the uci shell/base cd to /core/shell/base, then `git pull origin master` \ No newline at end of file diff --git a/test/build b/test/build index 9b3aeca..feca52c 100755 --- a/test/build +++ b/test/build @@ -3,7 +3,7 @@ if [[ ! $(udbuild image exists -e test.env) || $force ]] ; then echo $force building test image # udbuild -p -e test.env -n - udbuild -e test.env + udbuild -e test.env "$@" else echo using existing image, use -f to force rebuild fi diff --git a/test/test.env b/test/test.env index 9b97245..0aa2e9f 100644 --- a/test/test.env +++ b/test/test.env @@ -1,8 +1,10 @@ -VERBOSE=true +# VERBOSE=true +# VERBOSE=core # if SYSADMIN_PW is set a sysadmin user with UHID of 1001 will be creted # SYSADMIN_PW=ucommandit # default is alpine -# LINUX_DISTRO=alpine +# LINUX_DISTRO=arch +# BASE_IMAGE=archlinux # prepend image name with this user, typically your docker hub user # RUSER=ucommandit RUSER=testing @@ -10,5 +12,6 @@ RUSER=testing # TARGET=dev # by default will look in PWD directory then parent # BUILD_SRC=src -# APPEND_BUILD_ENV=./build.env +# BUILD_SRC="_core_" +APPEND_BUILD_ENV=./build.env diff --git a/test/try b/test/try index 09bfbcc..f5773fe 100755 --- a/test/try +++ b/test/try @@ -1,3 +1,3 @@ source ./build "$1" [[ $force ]] && shift 1 -udbuild try -e test.env -f try.env -m opt ${@:-shell} $@ \ No newline at end of file +udbuild try -e test.env -m opt ${@:-shell} $@ \ No newline at end of file