From 96127ca89f0754f554de4941b391c637f30d2205 Mon Sep 17 00:00:00 2001 From: "kebler.net" Date: Thu, 5 Jan 2023 12:36:23 -0800 Subject: [PATCH] move environment functions to env module from file module and helpers module feat env_subs_file and support functions now operational added shell_run function to module.lib so scripts can be run with the otherwise not loaded current shell added remove empty lines function to helpers --- load.sh | 2 +- module.lib | 13 +++- modules/network-dirs.sh | 40 ----------- modules/scripting/helpers.mod | 52 ++------------ modules/shell-process-directory.mod | 2 +- modules/utility/env.lib | 104 ++++++++++++++++++++++++++++ modules/utility/file.lib | 48 +------------ shell.env | 11 +-- 8 files changed, 131 insertions(+), 141 deletions(-) delete mode 100644 modules/network-dirs.sh create mode 100644 modules/utility/env.lib diff --git a/load.sh b/load.sh index 0f571bf..44e32f0 100755 --- a/load.sh +++ b/load.sh @@ -49,7 +49,7 @@ load_shell_host_user_dev () { load_shell_dev } -### START LOAD SCRIPT ##### +###################### START LOAD SCRIPT ############################# load_custom_shell_env diff --git a/module.lib b/module.lib index 960e8ea..fec1859 100644 --- a/module.lib +++ b/module.lib @@ -1,6 +1,8 @@ #!/bin/bash # echo loading module library +# source with set -a and set +a to have these bootstrap functions available in the environment + module_confirm() { local FILE @@ -55,7 +57,7 @@ fi # BASH_SHELL_DIRS holds shell directories to search for modules. If will be searched # in revsere order, user or network repos over host over base -DIRS=( ${BASH_SHELL_DIRS:-$BASH_SHELL_BASE} ) +DIRS=( ${BASH_SHELL_DIRS:-$BASH_SHELL_BASE} ${BASH_SHELL_NETWORKS_LOADED} ) [ -d BASH_SHELL_USER_DIR ] && DIRS=("${DIRS[@]}" "BASH_SHELL_USER_DIR") [ -d $BASH_SHELL_DEV ] && DIRS=("${DIRS[@]}" "$BASH_SHELL_DEV") @@ -126,7 +128,14 @@ module_loaded() { return $ret } -# exported for login environment +# put here so it's globally available +# runs a script but loads the current interactive shell environment, not recommended unless there is a need +# mostly for testing. Better to source some part of the shell as needed using shell-process-directory module +shell_run () { +bash -ci ". ${1}" +} + +# can one off export a function # declare -f -x module_load # declare -f -x module_find # declare -f -x module_confirm diff --git a/modules/network-dirs.sh b/modules/network-dirs.sh deleted file mode 100644 index 7cfde7b..0000000 --- a/modules/network-dirs.sh +++ /dev/null @@ -1,40 +0,0 @@ -# to be sourced (below) not run in subshell! -function network_dirs () { - - (return 0 2>/dev/null) || (echo network_dirs must be sourced; return 1) - - module_load path - [[ $? -ne 0 ]] && echo unable to access the file module, aboarting load && return 1 - - - local network - [[ ! -f $1 ]] && return - - while read -r network || [ -n "$network" ] - do - if [[ ${network} = *!* ]]; then - network=${network/!/} - NETWORK_HOME=$network - fi - [[ $2 ]] && network=$2/$network - if [[ $(isAbsPath $network) ]]; then - [[ ! -d $network ]] && continue - else - if [[ -d ${BASH_SHELL_USER}/${network} ]];then - network=${BASH_SHELL_USER}/${network} - else - if [[ -d ${BASH_NETWORKS_DIR}/${network} ]];then - network=${BASH_NETWORKS_DIR}/${network} - else - continue - fi - fi - fi - [[ $network =~ $NETWORK_HOME ]] && NETWORK_HOME_DIR=${network} - BASH_SHELL_NETWORK_DIRS+="${network} " - BASH_SHELL_DIRS+="${network} " - done < "$1" - export NETWORK_HOME - export NETWORK_HOME_DIR - export BASH_SHELL_NETWORK_DIRS -} \ No newline at end of file diff --git a/modules/scripting/helpers.mod b/modules/scripting/helpers.mod index c376822..1a492b5 100644 --- a/modules/scripting/helpers.mod +++ b/modules/scripting/helpers.mod @@ -60,16 +60,19 @@ parse_option () { [[ $f ]] && echo "$2 $ret" || echo $ret } +remove_empty_lines () { +if [[ -f $1 ]]; then cat $1; else echo "$1"; fi | sed -rz 's/^\n+//; s/\n+$/\n/g' +} + remove_end_spaces () { del=${2:-\'} # echo delimiter: $del # sed -e "s/[[:space:]]\{1,\}$del/$del/" <<< "$1" res=$(sed -e "s/^$del[[:space:]]*/$del/" <<< "$1") - # echo leading: $res sed -e "s/[[:space:]]*${del}$/$del/" <<< "$res" } -# pass any sed ' ' string and command lines will be ignored +# pass any sed ' ' string and comments lines will be ignored # https://unix.stackexchange.com/a/301432/201387 # https://stackoverflow.com/a/35874420/4695378 sed_ignore_comments () { @@ -80,48 +83,3 @@ sed_ignore_comments () { eval $cmd < /dev/stdin fi } - - -# https://stackoverflow.com/questions/19331497/set-environment-variables-from-file-of-key-value-pairs -env_file () { - env=${1:-.env} - [[ -f "${env}" ]] && { echo $env; return 0; } # || echo not $env - [[ -f "${env}.env" ]] && { echo "${env}.env"; return 0; } # || echo not ${env}.env - [[ -f "${env}/.env" ]] && { echo "${env}/.env"; return 0; } # || echo not ${env}/.env - return 1 -} - -# ignore non assignment lines -list_env_file_vars () { - env=$(env_file $1) - [[ ! $env ]] && { iecho "Env file ${1} doesn't exist"; return 1; } - echo $(grep -v '^#' ${env} | sed -n -E 's/(.*)=[^ =].*/\1/p' | xargs) -} - -list_env_file_vals() { - vars=$(list_env_file_vars $1) - [[ $? -gt 0 ]] && return 1 - for var in $(list_env_file_vars $1) ; - do - echo "$var=${!var}" - done -} - -source_env_file() { - env=$(env_file $1) - [[ ! $env ]] && { iecho "Env file ${1} doesn't exist"; return 1; } - source $env -} - -export_env_file () { - env=$(env_file $1) - [[ ! $env ]] && { iecho "Env file ${1} doesn't exist"; return 1; } - set -a; source $env; set +a -} - -unset_env_file () { - env=$(env_file $1) - [[ ! $env ]] && { iecho "Env file ${1} doesn't exist"; return 1; } - unset $(grep -v '^#' ${env} | sed -E 's/(.*)=.*/\1/' | xargs) -} - diff --git a/modules/shell-process-directory.mod b/modules/shell-process-directory.mod index 87091e1..e2d44fc 100644 --- a/modules/shell-process-directory.mod +++ b/modules/shell-process-directory.mod @@ -3,7 +3,7 @@ shell_source_subdirs () { # default processing [[ ! -d $1 ]] && echo no directory $1 from which to process subdirectories && return 1 - local DIR= + local DIR local SUBDIRS local IGNORE_FILE DIR=$1 diff --git a/modules/utility/env.lib b/modules/utility/env.lib new file mode 100644 index 0000000..c865ec9 --- /dev/null +++ b/modules/utility/env.lib @@ -0,0 +1,104 @@ +#!/bin/bash + +# environment and environment file functions + +clean_env_file () { + local compact + module_load helpers + # -c will also remove comments and all empty lines + [[ $1 == "-c" ]] && compact=true && shift 1 + # + # remove trailing spaces | remove double quotes + # | remove blanks after equals and quote value | remove any spaces before afer = + # remove blank lines | remove comment lines if requested + cat $1 | sed_ignore_comments s/\\s*$//g | sed_ignore_comments s/\"//g \ + | sed_ignore_comments s/\(=[[:blank:]]*\)\(.*\)/\\1\"\\2\"/ \ + | sed_ignore_comments s/\\s*=\\s*/=/g \ + | sed -rz 's/^\n+//; s/\n+$/\n/g' | if [[ $compact ]]; then grep -v '^#' | grep -v "^$" ; else cat; fi +} + +# https://stackoverflow.com/questions/19331497/set-environment-variables-from-file-of-key-value-pairs +env_file () { + env=${1:-.env} + [[ -f "${env}" ]] && { echo $env; return 0; } # || echo not $env + [[ -f "${env}.env" ]] && { echo "${env}.env"; return 0; } # || echo not ${env}.env + [[ -f "${env}/.env" ]] && { echo "${env}/.env"; return 0; } # || echo not ${env}/.env + return 1 +} + +# ignore non assignment lines +list_env_file_vars () { + env=$(env_file $1) +# cat $env + [[ ! $env ]] && { iecho "Env file ${1} doesn't exist"; return 1; } + clean_env_file $env | grep -v '^#'| sed -n -E 's/(.*)=[^ =].*/\1/p' | xargs +} + +function load_env_file() { + # second argument, "-x" will export within function (or anything called), "-gx" is equivalent of export and is global + env=$(env_file $1) + [[ ! $env ]] && { iecho "Env file ${1} doesn't exist"; return 1; } + while read line; do + # echo var to add $line + declare $2 "$(echo "${line}" | sed 's/\"//g' )" + done < <(clean_env_file -c $env) +} + +# list_env_file_vals() { +# local vars +# vars=$(list_env_file_vars $1) +# echo $vars +# [[ $? -gt 0 ]] && return 1 +# export_env_file $1 +# for var in $vars ; +# do +# "$var=${!var}" +# done +# } + +export_env_file () { + load_env_file $1 -gx +} + +unset_env_file () { + env=$(env_file $1) + [[ ! $env ]] && { iecho "Env file ${1} doesn't exist"; return 1; } + unset $(clean_env_file -c ${env} | sed -E 's/(.*)=.*/\1/' | xargs) +} + + +testenv () { + echo in testenv + echo TEST Variable $TEST + env | grep TEST + echo done testenv +} + +# https://www.baeldung.com/linux/envsubst-command +# todo allow input file -o overwrite or to another file -f +env_subs_file () { + local fin; local fenv; local useenv; local tout; local fout; local fenv; local tout2 + [[ $1 == "-e" ]] && useenv=true && shift 1 + [[ $1 == "-o" ]] && { shift 1; fout=$1; } + [[ $1 == "-f" ]] && { fout=$2; shift 2; } + fin=$1; fenv=$2 +# echo useenv $useenv, fout $fout, fin $fin, fenv $fenv + [[ ! $fin ]] && iecho "nothing was passed to merge with environment" && return 1 + [[ ! -f "$fin" ]] && iecho "file $fin does not exit nothing to merge with environment" && return 1 + [[ ! $(cat $fin | grep -F "$") ]] && iecho "no variables in $fin to merge" && return 2 + tout=/tmp/${USER}-mergedenvfile + if [[ $useenv ]] && [[ -f $fenv ]]; then + tout2=/tmp/${USER}-mergedenvfile2 + cat $fin | envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1) )" > $tout2 + fin=$tout2 + fi + if [[ -f $fenv ]]; then + (export_env_file $fenv; cat $fin | envsubst "$(printf '${%s} ' $(clean_env_file -c $fenv | cut -d'=' -f1) )") + else + cat $fin | envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1) )" + fi > $tout + [[ -f $tout2 ]] && rm -f $tout2 + [[ $fout ]] && mv -f $tout $fout || { echo merged; cat "$tout"; } +# env | grep BASE +# # unset_env_file $2 +} \ No newline at end of file diff --git a/modules/utility/file.lib b/modules/utility/file.lib index bd91279..4f4f89b 100644 --- a/modules/utility/file.lib +++ b/modules/utility/file.lib @@ -14,49 +14,6 @@ isFile() { fi } -#TODO - # env_file_pretty () { - # } - -# https://www.baeldung.com/linux/envsubst-command -env_subs_file () { - module_load helpers - local input; local fenv; local all - [[ $1 == "-a" ]] && all=true && shift 1 - input=$1 - fenv=$2 - [[ ! $input ]] && iecho "nothing was passed to merge with environment" && return 1 - [[ -f "$input" ]] && input=$(cat "$input") - [[ ! $(echo $input | grep -F "$") ]] && iecho "nothing to merge in $1" && return 2 - if [[ ($2 && ! -f $2) ]]; then - iecho "no enviroment file at $2" && return 3 - else - # [[ $all ]] && export_file $fenv && unset fenv - if [[ $all ]]; then - fenv="/tmp/$USER/_tenvf" - # put quotes around values to avoid issues - env | sed 's/\(=[[:blank:]]*\)\(.*\)/\1"\2"/' > $fenv - cat $2 | sed 's/\(=[[:blank:]]*\)\(.*\)/\1"\2"/' | sed 's/\s*=\s*/=/g' - return - cat "$2" | sed 's/\(=[[:blank:]]*\)\(.*\)/\1"\2"/' >> $fenv - fi - fi - if [[ $fenv ]]; then - (set -a;source "$fenv" ;set +a; echo -e "$input" | envsubst "$(printf '${%s} ' $(cat "$fenv" | cut -d'=' -f1) )") - else - echo -e "$input" | envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1) )" - fi - [[ $all ]] && rm -f $fenv -# [[ $all ]] && unset_file $2 -} - - -# shift 1 -# input=$1 -# set -a;. $2 ;set +a; -# echo env file is now $2 -# fi - build_file () { # local file # echo build file $1 $2 @@ -73,6 +30,7 @@ build_file () { fi } +# TODO need to test function lines_2_str () { [[ ! -f "$1" ]] && return 1 local str='' @@ -83,13 +41,11 @@ function lines_2_str () { # echo line: "${line}" >&2 str="$str ${line}" # echo str: $str >&2 -done < $1 + done < $1 # for line in $lines ; do # str+='"'$line'" ' # done echo $str - # set +o noglob - return 0 } # find, superceeds find use `command find` to get the super diff --git a/shell.env b/shell.env index 52f3d83..e59eaf9 100755 --- a/shell.env +++ b/shell.env @@ -78,7 +78,7 @@ declare parent parent="$(dirname $BASH_SHELL_BASE)" # Any Host -BASH_SHELL_ANY_HOST="${parent}/any/host # or set to" +BASH_SHELL_ANY_HOST="${parent}/any/host" # export BASH_SHELL_ALL_HOSTS= [[ -d $BASH_SHELL_ANY_HOST ]] && BASH_SHELL_DIRS+="$BASH_SHELL_ANY_HOST " && export BASH_SHELL_ANY_HOST @@ -104,20 +104,23 @@ BASH_SHELL_DEV=${BASH_SHELL_DEV:-/opt/shell} export BASH_SHELL_USER_DIR=$HOME/shell + +# #################### NETWORKING ######################### + # set up default location to look for other network shell directories BASH_NETWORKS_DIR=$parent/networks # BASH_NETWORKS_DIR= # alt directory to look for networks export BASH_NETWORKS_DIR -# set one or the other of these here for all users or put in shell.env under HOST/USER/DEV directories +# set one or the other of these in custom shell.env under HOST/USER/DEV directories # if you want to load networks with every shell # export BASH_SHELL_HOME_NETWORK_LOAD=true # export BASH_SHELL_NETWORKS_LOAD=true # export BASH_SHELL_NETWORK= -# now load and export module loading functions library so it is always available -source $BASH_SHELL_BASE/module.lib +# now load and export bootstrap module loading functions library so it is always available +set -a;source $BASH_SHELL_BASE/module.lib;set +a # echo done shell.env