move environment functions to env module from file module and helpers module

feat env_subs_file and support functions now operational
added shell_run function to module.lib so scripts can be run with the otherwise not loaded current shell
added remove empty lines function to helpers

load.sh

@@ -49,7 +49,7 @@ load_shell_host_user_dev () {
   load_shell_dev
 }
 
-###  START LOAD SCRIPT #####
+######################  START LOAD SCRIPT #############################
 
 load_custom_shell_env
 

module.lib

@@ -1,6 +1,8 @@
 #!/bin/bash
 # echo loading module library
 
+# source with set -a and set +a  to have these bootstrap functions available in the environment 
+
 module_confirm() {
 
 local FILE
@@ -55,7 +57,7 @@ fi
 
 # BASH_SHELL_DIRS holds shell directories to search for modules.  If will be searched
 # in revsere order, user or network repos over host over base
-DIRS=( ${BASH_SHELL_DIRS:-$BASH_SHELL_BASE} )
+DIRS=( ${BASH_SHELL_DIRS:-$BASH_SHELL_BASE} ${BASH_SHELL_NETWORKS_LOADED} )
 [ -d BASH_SHELL_USER_DIR ] && DIRS=("${DIRS[@]}" "BASH_SHELL_USER_DIR")
 [ -d $BASH_SHELL_DEV ] && DIRS=("${DIRS[@]}" "$BASH_SHELL_DEV")
 
@@ -126,7 +128,14 @@ module_loaded() {
    return $ret
 }
 
-# exported for login environment
+# put here so it's globally available
+# runs a script but loads the current interactive shell environment, not recommended unless there is a need
+# mostly for testing.  Better to source some part of the shell as needed using shell-process-directory module
+shell_run () {
+bash -ci ". ${1}"
+}
+
+# can one off export a function
 # declare -f -x module_load
 # declare -f -x module_find
 # declare -f -x module_confirm

modules/network-dirs.sh

@@ -1,40 +0,0 @@
-# to be sourced (below) not run in subshell!
-function network_dirs () {
-
-  (return 0 2>/dev/null) || (echo network_dirs must be sourced; return 1)
-  
-  module_load path 
-  [[ $? -ne 0 ]] && echo unable to access the file module, aboarting load && return 1
-
-
-  local network
-  [[ ! -f $1 ]] && return
-
-  while read -r network || [ -n "$network" ]
-   do
-     if [[ ${network} = *!* ]]; then 
-     network=${network/!/}
-     NETWORK_HOME=$network
-    fi
-    [[ $2 ]] && network=$2/$network
-    if [[ $(isAbsPath $network) ]]; then
-     [[ ! -d $network ]] && continue
-     else 
-       if [[ -d ${BASH_SHELL_USER}/${network} ]];then 
-      network=${BASH_SHELL_USER}/${network}
-      else 
-      if [[ -d ${BASH_NETWORKS_DIR}/${network} ]];then 
-        network=${BASH_NETWORKS_DIR}/${network}
-        else 
-        continue
-       fi 
-      fi  
-    fi
-    [[ $network =~ $NETWORK_HOME ]] && NETWORK_HOME_DIR=${network}
-    BASH_SHELL_NETWORK_DIRS+="${network} "
-    BASH_SHELL_DIRS+="${network} "
-  done < "$1"
-  export NETWORK_HOME
-  export NETWORK_HOME_DIR
-  export BASH_SHELL_NETWORK_DIRS
-}

modules/scripting/helpers.mod

@@ -60,16 +60,19 @@ parse_option () {
   [[ $f ]] && echo "$2 $ret" || echo $ret
 }
 
+remove_empty_lines () {
+if [[ -f $1 ]]; then cat $1; else echo "$1"; fi | sed -rz 's/^\n+//; s/\n+$/\n/g'
+}
+
 remove_end_spaces () {
     del=${2:-\'}
     # echo delimiter: $del
     # sed -e "s/[[:space:]]\{1,\}$del/$del/" <<< "$1"
     res=$(sed -e "s/^$del[[:space:]]*/$del/" <<< "$1")
-    # echo leading: $res
     sed -e "s/[[:space:]]*${del}$/$del/" <<< "$res"
 }
 
-# pass any sed ' ' string and command lines will be ignored
+# pass any sed ' ' string and comments lines will be ignored
 # https://unix.stackexchange.com/a/301432/201387
 # https://stackoverflow.com/a/35874420/4695378
 sed_ignore_comments () {
@@ -80,48 +83,3 @@ sed_ignore_comments () {
         eval $cmd < /dev/stdin
     fi
 }
-
-
-# https://stackoverflow.com/questions/19331497/set-environment-variables-from-file-of-key-value-pairs
-env_file () {
-   env=${1:-.env}
-   [[ -f "${env}" ]] && { echo $env; return 0; } # || echo not $env
-   [[ -f "${env}.env" ]] && { echo "${env}.env"; return 0; } # || echo not ${env}.env
-   [[ -f "${env}/.env" ]] && { echo "${env}/.env"; return 0; } # || echo not ${env}/.env
-   return 1
-}  
-  
-# ignore non assignment lines
-list_env_file_vars () {
-  env=$(env_file $1)
-  [[ ! $env ]] && { iecho "Env file ${1} doesn't exist"; return 1; }
-  echo $(grep -v '^#' ${env} | sed -n -E 's/(.*)=[^ =].*/\1/p' | xargs)
-}
-
-list_env_file_vals() {
-   vars=$(list_env_file_vars $1)
-   [[ $? -gt 0 ]] && return 1
-   for var in $(list_env_file_vars $1) ;
-   do
-   echo "$var=${!var}"
-   done
-}
-
-source_env_file() {
-  env=$(env_file $1)
-  [[ ! $env ]] && { iecho "Env file ${1} doesn't exist"; return 1; }
-  source $env
-}
-
-export_env_file () {
-  env=$(env_file $1)
-  [[ ! $env ]] && { iecho "Env file ${1} doesn't exist"; return 1; }
-  set -a; source $env; set +a
-}
-
-unset_env_file () {
-  env=$(env_file $1)
-  [[ ! $env ]] && { iecho "Env file ${1} doesn't exist"; return 1; }
-  unset $(grep -v '^#' ${env} | sed -E 's/(.*)=.*/\1/' | xargs)
-}
-

modules/shell-process-directory.mod

@@ -3,7 +3,7 @@
 shell_source_subdirs () {
           # default processing
           [[ ! -d $1 ]] && echo no directory $1 from which to process subdirectories && return 1
-          local DIR=
+          local DIR
           local SUBDIRS
           local IGNORE_FILE
           DIR=$1

modules/utility/env.lib

@@ -0,0 +1,104 @@
+#!/bin/bash 
+
+# environment and environment file functions
+
+clean_env_file () {
+  local compact
+  module_load helpers 
+  # -c will also remove comments and all empty lines
+  [[ $1 == "-c" ]] && compact=true && shift 1
+  # 
+  #  remove trailing spaces | remove double quotes 
+  # | remove blanks after equals and quote value | remove any spaces before afer =
+  # remove blank lines | remove comment lines if requested
+  cat $1 | sed_ignore_comments s/\\s*$//g  | sed_ignore_comments s/\"//g \
+  | sed_ignore_comments s/\(=[[:blank:]]*\)\(.*\)/\\1\"\\2\"/  \
+  | sed_ignore_comments s/\\s*=\\s*/=/g \
+  | sed -rz 's/^\n+//; s/\n+$/\n/g' | if [[ $compact ]]; then grep -v '^#' | grep -v "^$" ; else cat; fi 
+}
+
+# https://stackoverflow.com/questions/19331497/set-environment-variables-from-file-of-key-value-pairs
+env_file () {
+   env=${1:-.env}
+   [[ -f "${env}" ]] && { echo $env; return 0; } # || echo not $env
+   [[ -f "${env}.env" ]] && { echo "${env}.env"; return 0; } # || echo not ${env}.env
+   [[ -f "${env}/.env" ]] && { echo "${env}/.env"; return 0; } # || echo not ${env}/.env
+   return 1
+}  
+  
+# ignore non assignment lines
+list_env_file_vars () {
+  env=$(env_file $1)
+#   cat $env
+  [[ ! $env ]] && { iecho "Env file ${1} doesn't exist"; return 1; }
+  clean_env_file $env | grep -v '^#'| sed -n -E 's/(.*)=[^ =].*/\1/p' | xargs
+}
+
+function load_env_file() {
+  # second argument, "-x"  will export within function (or anything called), "-gx" is equivalent of export and is global
+  env=$(env_file $1)
+  [[ ! $env ]] && { iecho "Env file ${1} doesn't exist"; return 1; }
+  while read line; do
+  # echo var to add $line
+  declare $2 "$(echo "${line}" | sed 's/\"//g' )"
+  done < <(clean_env_file -c $env) 
+}
+
+# list_env_file_vals() {
+#    local vars
+#    vars=$(list_env_file_vars $1)
+#    echo $vars
+#    [[ $? -gt 0 ]] && return 1
+#    export_env_file $1
+#    for var in $vars ;
+#    do
+#    "$var=${!var}"
+#    done
+# }
+
+export_env_file () {
+  load_env_file $1 -gx
+}
+
+unset_env_file () {
+  env=$(env_file $1)
+  [[ ! $env ]] && { iecho "Env file ${1} doesn't exist"; return 1; }
+  unset $(clean_env_file -c ${env} | sed -E 's/(.*)=.*/\1/' | xargs)
+}
+
+
+testenv () {
+    echo in testenv
+    echo TEST Variable $TEST
+    env | grep TEST
+    echo done testenv
+}
+
+# https://www.baeldung.com/linux/envsubst-command
+# todo allow input file -o overwrite or to another file -f <name>
+env_subs_file () {
+ local fin; local fenv; local useenv; local tout; local fout; local fenv; local tout2
+ [[ $1 == "-e" ]] && useenv=true && shift 1
+ [[ $1 == "-o" ]] && { shift 1; fout=$1; }
+ [[ $1 == "-f" ]] && { fout=$2; shift 2; }
+ fin=$1; fenv=$2
+#  echo useenv $useenv, fout $fout, fin  $fin, fenv $fenv
+ [[ ! $fin ]] && iecho "nothing was passed to merge with environment" && return 1
+ [[ ! -f "$fin" ]] && iecho "file $fin does not exit nothing to merge with environment" && return 1
+ [[ ! $(cat $fin | grep -F "$") ]] && iecho "no variables in $fin to merge" && return 2
+ tout=/tmp/${USER}-mergedenvfile
+ if [[ $useenv ]] && [[ -f $fenv ]]; then 
+  tout2=/tmp/${USER}-mergedenvfile2
+  cat $fin | envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1) )" > $tout2
+  fin=$tout2
+ fi 
+ if [[ -f $fenv ]]; then
+    (export_env_file $fenv; cat $fin | envsubst "$(printf '${%s} ' $(clean_env_file -c $fenv | cut -d'=' -f1) )") 
+   else
+    cat $fin | envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1) )" 
+ fi > $tout
+ [[ -f $tout2 ]] && rm -f  $tout2
+ [[ $fout ]] && mv -f $tout $fout || { echo merged; cat "$tout"; }
+#  env | grep BASE
+# #  unset_env_file $2
+}

modules/utility/file.lib

@@ -14,49 +14,6 @@ isFile() {
     fi
 }
 
-#TODO
- # env_file_pretty () {
- # }
-
-# https://www.baeldung.com/linux/envsubst-command
-env_subs_file () {
- module_load helpers 
- local input; local fenv; local all
- [[ $1 == "-a" ]] && all=true && shift 1
- input=$1
- fenv=$2
- [[ ! $input ]] && iecho "nothing was passed to merge with environment" && return 1
- [[ -f "$input" ]] && input=$(cat "$input")
- [[ ! $(echo $input | grep -F "$") ]] && iecho "nothing to merge in $1" && return 2
- if [[ ($2 && ! -f $2) ]]; then
-  iecho "no enviroment file at $2" && return 3
- else 
-  # [[ $all ]] && export_file $fenv && unset fenv
-  if [[ $all ]]; then
-   fenv="/tmp/$USER/_tenvf" 
-   # put quotes around values to avoid issues
-   env | sed 's/\(=[[:blank:]]*\)\(.*\)/\1"\2"/' > $fenv 
-   cat $2 | sed 's/\(=[[:blank:]]*\)\(.*\)/\1"\2"/' | sed 's/\s*=\s*/=/g'
-   return
-   cat "$2" | sed 's/\(=[[:blank:]]*\)\(.*\)/\1"\2"/' >> $fenv
-  fi  
- fi
- if [[ $fenv ]]; then
-    (set -a;source "$fenv" ;set +a; echo -e "$input" | envsubst "$(printf '${%s} ' $(cat "$fenv" | cut -d'=' -f1) )")  
-  else
-      echo -e "$input" | envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1) )" 
- fi 
- [[ $all ]] && rm -f  $fenv
-# [[ $all ]] && unset_file $2
-}
-
-
-#  shift 1
-#  input=$1
-#  set -a;. $2 ;set +a;
-#  echo env file is now $2
-#  fi
-
 build_file () {
   # local file
   # echo build file $1 $2
@@ -73,6 +30,7 @@ build_file () {
   fi
 }
 
+# TODO need to test
 function lines_2_str () {
   [[ ! -f "$1" ]] && return 1
   local str=''
@@ -83,13 +41,11 @@ function lines_2_str () {
     # echo line: "${line}" >&2
     str="$str ${line}"
    # echo str: $str >&2
-done < $1
+  done < $1
   # for line in $lines ; do
   #    str+='"'$line'" '
   # done
   echo $str
-  # set +o noglob
-  return 0
 }
 
 # find, superceeds find use `command find` to get the super

shell.env

@@ -78,7 +78,7 @@ declare parent
 parent="$(dirname $BASH_SHELL_BASE)"
 
 # Any Host
-BASH_SHELL_ANY_HOST="${parent}/any/host  # or set to"
+BASH_SHELL_ANY_HOST="${parent}/any/host"
 # export BASH_SHELL_ALL_HOSTS=<some directory>
 [[ -d $BASH_SHELL_ANY_HOST ]] && BASH_SHELL_DIRS+="$BASH_SHELL_ANY_HOST " && export BASH_SHELL_ANY_HOST
 
@@ -104,20 +104,23 @@ BASH_SHELL_DEV=${BASH_SHELL_DEV:-/opt/shell}
 export BASH_SHELL_USER_DIR=$HOME/shell
 
 
+
+# #################### NETWORKING #########################
+
 # set up default location to look for other network shell directories
 BASH_NETWORKS_DIR=$parent/networks
 # BASH_NETWORKS_DIR=<some directory>  # alt directory to look for networks
 export BASH_NETWORKS_DIR
 
-# set one or the other of these here for all users or put in shell.env under HOST/USER/DEV directories
+# set one or the other of these in custom shell.env under HOST/USER/DEV directories
 # if you want to load networks with every shell
 # export BASH_SHELL_HOME_NETWORK_LOAD=true
 # export BASH_SHELL_NETWORKS_LOAD=true
 # export BASH_SHELL_NETWORK=<some directory>
 
 
-# now load and export module loading functions library so it is always available
+# now load and export bootstrap module loading functions library so it is always available
-source $BASH_SHELL_BASE/module.lib
+set -a;source $BASH_SHELL_BASE/module.lib;set +a
 
 # echo done shell.env