add: tools to shell directory includes

feat: refactor repo organization that include corresponding changes to load script, module library plus shell.env
add: python based yaml parser
add:  functions isDir isFile to corresponding modules

.bash-shell-ignore

@@ -1,6 +1,6 @@
 # hidden files and directories are ignored by default
 # i.e. beginning with .
-# no ignores here may begin with .
+# so NO ignores here may begin with .
 *.off
 *.example
 *.tmpl

.bash-shell-include

@@ -4,3 +4,4 @@ env
 misc
 lang
 app
+tools

alias/00-basic

@@ -67,3 +67,6 @@ alias follow="readlink -f"
 
 # Will scrub all and reload only aliases in .bash_aliaes
 alias reloada="unalias -a && source ~/.bash_aliases && compgen -a"
+
+# use this when there are transport errors after unmounting or killing a mount
+alias umountf="sudo umount -l"

env/modules.env

@@ -1,7 +1,6 @@
-# All the directories which contain based modules/libraries
+# Uncomment and add directories to this variable for 
-# module libraries will be look up in this order
+# searching custom directories for shell modules
-# first set any additional modules directories to search (if any)
+# MODULE_DIRS=(some/dir another/dir)
-# MODULE_DIRS=()
 # those will be searched first then /modules under the shell directories
-# user,host,network,base  in that order
+# based on $BASH_SHELL_DIRS
-# this provide a way to override a module with a more specific version
+# this ordering provides a way to override a module with a more specific version

load.sh

@@ -1,33 +1,13 @@
 #!/bin/bash
 
-# [[ ! $- == *i* ]] && exec >> ~/logs/load.log && exec 2>&1
+# PRIMARY SHELL LOAD SCRIPT - RUN FOR ALL INTERACTIVE SHELLS
-# export BASH_DEBUG=true
-function nilog () {
-  [[ ! $- == *i* ]] && echo -e "-----\n $USER $*" &>> ~/logs/load.log
-}
-
-nilog "called load.sh  $(date)"
-# nilog caller: $(caller)
-# nilog pid: $(ps -o comm= $PPID)
 
 # don't bother if bash is not the shell
 [ ! "$SHELL" = "/bin/bash" ] && return 1
 
-DIR=${1:-$(dirname ${BASH_SOURCE[0]})}
+# ************ be sure module_load function is availABLE ***************************
-
 [[ $(declare -F | grep module_load) ]] || source "$BASH_SHELL_BASE/module.lib"
 
-# uncomment for debuggin
-# echo $USER running load script in $DIR
-# echo callers
-# caller
-# echo $(ps -o comm= $PPID)
-# echo -----
-# echo $BASH_SHELL_DIRS
-
-module_load file
-[[ $? -ne 0 ]] && echo unable to access the file module, aboarting load && return 1
-
 module_load debug
 if [[ $? -ne 0 ]]; then
 echo "unable to load a 'debug' module using a noop for debug function"
@@ -39,102 +19,12 @@ fi
 
 ([[ $BASH_ENV ]] && [[ ! $- == *i* ]] && [[ $BASH_USE_ALIAS ]]) && shopt -s expand_aliases  
 
-function shell_process_directory () {
-    local SUBDIRS
-    local DIR
-
-    DIR=${1:-$BASH_SHELL_BASE}
-    # echo soucring directory $DIR
-    # if [[ $DIR = "$BASH_SHELL_BASE" ]]; then
-    #   BASH_SHELL_IGNORE=$(shell_get_ignores)
-    #   excludes=$BASH_SHELL_IGNORE
-    # else
-    #   excludes=$(shell_get_ignores $DIR)
-    #   [[ $? -ne 0 ]] && excludes=$BASH_SHELL_IGNORE
-    # fi
-
-    if [ -d  "$DIR" ]; then
-      if [ "$DIR" = "$BASH_SHELL_BASE" ] && [ "$BASH_SHELL_BASE_LOADED" = "true" ]; then
-          if [ -v PS1 ]; then
-          echo base directory already sourced
-          read -p "do you want to re-source the base (not recommended)? " -n 1 -r
-          echo
-          [[ ! $REPLY =~ ^[Yy]$ ]] && return 1
-          # else
-            # echo non-interactive shell
-            # return 1
-          fi
-        fi
-
-      if [[ $DIR != "$BASH_SHELL_BASE" ]] && [[ -f  $DIR/load.sh ]]; then
-        # echo "$DIR/load.sh" found, running instead of default load
-        source "$DIR/load.sh"
-       else
-          # default processing
-          local SUBDIRS
-          local IGNORE_FILE
-          SUBDIRS=$([[ -f "$DIR/.bash-shell-include" ]] && cat "$DIR/.bash-shell-include" || echo $BASH_SHELL_BASE_SUBDIRS)
-          [[ $SSH_SESSION ]] && SUBDIRS+=" ssh/session"
-          IGNORE_FILE="$([[ -f "$DIR/.bash-shell-ignore" ]] && echo "$DIR" || echo "$BASH_SHELL_BASE")/.bash-shell-ignore"
-          for SUBDIR in $SUBDIRS;  do
-            if [ -e  "$DIR/$SUBDIR" ]; then
-              # echo processing subdirectory $DIR/$SUBDIR
-              # must quote all globs to avoid bash glob expansion which is likely on
-              # TODO have default set of ignores plus check for shell-ignore
-              # source_dir -p "archive" -x '"*.off" "*.md"  "*TODO*" "LICENSE" "*.sample"'   -d 0 $DIR/$SUBDIR
-              # source_dir -p "archive" -x "$excludes" -d 0 $DIR/$SUBDIR
-              source_dir -f "$IGNORE_FILE" -d 0 $DIR/$SUBDIR
-             #  else
-                # echo no subdirectory $DIR/$SUBDIR to process, ignorning
-            fi
-          done
-        fi
-        # echo "done sourcing directory $DIR"
-     # else
-        # echo $DIR does not exist nothing to source
-    fi
-
-  }
-
 # process the base directory by default
 unset BASH_SHELL_BASE_LOADED
 unset BASH_SHELL_LOADED
 
-# add in any network directories
+module_load shell-process-directory
-declare networks
+for dir in ${1:-$BASH_SHELL_DIRS}; do
-source "$HOME/.bashrc"
-networks="$HOME/${BASH_SHELL_USER:-"shell"}/.networks"
-NETWORK_DOMAINS="$([[ -f $networks ]] && cat $networks) " 
-# These will be loaded for all users
-networks="$BASH_SHELL_HOST/$(hostname)/.networks"
-NETWORK_DOMAINS+="$([[ -f $networks ]] && cat $networks)" 
-NETWORK_DOMAINS=(${NETWORK_DOMAINS})
-
-if [[ ! $NETWORK_DOMAINS = "" ]]; then
-declare domain
-BASH_SHELL_NETWORK_DIRS=""
-for domain in "${NETWORK_DOMAINS[@]}"; do
-# echo processing ${BASH_SHELL_NETWORK}/${domain}
-if [[ ${domain} = !* ]]; then 
-domain=${domain/#!/}
-export NETWORK_HOME=$domain
-fi
-if [[ -d ${BASH_SHELL_NETWORK}/${domain} ]]; then
-   BASH_SHELL_NETWORK_DIRS+="${BASH_SHELL_NETWORK}/${domain} "
-   fi
-done
-unset domain
-export BASH_SHELL_NETWORK_DIRS
-BASH_SHELL_DIRS+=" ${BASH_SHELL_NETWORK_DIRS}"
-fi
-
-# echo bash shell dirs: $BASH_SHELL_DIRS
-dirs=${1:-$BASH_SHELL_DIRS}
-
-BASH_SHELL_BASE_SUBDIRS=$(cat "$BASH_SHELL_BASE/.bash-shell-include")
-# echo subdir includes: $BASH_SHELL_BASE_SUBDIRS
-
-for dir in $dirs; do
   # echo $dir
   shell_process_directory $dir
   [[ $dir == "$BASH_SHELL_BASE" ]] && BASH_SHELL_BASE_LOADED=true
@@ -142,6 +32,4 @@ done
 
 export BASH_SHELL_LOADED=true
 
-# process user
+# Note: $HOME/shell or $HOME/BASH_SHELL_USER are processed via $HOME/.bashrc
-# Note: $HOME/shell or $HOME/BASH_SHELL_USER are sourced via $HOME/.bashrc
-# echo $(envg BASH) | xargs -n 1

module.lib

@@ -34,36 +34,37 @@ local MDIRS
 local MDIR
 local DIRS
 local MODULE=$1
-# Precidence is user. current hosts, all hosts, current network, all networks, base
-DIRS=( \
-$([ $BASH_SHELL_USER ] && echo $HOME/$BASH_SHELL_USER || echo $HOME/shell) \
-$BASH_SHELL_HOST/$(hostname)
-$BASH_SHELL_HOST/all \
-$BASH_SHELL_NETWORK/$NETWORKNAME \
-$BASH_SHELL_NETWORK/all \
-$BASH_SHELL_BASE \
-)
 
-# echo ${DIRS[@]}
 
-MDIRS=()
+# MODULE_DIRS is array set in the environment and is an array of additional directories to 
-j=0
+# search for modules.  This makes is easy to include a custom module libraries outside
-cnt=${#DIRS[@]}
+# the shell system.  These take precedence over any modules found in shell directories below 
-for ((i=0;i<cnt;i++)); do
-    DIR="${DIRS[i]}/modules"
-    [ -d $DIR ] && MDIRS[j]=$DIR && j+=1
-done
 
-[ $MODULE_DIRS ] && MDIRS=("$MODULE_DIRS" "${MDIRS[@]}")
+if [[ $MODULE_DIRS ]]; then
-
+for DIR in "${MODULE_DIRS[@]}"
-# echo Module Directories ${MDIRS[@]}
-
-for MDIR in "${MDIRS[@]}"
    do
-     # echo mdir $MDIR name $MODULE
+     if [[ -d $DIR ]]; then 
-     RES=$(module_confirm "$MDIR" "$MODULE")
+     RES=$(module_confirm "$DIR" "$MODULE")
      [ $? -eq 0 ] && echo $RES && return 0
+     fi
   done
+fi
+
+# BASH_SHELL_DIRS holds shell directories to search for modules.  If will be searched
+# in revsere order, user or network repos over host over base
+DIRS=( ${BASH_SHELL_DIRS:-$BASH_SHELL_BASE} )
+[ -d $HOME/$BASH_SHELL_USER ] && DIRS=("${DIRS[@]}" "$HOME/$BASH_SHELL_USER")
+
+cnt=${#DIRS[@]}
+for ((i=1;i<=cnt;i++)); do
+    # find modules in reverse order so more specific ones override
+    DIR="${DIRS[cnt-i]}/modules"
+    if [[ -d $DIR ]]; then 
+    RES=$(module_confirm "$DIR" "$MODULE")
+     [ $? -eq 0 ] && echo $RES && return 0
+    fi 
+  done
+  # no module found anywhere
   return 1
   
 }

modules/utility/debug.lib

@@ -6,3 +6,37 @@ function debug () {
 
 # alias debug_on="sudo -i uncomment BASH_DEBUG /etc/bash.bashrc"
 # alias debug_off="sudo -i comment BASH_DEBUG /etc/bash.bashrc"
+
+
+# ***************  DEBUGGING ***********************
+# module_load debug
+# if [[ $? -ne 0 ]]; then
+# echo "unable to load a 'debug' module using a noop for debug function"
+# # noop
+# function debug () {
+# :
+# }
+# fi
+
+# [[ ! $- == *i* ]] && exec >> ~/logs/load.log && exec 2>&1
+# export BASH_DEBUG=true
+
+# uncomment for debugging
+# echo $USER running load script in $DIR
+# echo callers
+# caller
+# echo $(ps -o comm= $PPID)
+# echo -----
+# echo $BASH_SHELL_DIRS
+# ******************END DEBUGGING *******************************
+
+# ***************** LOGGING  *****************
+# function nilog () {
+#   [[ ! $- == *i* ]] && echo -e "-----\n $USER $*" &>> ~/logs/load.log
+# }
+
+# nilog "called load.sh  $(date)"
+# nilog caller: $(caller)
+# nilog pid: $(ps -o comm= $PPID)
+
+#  *

modules/utility/dirs.sh

@@ -31,6 +31,16 @@
 
 # Usage message
 
+isDir() {
+    if [[ -d $1 ]]
+    then
+        echo "true"
+        return 0
+    else
+        return 1
+    fi
+}
+
 chmodr () {
 
 usage()

modules/utility/file.lib

@@ -3,6 +3,16 @@
 # export BASH_DEBUG=true
 module_load debug
 
+isFile() {
+    if [[ -f $1 ]]
+    then
+        echo "true"
+        return 0
+    else
+        return 1
+    fi
+}
+
 build_file () {
   [[ -f "$2" ]] || (echo "output file $2 does not exist";return 1)
   if [[ -f "$1" ]]; then

modules/utility/yaml.sh

@@ -18,3 +18,8 @@ parse_yaml() {
    }'
 }
 
+yaml() {
+    python3 -c "import yaml;print(yaml.safe_load(open('$1'))$2)"
+}
+
+

shell.env

@@ -1,13 +1,16 @@
 #!/bin/bash
 # Shell Repos Environment
+# Customize Bash Shell setup here
 # if bash is not installed then don't bother to continue
 ! command -v bash >/dev/null 2>&1 && echo no bash command && return 1
 export SHELL=$(command -v bash )
+
 # sourced for non-login interactive shells
 # sourced via /etc/bash.bashrc so for all machine users
 # $BASH_SHELL_BASE  # this MUST be set in /etc/profile
+# $BASH_SHELL_HOST=$BASH_SHELL_BASE
 # $BASH_SHELL_NETWORK/all  
-# $BASH_SHELL_NETWORK/(array of Domains)
+# $BASH_NETWORK_DIR  # extra directory to look for networks, $BASH_SHELL_NETWORK is always checked
 # $BASH_SHELL_HOST/all
 # $BASH_SHELL_HOST/<hostname>
 # also
@@ -25,22 +28,23 @@ export SHELL=$(command -v bash )
 # Using base shell setup repository
 # https://git.kebler.net/kebler.net/bash-shell-base.git
 
-# use these two if you want a common directory for all shell repos
-#  BASH_SHELL_BASE is set in /etc/profile
 
-# load script in base repo to run
+#  BASH_SHELL_BASE is set in /etc/profile do not change here!
+
 export BASH_SHELL_LOAD=$BASH_SHELL_BASE/load.sh # load.sh is default
 export BASH_SHELL_STARTUP=$BASH_SHELL_BASE/startup.sh # strtup.sh is default
 # if uncommented next lines sets up implicit sourcing for non-interactive shells
-# echo ----NON_INTERACTIVE SHELL INFO-----
+
+# echo ----NON_INTERACTIVE SHELL-----
 # echo enabling bash shell repos for non-inactive shell
 # export BASH_ENV=$BASH_SHELL_LOAD # same as interactive shell, beware usually too much
-export BASH_ENV=$BASH_SHELL_BASE/module.lib #  only load module loading functions
+export BASH_ENV=$BASH_SHELL_BASE/module.lib #  load module loading functions
 # echo enabling aliases with non-interactive shell
 export BASH_USE_ALIAS=true  # will source aliases for non-interactive
 # echo see $BASH_SHELL_BASE/shell.env
 # echo ---------
-# if not using implicit sourcing for non-interactive shells then on can do this per script
+
+# if not using implicit sourcing for non-interactive shells then one can do this per script
 ##################
 # expanding aliases is optional
 # shopt -s expand_aliases
@@ -53,33 +57,51 @@ export BASH_USE_ALIAS=true  # will source aliases for non-interactive
 ([ -n "$SSH_CONNECTION" ] || [ -n "$SSH_CLIENT" ] || [ -n "$SSH_TTY" ]) && export SSH_SESSION=true
 
 
+BASH_SHELL_DIRS="$BASH_SHELL_BASE "
 declare parent
 parent=$(dirname $BASH_SHELL_BASE)
 
-export BASH_SHELL_HOST=$([[ -d $parent/host ]] && echo $parent/host || echo /opt/bash/shell/host)
+# Any Host
-# or comment above and set explictly
+BASH_SHELL_ANY_HOST=${parent}/any/host  # or set to 
+# export BASH_SHELL_ALL_HOSTS=<some directory>
+# Host
+BASH_SHELL_HOST=$parent/$(hostname)
 # export BASH_SHELL_HOST=<some directory>
 
-# identify a network name that this host resides on
+BASH_SHELL_ANY_NETWORK=$parent/any/network
-# make a directory of the same name
+# export BASH_SHELL_ALL_NETWORKS=<some directory>
-
-export BASH_SHELL_NETWORK=$([[ -d $parent/network ]] && echo $parent/network || echo /opt/bash/shell/network)
-# or comment above and set explictly
-# export BASH_SHELL_NETWORK=<some directory>
 
 # by default USER shell sources will be looked for under $HOME/shell
 # but the default directory for all users can be set manually below 
 # export BASH_SHELL_USER=<some directory under $HOME>
 # add this export to .bashrc for custom directory.  This can be done during or after using the user setup script
 
-temp="
+[[ -d $BASH_SHELL_ANY_HOST ]] && BASH_SHELL_DIRS+="$BASH_SHELL_ANY_HOST " && export BASH_SHELL_ANY_HOST
-$BASH_SHELL_BASE
+[[ -d $BASH_SHELL_ANY_NETWORK ]] && BASH_SHELL_DIRS+="$BASH_SHELL_ANY_NETWORK " && export BASH_SHELL_ANY_NETWORK 
-$BASH_SHELL_NETWORK/all
-$BASH_SHELL_HOST/all
-$BASH_SHELL_HOST/$(hostname)
-"
 
-export BASH_SHELL_DIRS=$(echo $temp)
+# now load and export module loading functions library
+source $BASH_SHELL_BASE/module.lib
+
+module_load network-dirs
+
+BASH_NETWORKS_DIR=$parent/networks
+# BASH_NETWORKS_DIR=<some directory>  # alt directory to look for networks
+export BASH_NETWORKS_DIR
+
+# These will be loaded for all users on a host
+network_dirs "$BASH_SHELL_HOST/.networks"
+
+[[ -d $BASH_SHELL_HOST ]] && BASH_SHELL_DIRS+="$BASH_SHELL_HOST " && export BASH_SHELL_HOST
+
+# these loaded only for specific user on a host
+network_dirs "$HOME/${BASH_SHELL_USER:-"shell"}/.networks"
+
+export BASH_SHELL_DIRS
+
+# echo ALL DIRS: $BASH_SHELL_DIRS
+
+# cat $HOME/.bashrc
+# source "$HOME/.bashrc"
 
 # env | grep BASH
 # env | grep NETWORK_DIRS
@@ -87,7 +109,6 @@ export BASH_SHELL_DIRS=$(echo $temp)
 # echo $BASH_SHELL_DIRS
 # echo  ---------------------------------
 
-# now load and export module loading functions library
+
-source $BASH_SHELL_BASE/module.lib
 
 # echo end shell env