refactored to use ignore and include files for finding files to source

moved source_dir to file.lib.
refactored to use directory list and a loop for shell directory processing
added array for networks so vpn networks be added for sourcing.

.bash-shell-ignore

@@ -1,5 +1,12 @@
+# hidden files and directories are ignored by default
+# i.e. beginning with .
+# no ignores here may begin with .
 *.off
 *.md
 *TODO*
+*.log
 LICENSE
 *.sample
+*archive*/
+off/
+log*/

.bash-shell-include

@@ -0,0 +1,6 @@
+function 
+alias
+env
+misc
+lang
+app

load.sh

@@ -5,6 +5,8 @@
 
 DIR=${1:-$(dirname ${BASH_SOURCE[0]})}
 
+[[ $(declare -F | grep module_load) ]] && echo module lib already loaded || source "$BASH_SHELL_BASE/module.lib"
+
 # uncomment for debuggin
 # echo $USER running load script in $DIR
 # echo callers
@@ -13,48 +15,22 @@ DIR=${1:-$(dirname ${BASH_SOURCE[0]})}
 # echo -----
 # echo $BASH_SHELL_DIRS
 
-# source the required directory source function
+module_load file
-if [ ! -e  "$BASH_SHELL_BASE/source-dir.func" ]; then
+[[ $? -ne 0 ]] && echo unable to access the file module, aboarting load && return 1
-  echo "!Unable to load sourcing function at $DIR/source-dir, shell not set up!"
-  return 1
-else
-  source "$BASH_SHELL_BASE/source-dir.func"
-fi
 
-#   if [ ! -e  "$BASH_SHELL_BASE/module.lib" ]; then
+function shell_process_directory () {
-#     echo "!Unable to load module library functions function at $DIR/module.lib, shell not set up!"
-#     return 1
-#   else
-#     source "$BASH_SHELL_BASE/module.lib"
-# fi
-
-function shell_get_ignores () (
-local file=${1:-"$BASH_SHELL_BASE"}/.bash-shell-ignore
-[[ ! -f "$file" ]] && [[ $1 ]] && return 1
-  local excludes=''
-  local ignores=$(cat "$file")
-  for exclude in $ignores ; do
-    excludes+='"'$exclude'" '
-  done
-   echo $excludes
-   return 0
-)
-
-  function shell_process_directory () {
     local SUBDIRS
     local DIR
-    local DSUBDIRS
-    local excludes
 
     DIR=${1:-$BASH_SHELL_BASE}
     # echo soucring directory $DIR
-    if [[ $DIR = "$BASH_SHELL_BASE" ]]; then
+    # if [[ $DIR = "$BASH_SHELL_BASE" ]]; then
-      BASH_SHELL_IGNORE=$(shell_get_ignores)
+    #   BASH_SHELL_IGNORE=$(shell_get_ignores)
-      excludes=$BASH_SHELL_IGNORE
+    #   excludes=$BASH_SHELL_IGNORE
-    else
+    # else
-      excludes=$(shell_get_ignores $DIR)
+    #   excludes=$(shell_get_ignores $DIR)
-      [[ $? -ne 0 ]] && excludes=$BASH_SHELL_IGNORE
+    #   [[ $? -ne 0 ]] && excludes=$BASH_SHELL_IGNORE
-    fi
+    # fi
 
     if [ -d  "$DIR" ]; then
       if [ "$DIR" = "$BASH_SHELL_BASE" ] && [ "$BASH_SHELL_BASE_LOADED" = "true" ]; then
@@ -69,27 +45,24 @@ local file=${1:-"$BASH_SHELL_BASE"}/.bash-shell-ignore
           fi
         fi
 
-      if [ -f  "$DIR/load.sh" ] && [ "$DIR" != "$BASH_SHELL_BASE" ]; then
+      if [[ $DIR != "$BASH_SHELL_BASE" ]] && [[ -f  $DIR/load.sh ]]; then
-          # echo "$DIR/$BASH_SHELL_LOAD" found, running instead of default processing
+        # echo "$DIR/load.sh" found, running instead of default load
         source "$DIR/load.sh"
        else
           # default processing
           local SUBDIRS
-          local DSUBDIRS
+          local IGNORE_FILE
-          # default subdirectories to source
+          SUBDIRS=$([[ -f "$DIR/.bash-shell-include" ]] && cat "$DIR/.bash-shell-include" || echo $BASH_SHELL_BASE_SUBDIRS)
-          # TODO allow passing in additional directories
+          IGNORE_FILE="$([[ -f "$DIR/.bash-shell-ignore" ]] && echo "$DIR" || echo "$BASH_SHELL_BASE")/.bash-shell-ignore"
-
+          for SUBDIR in $SUBDIRS;  do
-          DSUBDIRS="function alias env misc lang app"
-          IFS=' ' read -r -a SUBDIRS <<< "${2:-$DSUBDIRS}"
-          # echo ${SUBDIRS[@]}
-          for SUBDIR in "${SUBDIRS[@]}"
-          do
             if [ -e  "$DIR/$SUBDIR" ]; then
+
               # echo processing subdirectory $DIR/$SUBDIR
               # must quote all globs to avoid bash glob expansion which is likely on
               # TODO have default set of ignores plus check for .shell-ignore
               # source_dir -p "archive" -x '"*.off" "*.md"  "*TODO*" "LICENSE" "*.sample"'   -d 0 $DIR/$SUBDIR
-            source_dir -p "archive" -x "$excludes" -d 0 $DIR/$SUBDIR
+              # source_dir -p "archive" -x "$excludes" -d 0 $DIR/$SUBDIR
+              source_dir -f "$IGNORE_FILE" -d 0 $DIR/$SUBDIR
              #  else
                 # echo no subdirectory $DIR/$SUBDIR to process, ignorning
             fi
@@ -109,6 +82,9 @@ unset BASH_SHELL_LOADED
 # echo bash shell dirs: $BASH_SHELL_DIRS
 DIRS=${1:-$BASH_SHELL_DIRS}
 
+BASH_SHELL_BASE_SUBDIRS=$(cat "$BASH_SHELL_BASE/.bash-shell-include")
+# echo subdir includes: $BASH_SHELL_BASE_SUBDIRS
+
 for dir in $DIRS; do
   # echo $dir
   shell_process_directory $dir

module.lib

@@ -1,7 +1,5 @@
 #!/bin/bash
-
+# echo loading module library
-echo loading module library
-
 module_confirm () {
 local FILE
 local DIR
@@ -9,7 +7,7 @@ local NAME
 DIR=$1
 NAME=$2
 # echo finding $NAME in $DIR
-FILE="$(find ${DIR} -type f -name "${NAME}.mod" -o -name "${NAME}".sh)"
+FILE="$(find ${DIR} -type f -name "${NAME}.mod" -o -name "${NAME}".lib -o -name "${NAME}".func -o -name "${NAME}".sh)"
 # echo files found $FILE
 COUNT=0
 if [ "$FILE" ]; then
@@ -76,7 +74,6 @@ FILE=$(module_find $1)
 # source $FILE "$0"
 [[ $BASHPID -eq $$ ]] || echo $FILE
 source $FILE
-
 return 0
 }
 

modules/utility/file.lib

@@ -0,0 +1,156 @@
+#!/bin/bash
+
+function lines_2_str () {
+  [[ ! -f "$1" ]] && return 1
+  local str=''
+  # echo the lines: $lines >&2
+  # lines="$(cat "$1")"
+  while read line; do
+  [[ "$line" =~ ^[[:space:]]*# ]] && continue
+    # echo line: "${line}" >&2
+    str="$str ${line}"
+   # echo str: $str >&2
+done < $1
+  # for line in $lines ; do
+  #    str+='"'$line'" '
+  # done
+  echo $str
+  # set +o noglob
+  return 0
+}
+
+function super_find () {
+  # echo super_find $@ >&2
+#  USAGE
+# all option arguments that contain globs/wildcards must be quoted to avoid expansion
+# f sets path and file excludes from a supplied file path
+# all lines ending in /  will be treated as directory names to ignore, otherwise files
+# p option explictly excludes paths(directories)
+# d option sets the directory depth which is current directy by default, 0 is all
+# x excitly excludes file globs as a string
+# n inclucdes only file globs otherwise it's all except .files
+# if no directory is given it will attempt to source the present working directory
+# example:
+# source_dir -p "archive" -x '"*.off" "*.md"'  -d 0 # $DIR/$SUBDIR
+declare OPTION
+declare OPTARG
+declare OPTIND
+local EXCLUDE_FILE
+local PATHS
+local NAMES
+local ENAMES
+local DEPTH=1
+local VERBOSE=0
+
+while getopts 'p:d:e:n:f:' OPTION; do
+case "$OPTION" in
+  f)
+    EXCLUDE_FILE=$OPTARG
+    # echo EXCLUDE FILE $EXCLUDE_FILE >&2
+    ;;
+  p)
+    # PATHS=("$OPTARG")
+    IFS=',' read -r -a PATHS <<< "$OPTARG"
+    # echo EXCLUDING THESE PATHS ${PATHS[*]}
+    ;;
+  e)
+    IFS=',' read -r -a ENAMES <<< "${OPTARG}"
+    # echo EXCLUDING THESE FILE NAMES ${ENAMES[*]}
+    ;;
+  n)
+    # NAMES=("$OPTARG")
+    IFS=',' read -r -a NAMES <<< "${OPTARG}"
+    # echo INCLUDING ONLY THESE FILE NAMES ${NAMES[*]}
+    ;;
+  d)
+    DEPTH=$OPTARG
+    # echo "SOURCING TO DEPTH (0=any)" "$DEPTH"
+    ;;
+  *)
+    echo unknown option $OPTION
+    ;;
+esac
+done
+
+shift $(( OPTIND - 1 ))
+
+local DIR
+DIR="$*"
+if [ ! "$DIR" ]; then
+  if [ -v PS1 ]; then
+  echo no directory to source provided
+  echo sourcing present working directory $(pwd)
+  read -p "Do you want to continue? " -n 1 -r
+  [[ $REPLY =~ ^[Yy]$ ]] && DIR=$(pwd) || return 1
+  else
+    return 1
+  fi
+fi
+
+[ ! -d "$DIR" ] && echo " directory $DIR does not exist, aborting" && return 1
+
+# echo dir $DIR
+
+local FIND
+FIND="find $DIR"
+FIND+=$([ ! $DEPTH == 0 ] && echo " -maxdepth $DEPTH ")
+# ignore all .name and .path by default
+FIND+=" -type f ! -path \"*/.*/*\" ! -name \".*\" "
+
+local name
+local path
+
+if [[ -f $EXCLUDE_FILE ]]; then
+  local ignores=$(lines_2_str "$EXCLUDE_FILE")
+  # echo ignores: $ignores >&2
+  for exclude in $ignores ; do
+    # echo exclude: ${exclude} >&2
+    [[ "$exclude" == */ ]] && PATHS+=("${exclude::-1}") || ENAMES+=("$exclude")
+  done
+fi
+
+# echo paths ${PATHS[@]} >&2
+# echo exclude names ${ENAMES[@]} >&2
+
+if [[ ${PATHS[0]} ]]; then
+  for path in ${PATHS[@]}; do
+     # echo excluding $path
+     FIND+=$(echo ' ! -path "*/'$path'/*"')
+  done
+fi
+
+if [[ ${ENAMES[0]} ]]; then
+  for name in ${ENAMES[@]}; do
+     # echo excluding name $name
+     FIND+=$(echo ' ! -name "'${name}'"')
+  done
+fi
+
+if [[ ${NAMES[0]} ]]; then
+  for name in ${NAMES[@]}; do
+     # echo only finding $name
+     FIND+=$(echo " -name ${name}")
+  done
+fi
+
+# echo
+# echo find dir: $DIR >&2
+# echo find command: $FIND >&2
+
+local FILES
+FILES=$(eval $FIND | sort)
+echo $FILES
+return 0
+}
+
+source_dir () {
+  # echo passed: $*
+  local FILES
+  FILES=$(super_find "$@")
+  [[ $? -ne 0 ]] && return 1
+    for f in $FILES; do
+    # echo sourcing: $f
+    source "$f"
+  done
+
+}

modules/utility/source.func

@@ -1,36 +1,43 @@
 #!/bin/bash
-source_dir () {
+
+function source_dir () {
 #  USAGE
 # will never source source .files or .directories
 # all option arguments that contain globs/wildcards must be quoted to avoid expansion
+# f sets path and file excludes from a supplied file path
 # p option excludes additional paths below of the given name (and depth)
 # d option sets the directory depth which is current directy by default, 0 is all
 # x excludes file globs
-# f inclucdes only file globs otherwise it's all except .files
+# n inclucdes only file globs otherwise it's all except .files
 # if no directory is given it will attempt to source the present working directory
 # example:
 # source_dir -p "archive" -x '"*.off" "*.md"'  -d 0 # $DIR/$SUBDIR
 declare OPTION
 declare OPTARG
 declare OPTIND
+local EXCLUDE_FILE
 local PATHS
 local NAMES
 local ENAMES
 local DEPTH=1
 local VERBOSE=0
 
-while getopts 'p:d:x:f:' OPTION; do
+while getopts 'p:d:e:n:f:' OPTION; do
 case "$OPTION" in
+  f)
+    EXCLUDE_FILE=$OPTARG
+    # echo EXCLUDE FILE $EXCLUDE_FILE
+    ;;
   p)
     # PATHS=("$OPTARG")
     IFS=',' read -r -a PATHS <<< "$OPTARG"
     # echo EXCLUDING THESE PATHS ${PATHS[*]}
     ;;
-  x)
+  e)
     IFS=',' read -r -a ENAMES <<< "${OPTARG}"
     # echo EXCLUDING THESE FILE NAMES ${ENAMES[*]}
     ;;
-  f)
+  n)
     # NAMES=("$OPTARG")
     IFS=',' read -r -a NAMES <<< "${OPTARG}"
     # echo INCLUDING ONLY THESE FILE NAMES ${NAMES[*]}
@@ -75,6 +82,13 @@ FIND+=" -type f ! -path \"*/.*/*\" ! -name \".*\" "
 local name
 local path
 
+if [[ -f $EXCLUDE_FILE ]]; then
+  local ignores=$(cat "$EXLUCDE_FILE")
+  for exclude in $ignores ; do
+    [[ "$exclude" =~ '/'$ ]] && PATHS+=(${exclude::-1}) || ENAMES+=($exclude)
+  done
+fi
+
 if [[ ${PATHS[0]} ]]; then
   for path in ${PATHS[@]}; do
      # echo excluding $path
@@ -96,16 +110,16 @@ if [[ ${NAMES[0]} ]]; then
   done
 fi
 
-# echo
+echo
-# echo find dir: $DIR
+echo find dir: $DIR
-# echo find command: $FIND
+echo find command: $FIND
 
 local FILES
 FILES=$(eval $FIND | sort)
 # echo $FILES | xargs -n 1
   for f in $FILES; do
-  # echo sourcing: $f
+  echo sourcing: $f
-  source "$f"
+  # source "$f"
 done
 
 }

shell.env

@@ -33,7 +33,7 @@ export BASH_SHELL_LOAD=$BASH_SHELL_BASE/load.sh # load.sh is default
 export BASH_SHELL_STARTUP=$BASH_SHELL_BASE/startup.sh # strtup.sh is default
 
 # if uncommented next line sets up implicit sourcing for non-interactive shells
-export BASH_ENV=$BASH_SHELL_LOAD # use base repo
+# export BASH_ENV=$BASH_SHELL_LOAD # use base repo
 # if not using implicit sourcing for non-interactive shells then on can do this per script
 ##################
 # expanding aliases is optional
@@ -46,7 +46,7 @@ export BASH_ENV=$BASH_SHELL_LOAD # use base repo
 # identify a network name that this host resides on
 # make a directory of the same name
 # if unset then only /all will be sourced
-export NETWORK_DOMAINS=(238.kebler.net 645.kebler.net)
+export NETWORK_DOMAINS=(238.kebler.net 645.kebler.net 3115.kebler.net)
 # network domain folder entry will be made for each domain set, first is home domain, others via vpn
 if [[ $NETWORK_DOMAINS ]]; then
 export NETWORK_HOME=${NETWORK_DOMAINS[0]}

shell.lib

@@ -0,0 +1,135 @@
+#!/bin/bash
+
+function lines_2_str () (
+  [[ ! -f "$1" ]] && return 1
+  local str=''
+  local lines=$(cat "$1")
+  for line in $lines ; do
+     str+='"'$line'" '
+  done
+  echo $str
+  return 0
+)
+
+function source_dir () {
+#  USAGE
+# will never source source .files or .directories
+# all option arguments that contain globs/wildcards must be quoted to avoid expansion
+# p option excludes additional paths below of the given name (and depth)
+# d option sets the directory depth which is current directy by default, 0 is all
+# x excludes file globs
+# f inclucdes only file globs otherwise it's all except .files
+# if no directory is given it will attempt to source the present working directory
+# example:
+# source_dir -p "archive" -x '"*.off" "*.md"'  -d 0 # $DIR/$SUBDIR
+declare OPTION
+declare OPTARG
+declare OPTIND
+local EXCLUDE_FILE
+local PATHS
+local NAMES
+local ENAMES
+local DEPTH=1
+local VERBOSE=0
+
+while getopts 'p:d:e:n:f:' OPTION; do
+case "$OPTION" in
+  f)
+    EXCLUDE_FILE=$OPTARG
+    # echo EXCLUDE FILE $EXCLUDE_FILE
+    ;;
+  p)
+    # PATHS=("$OPTARG")
+    IFS=',' read -r -a PATHS <<< "$OPTARG"
+    # echo EXCLUDING THESE PATHS ${PATHS[*]}
+    ;;
+  e)
+    IFS=',' read -r -a ENAMES <<< "${OPTARG}"
+    # echo EXCLUDING THESE FILE NAMES ${ENAMES[*]}
+    ;;
+  n)
+    # NAMES=("$OPTARG")
+    IFS=',' read -r -a NAMES <<< "${OPTARG}"
+    # echo INCLUDING ONLY THESE FILE NAMES ${NAMES[*]}
+    ;;
+  d)
+    DEPTH=$OPTARG
+    # echo "SOURCING TO DEPTH (0=any)" "$DEPTH"
+    ;;
+  *)
+    echo unknown option $OPTION
+    ;;
+esac
+done
+
+shift $(( OPTIND - 1 ))
+
+local DIR
+DIR="$@"
+if [ ! "$DIR" ]; then
+  if [ -v PS1 ]; then
+  echo no directory to source provided
+  echo sourcing present working directory $(pwd)
+  read -p "Do you want to continue? " -n 1 -r
+  [[ $REPLY =~ ^[Yy]$ ]] && DIR=$(pwd) || return 1
+  else
+    return 1
+  fi
+fi
+
+[ ! -d "$DIR" ] && echo " directory $DIR does not exist, aborting" && return 1
+
+# echo dir $DIR
+
+local FIND
+FIND="find $DIR"
+FIND+=$([ ! $DEPTH == 0 ] && echo " -maxdepth $DEPTH ")
+# ignore all .name and .path by default
+# TODO change to having a string
+FIND+=" -type f ! -path \"*/.*/*\" ! -name \".*\" "
+
+
+local name
+local path
+
+if [[ -f $EXCLUDE_FILE ]]; then
+  local ignores=$(cat "$EXLUCDE_FILE")
+  for exclude in $ignores ; do
+    [[ "$exclude" =~ '/'$ ]] && PATHS+=(${exclude::-1}) || ENAMES+=($exclude)
+  done
+fi
+
+if [[ ${PATHS[0]} ]]; then
+  for path in ${PATHS[@]}; do
+     # echo excluding $path
+     FIND+=$(echo ' ! -path "*/'$path'/*"')
+  done
+fi
+
+if [[ ${ENAMES[0]} ]]; then
+  for name in ${ENAMES[@]}; do
+     # echo excluding name $name
+     FIND+=$(echo " ! -name ${name}")
+  done
+fi
+
+if [[ ${NAMES[0]} ]]; then
+  for name in ${NAMES[@]}; do
+     # echo only finding $name
+     FIND+=$(echo " -name ${name}")
+  done
+fi
+
+echo
+echo find dir: $DIR
+echo find command: $FIND
+
+local FILES
+FILES=$(eval $FIND | sort)
+# echo $FILES | xargs -n 1
+  for f in $FILES; do
+  echo sourcing: $f
+  # source "$f"
+done
+
+}

startup.sh

@@ -1,30 +1,39 @@
 #!/bin/bash
 function startup_load () {
-
 local SDIRS
 local SDIR
 local DIRS
-[[ $1 ]] && DIRS=("$1") || DIRS=("${BASH_SHELL_DIRS}")
+local DIR
-llog "startup directories to try ${DIRS[*]}"
+local excludes
+
+
+[[ $(declare -F | grep module_load) ]] && echo module lib already loaded || source "$BASH_SHELL_BASE/module.lib"
+module_load file
+[[ $? -ne 0 ]] && echo unable to access the file module, aboarting load && return 1
+
+
+[[ $1 ]] && DIRS=($1) || DIRS=(${BASH_SHELL_DIRS})
+# llog "startup directories to try ${DIRS[*]}"
 SDIRS=()
 j=0
 cnt=${#DIRS[@]}
 for ((i=0;i<cnt;i++)); do
+    # echo $i of $cnt
     DIR="${DIRS[i]}/startup"
+    # echo ----- trying $DIR
     [ -d $DIR ] && SDIRS[j]=$DIR && j+=1
 done
 
 # STARTUP_DIRS is array of any other startup directores where scripts may be loaded from
 [ $STARTUP_DIRS ] && SDIRS=("$STARTUP_DIRS" "${SDIRS[@]}")
 
-# echo -e "Statup Directories to source \n ${SDIRS[@]} \n"
-source $BASH_SHELL_BASE/source-dir.func
-
 for SDIR in "${SDIRS[@]}"
-   do
+do
-     llog "startup dir to source $SDIR"
+DIR=$(dirname $SDIR)
-     source_dir -p "archive" -x '"*.off" "*.md"'  -d 0 $SDIR
+IGNORE_FILE="$([[ -f "$DIR/.bash-shell-ignore" ]] && echo "$DIR" || echo "$BASH_SHELL_BASE")/.bash-shell-ignore"
-  done
+llog "startup dir to source $SDIR"
+source_dir -f "$IGNORE_FILE" -d 0 $SDIR
+done
 }
 
 # if script was executed then call the function