From c436c07493ee5c5b2cb3e1c559e5d96071879783 Mon Sep 17 00:00:00 2001 From: David Kebler Date: Fri, 20 Nov 2020 08:48:48 -0800 Subject: [PATCH] refactored to use ignore and include files for finding files to source moved source_dir to file.lib. refactored to use directory list and a loop for shell directory processing added array for networks so vpn networks be added for sourcing. --- .bash-shell-ignore | 7 + .bash-shell-include | 6 + load.sh | 88 ++++------ module.lib | 7 +- modules/utility/file.lib | 156 ++++++++++++++++++ .../utility/source.func | 34 ++-- shell.env | 4 +- shell.lib | 135 +++++++++++++++ startup.sh | 29 ++-- 9 files changed, 383 insertions(+), 83 deletions(-) create mode 100644 .bash-shell-include create mode 100644 modules/utility/file.lib rename source-dir.func => modules/utility/source.func (80%) create mode 100644 shell.lib diff --git a/.bash-shell-ignore b/.bash-shell-ignore index d9f5bfe..abd5a1e 100644 --- a/.bash-shell-ignore +++ b/.bash-shell-ignore @@ -1,5 +1,12 @@ +# hidden files and directories are ignored by default +# i.e. beginning with . +# no ignores here may begin with . *.off *.md *TODO* +*.log LICENSE *.sample +*archive*/ +off/ +log*/ diff --git a/.bash-shell-include b/.bash-shell-include new file mode 100644 index 0000000..b63aa53 --- /dev/null +++ b/.bash-shell-include @@ -0,0 +1,6 @@ +function +alias +env +misc +lang +app diff --git a/load.sh b/load.sh index 7fc36bd..ea8da2b 100755 --- a/load.sh +++ b/load.sh @@ -5,6 +5,8 @@ DIR=${1:-$(dirname ${BASH_SOURCE[0]})} +[[ $(declare -F | grep module_load) ]] && echo module lib already loaded || source "$BASH_SHELL_BASE/module.lib" + # uncomment for debuggin # echo $USER running load script in $DIR # echo callers @@ -13,48 +15,22 @@ DIR=${1:-$(dirname ${BASH_SOURCE[0]})} # echo ----- # echo $BASH_SHELL_DIRS -# source the required directory source function -if [ ! -e "$BASH_SHELL_BASE/source-dir.func" ]; then - echo "!Unable to load sourcing function at $DIR/source-dir, shell not set up!" - return 1 -else - source "$BASH_SHELL_BASE/source-dir.func" -fi +module_load file +[[ $? -ne 0 ]] && echo unable to access the file module, aboarting load && return 1 -# if [ ! -e "$BASH_SHELL_BASE/module.lib" ]; then -# echo "!Unable to load module library functions function at $DIR/module.lib, shell not set up!" -# return 1 -# else -# source "$BASH_SHELL_BASE/module.lib" -# fi - -function shell_get_ignores () ( -local file=${1:-"$BASH_SHELL_BASE"}/.bash-shell-ignore -[[ ! -f "$file" ]] && [[ $1 ]] && return 1 - local excludes='' - local ignores=$(cat "$file") - for exclude in $ignores ; do - excludes+='"'$exclude'" ' - done - echo $excludes - return 0 -) - - function shell_process_directory () { +function shell_process_directory () { local SUBDIRS local DIR - local DSUBDIRS - local excludes DIR=${1:-$BASH_SHELL_BASE} # echo soucring directory $DIR - if [[ $DIR = "$BASH_SHELL_BASE" ]]; then - BASH_SHELL_IGNORE=$(shell_get_ignores) - excludes=$BASH_SHELL_IGNORE - else - excludes=$(shell_get_ignores $DIR) - [[ $? -ne 0 ]] && excludes=$BASH_SHELL_IGNORE - fi + # if [[ $DIR = "$BASH_SHELL_BASE" ]]; then + # BASH_SHELL_IGNORE=$(shell_get_ignores) + # excludes=$BASH_SHELL_IGNORE + # else + # excludes=$(shell_get_ignores $DIR) + # [[ $? -ne 0 ]] && excludes=$BASH_SHELL_IGNORE + # fi if [ -d "$DIR" ]; then if [ "$DIR" = "$BASH_SHELL_BASE" ] && [ "$BASH_SHELL_BASE_LOADED" = "true" ]; then @@ -69,30 +45,27 @@ local file=${1:-"$BASH_SHELL_BASE"}/.bash-shell-ignore fi fi - if [ -f "$DIR/load.sh" ] && [ "$DIR" != "$BASH_SHELL_BASE" ]; then - # echo "$DIR/$BASH_SHELL_LOAD" found, running instead of default processing - source "$DIR/load.sh" + if [[ $DIR != "$BASH_SHELL_BASE" ]] && [[ -f $DIR/load.sh ]]; then + # echo "$DIR/load.sh" found, running instead of default load + source "$DIR/load.sh" else # default processing local SUBDIRS - local DSUBDIRS - # default subdirectories to source - # TODO allow passing in additional directories + local IGNORE_FILE + SUBDIRS=$([[ -f "$DIR/.bash-shell-include" ]] && cat "$DIR/.bash-shell-include" || echo $BASH_SHELL_BASE_SUBDIRS) + IGNORE_FILE="$([[ -f "$DIR/.bash-shell-ignore" ]] && echo "$DIR" || echo "$BASH_SHELL_BASE")/.bash-shell-ignore" + for SUBDIR in $SUBDIRS; do + if [ -e "$DIR/$SUBDIR" ]; then - DSUBDIRS="function alias env misc lang app" - IFS=' ' read -r -a SUBDIRS <<< "${2:-$DSUBDIRS}" - # echo ${SUBDIRS[@]} - for SUBDIR in "${SUBDIRS[@]}" - do - if [ -e "$DIR/$SUBDIR" ]; then - # echo processing subdirectory $DIR/$SUBDIR - # must quote all globs to avoid bash glob expansion which is likely on - # TODO have default set of ignores plus check for .shell-ignore - # source_dir -p "archive" -x '"*.off" "*.md" "*TODO*" "LICENSE" "*.sample"' -d 0 $DIR/$SUBDIR - source_dir -p "archive" -x "$excludes" -d 0 $DIR/$SUBDIR - # else - # echo no subdirectory $DIR/$SUBDIR to process, ignorning - fi + # echo processing subdirectory $DIR/$SUBDIR + # must quote all globs to avoid bash glob expansion which is likely on + # TODO have default set of ignores plus check for .shell-ignore + # source_dir -p "archive" -x '"*.off" "*.md" "*TODO*" "LICENSE" "*.sample"' -d 0 $DIR/$SUBDIR + # source_dir -p "archive" -x "$excludes" -d 0 $DIR/$SUBDIR + source_dir -f "$IGNORE_FILE" -d 0 $DIR/$SUBDIR + # else + # echo no subdirectory $DIR/$SUBDIR to process, ignorning + fi done fi # echo "done sourcing directory $DIR" @@ -109,6 +82,9 @@ unset BASH_SHELL_LOADED # echo bash shell dirs: $BASH_SHELL_DIRS DIRS=${1:-$BASH_SHELL_DIRS} +BASH_SHELL_BASE_SUBDIRS=$(cat "$BASH_SHELL_BASE/.bash-shell-include") +# echo subdir includes: $BASH_SHELL_BASE_SUBDIRS + for dir in $DIRS; do # echo $dir shell_process_directory $dir diff --git a/module.lib b/module.lib index dc25fb5..98cbed1 100644 --- a/module.lib +++ b/module.lib @@ -1,7 +1,5 @@ #!/bin/bash - -echo loading module library - +# echo loading module library module_confirm () { local FILE local DIR @@ -9,7 +7,7 @@ local NAME DIR=$1 NAME=$2 # echo finding $NAME in $DIR -FILE="$(find ${DIR} -type f -name "${NAME}.mod" -o -name "${NAME}".sh)" +FILE="$(find ${DIR} -type f -name "${NAME}.mod" -o -name "${NAME}".lib -o -name "${NAME}".func -o -name "${NAME}".sh)" # echo files found $FILE COUNT=0 if [ "$FILE" ]; then @@ -76,7 +74,6 @@ FILE=$(module_find $1) # source $FILE "$0" [[ $BASHPID -eq $$ ]] || echo $FILE source $FILE - return 0 } diff --git a/modules/utility/file.lib b/modules/utility/file.lib new file mode 100644 index 0000000..a90cf81 --- /dev/null +++ b/modules/utility/file.lib @@ -0,0 +1,156 @@ +#!/bin/bash + +function lines_2_str () { + [[ ! -f "$1" ]] && return 1 + local str='' + # echo the lines: $lines >&2 + # lines="$(cat "$1")" + while read line; do + [[ "$line" =~ ^[[:space:]]*# ]] && continue + # echo line: "${line}" >&2 + str="$str ${line}" + # echo str: $str >&2 +done < $1 + # for line in $lines ; do + # str+='"'$line'" ' + # done + echo $str + # set +o noglob + return 0 +} + +function super_find () { + # echo super_find $@ >&2 +# USAGE +# all option arguments that contain globs/wildcards must be quoted to avoid expansion +# f sets path and file excludes from a supplied file path +# all lines ending in / will be treated as directory names to ignore, otherwise files +# p option explictly excludes paths(directories) +# d option sets the directory depth which is current directy by default, 0 is all +# x excitly excludes file globs as a string +# n inclucdes only file globs otherwise it's all except .files +# if no directory is given it will attempt to source the present working directory +# example: +# source_dir -p "archive" -x '"*.off" "*.md"' -d 0 # $DIR/$SUBDIR +declare OPTION +declare OPTARG +declare OPTIND +local EXCLUDE_FILE +local PATHS +local NAMES +local ENAMES +local DEPTH=1 +local VERBOSE=0 + +while getopts 'p:d:e:n:f:' OPTION; do +case "$OPTION" in + f) + EXCLUDE_FILE=$OPTARG + # echo EXCLUDE FILE $EXCLUDE_FILE >&2 + ;; + p) + # PATHS=("$OPTARG") + IFS=',' read -r -a PATHS <<< "$OPTARG" + # echo EXCLUDING THESE PATHS ${PATHS[*]} + ;; + e) + IFS=',' read -r -a ENAMES <<< "${OPTARG}" + # echo EXCLUDING THESE FILE NAMES ${ENAMES[*]} + ;; + n) + # NAMES=("$OPTARG") + IFS=',' read -r -a NAMES <<< "${OPTARG}" + # echo INCLUDING ONLY THESE FILE NAMES ${NAMES[*]} + ;; + d) + DEPTH=$OPTARG + # echo "SOURCING TO DEPTH (0=any)" "$DEPTH" + ;; + *) + echo unknown option $OPTION + ;; +esac +done + +shift $(( OPTIND - 1 )) + +local DIR +DIR="$*" +if [ ! "$DIR" ]; then + if [ -v PS1 ]; then + echo no directory to source provided + echo sourcing present working directory $(pwd) + read -p "Do you want to continue? " -n 1 -r + [[ $REPLY =~ ^[Yy]$ ]] && DIR=$(pwd) || return 1 + else + return 1 + fi +fi + +[ ! -d "$DIR" ] && echo " directory $DIR does not exist, aborting" && return 1 + +# echo dir $DIR + +local FIND +FIND="find $DIR" +FIND+=$([ ! $DEPTH == 0 ] && echo " -maxdepth $DEPTH ") +# ignore all .name and .path by default +FIND+=" -type f ! -path \"*/.*/*\" ! -name \".*\" " + +local name +local path + +if [[ -f $EXCLUDE_FILE ]]; then + local ignores=$(lines_2_str "$EXCLUDE_FILE") + # echo ignores: $ignores >&2 + for exclude in $ignores ; do + # echo exclude: ${exclude} >&2 + [[ "$exclude" == */ ]] && PATHS+=("${exclude::-1}") || ENAMES+=("$exclude") + done +fi + +# echo paths ${PATHS[@]} >&2 +# echo exclude names ${ENAMES[@]} >&2 + +if [[ ${PATHS[0]} ]]; then + for path in ${PATHS[@]}; do + # echo excluding $path + FIND+=$(echo ' ! -path "*/'$path'/*"') + done +fi + +if [[ ${ENAMES[0]} ]]; then + for name in ${ENAMES[@]}; do + # echo excluding name $name + FIND+=$(echo ' ! -name "'${name}'"') + done +fi + +if [[ ${NAMES[0]} ]]; then + for name in ${NAMES[@]}; do + # echo only finding $name + FIND+=$(echo " -name ${name}") + done +fi + +# echo +# echo find dir: $DIR >&2 +# echo find command: $FIND >&2 + +local FILES +FILES=$(eval $FIND | sort) +echo $FILES +return 0 +} + +source_dir () { + # echo passed: $* + local FILES + FILES=$(super_find "$@") + [[ $? -ne 0 ]] && return 1 + for f in $FILES; do + # echo sourcing: $f + source "$f" + done + +} diff --git a/source-dir.func b/modules/utility/source.func similarity index 80% rename from source-dir.func rename to modules/utility/source.func index d74fdc4..17a2b2d 100644 --- a/source-dir.func +++ b/modules/utility/source.func @@ -1,36 +1,43 @@ #!/bin/bash -source_dir () { + +function source_dir () { # USAGE # will never source source .files or .directories # all option arguments that contain globs/wildcards must be quoted to avoid expansion +# f sets path and file excludes from a supplied file path # p option excludes additional paths below of the given name (and depth) # d option sets the directory depth which is current directy by default, 0 is all # x excludes file globs -# f inclucdes only file globs otherwise it's all except .files +# n inclucdes only file globs otherwise it's all except .files # if no directory is given it will attempt to source the present working directory # example: # source_dir -p "archive" -x '"*.off" "*.md"' -d 0 # $DIR/$SUBDIR declare OPTION declare OPTARG declare OPTIND +local EXCLUDE_FILE local PATHS local NAMES local ENAMES local DEPTH=1 local VERBOSE=0 -while getopts 'p:d:x:f:' OPTION; do +while getopts 'p:d:e:n:f:' OPTION; do case "$OPTION" in + f) + EXCLUDE_FILE=$OPTARG + # echo EXCLUDE FILE $EXCLUDE_FILE + ;; p) # PATHS=("$OPTARG") IFS=',' read -r -a PATHS <<< "$OPTARG" # echo EXCLUDING THESE PATHS ${PATHS[*]} ;; - x) + e) IFS=',' read -r -a ENAMES <<< "${OPTARG}" # echo EXCLUDING THESE FILE NAMES ${ENAMES[*]} ;; - f) + n) # NAMES=("$OPTARG") IFS=',' read -r -a NAMES <<< "${OPTARG}" # echo INCLUDING ONLY THESE FILE NAMES ${NAMES[*]} @@ -75,6 +82,13 @@ FIND+=" -type f ! -path \"*/.*/*\" ! -name \".*\" " local name local path +if [[ -f $EXCLUDE_FILE ]]; then + local ignores=$(cat "$EXLUCDE_FILE") + for exclude in $ignores ; do + [[ "$exclude" =~ '/'$ ]] && PATHS+=(${exclude::-1}) || ENAMES+=($exclude) + done +fi + if [[ ${PATHS[0]} ]]; then for path in ${PATHS[@]}; do # echo excluding $path @@ -96,16 +110,16 @@ if [[ ${NAMES[0]} ]]; then done fi -# echo -# echo find dir: $DIR -# echo find command: $FIND +echo +echo find dir: $DIR +echo find command: $FIND local FILES FILES=$(eval $FIND | sort) # echo $FILES | xargs -n 1 for f in $FILES; do - # echo sourcing: $f - source "$f" + echo sourcing: $f + # source "$f" done } diff --git a/shell.env b/shell.env index 8d26c12..7ab11dc 100644 --- a/shell.env +++ b/shell.env @@ -33,7 +33,7 @@ export BASH_SHELL_LOAD=$BASH_SHELL_BASE/load.sh # load.sh is default export BASH_SHELL_STARTUP=$BASH_SHELL_BASE/startup.sh # strtup.sh is default # if uncommented next line sets up implicit sourcing for non-interactive shells -export BASH_ENV=$BASH_SHELL_LOAD # use base repo +# export BASH_ENV=$BASH_SHELL_LOAD # use base repo # if not using implicit sourcing for non-interactive shells then on can do this per script ################## # expanding aliases is optional @@ -46,7 +46,7 @@ export BASH_ENV=$BASH_SHELL_LOAD # use base repo # identify a network name that this host resides on # make a directory of the same name # if unset then only /all will be sourced -export NETWORK_DOMAINS=(238.kebler.net 645.kebler.net) +export NETWORK_DOMAINS=(238.kebler.net 645.kebler.net 3115.kebler.net) # network domain folder entry will be made for each domain set, first is home domain, others via vpn if [[ $NETWORK_DOMAINS ]]; then export NETWORK_HOME=${NETWORK_DOMAINS[0]} diff --git a/shell.lib b/shell.lib new file mode 100644 index 0000000..ee20ed5 --- /dev/null +++ b/shell.lib @@ -0,0 +1,135 @@ +#!/bin/bash + +function lines_2_str () ( + [[ ! -f "$1" ]] && return 1 + local str='' + local lines=$(cat "$1") + for line in $lines ; do + str+='"'$line'" ' + done + echo $str + return 0 +) + +function source_dir () { +# USAGE +# will never source source .files or .directories +# all option arguments that contain globs/wildcards must be quoted to avoid expansion +# p option excludes additional paths below of the given name (and depth) +# d option sets the directory depth which is current directy by default, 0 is all +# x excludes file globs +# f inclucdes only file globs otherwise it's all except .files +# if no directory is given it will attempt to source the present working directory +# example: +# source_dir -p "archive" -x '"*.off" "*.md"' -d 0 # $DIR/$SUBDIR +declare OPTION +declare OPTARG +declare OPTIND +local EXCLUDE_FILE +local PATHS +local NAMES +local ENAMES +local DEPTH=1 +local VERBOSE=0 + +while getopts 'p:d:e:n:f:' OPTION; do +case "$OPTION" in + f) + EXCLUDE_FILE=$OPTARG + # echo EXCLUDE FILE $EXCLUDE_FILE + ;; + p) + # PATHS=("$OPTARG") + IFS=',' read -r -a PATHS <<< "$OPTARG" + # echo EXCLUDING THESE PATHS ${PATHS[*]} + ;; + e) + IFS=',' read -r -a ENAMES <<< "${OPTARG}" + # echo EXCLUDING THESE FILE NAMES ${ENAMES[*]} + ;; + n) + # NAMES=("$OPTARG") + IFS=',' read -r -a NAMES <<< "${OPTARG}" + # echo INCLUDING ONLY THESE FILE NAMES ${NAMES[*]} + ;; + d) + DEPTH=$OPTARG + # echo "SOURCING TO DEPTH (0=any)" "$DEPTH" + ;; + *) + echo unknown option $OPTION + ;; +esac +done + +shift $(( OPTIND - 1 )) + +local DIR +DIR="$@" +if [ ! "$DIR" ]; then + if [ -v PS1 ]; then + echo no directory to source provided + echo sourcing present working directory $(pwd) + read -p "Do you want to continue? " -n 1 -r + [[ $REPLY =~ ^[Yy]$ ]] && DIR=$(pwd) || return 1 + else + return 1 + fi +fi + +[ ! -d "$DIR" ] && echo " directory $DIR does not exist, aborting" && return 1 + +# echo dir $DIR + +local FIND +FIND="find $DIR" +FIND+=$([ ! $DEPTH == 0 ] && echo " -maxdepth $DEPTH ") +# ignore all .name and .path by default +# TODO change to having a string +FIND+=" -type f ! -path \"*/.*/*\" ! -name \".*\" " + + +local name +local path + +if [[ -f $EXCLUDE_FILE ]]; then + local ignores=$(cat "$EXLUCDE_FILE") + for exclude in $ignores ; do + [[ "$exclude" =~ '/'$ ]] && PATHS+=(${exclude::-1}) || ENAMES+=($exclude) + done +fi + +if [[ ${PATHS[0]} ]]; then + for path in ${PATHS[@]}; do + # echo excluding $path + FIND+=$(echo ' ! -path "*/'$path'/*"') + done +fi + +if [[ ${ENAMES[0]} ]]; then + for name in ${ENAMES[@]}; do + # echo excluding name $name + FIND+=$(echo " ! -name ${name}") + done +fi + +if [[ ${NAMES[0]} ]]; then + for name in ${NAMES[@]}; do + # echo only finding $name + FIND+=$(echo " -name ${name}") + done +fi + +echo +echo find dir: $DIR +echo find command: $FIND + +local FILES +FILES=$(eval $FIND | sort) +# echo $FILES | xargs -n 1 + for f in $FILES; do + echo sourcing: $f + # source "$f" +done + +} diff --git a/startup.sh b/startup.sh index 9635d84..003b431 100755 --- a/startup.sh +++ b/startup.sh @@ -1,30 +1,39 @@ #!/bin/bash function startup_load () { - local SDIRS local SDIR local DIRS -[[ $1 ]] && DIRS=("$1") || DIRS=("${BASH_SHELL_DIRS}") -llog "startup directories to try ${DIRS[*]}" +local DIR +local excludes + + +[[ $(declare -F | grep module_load) ]] && echo module lib already loaded || source "$BASH_SHELL_BASE/module.lib" +module_load file +[[ $? -ne 0 ]] && echo unable to access the file module, aboarting load && return 1 + + +[[ $1 ]] && DIRS=($1) || DIRS=(${BASH_SHELL_DIRS}) +# llog "startup directories to try ${DIRS[*]}" SDIRS=() j=0 cnt=${#DIRS[@]} for ((i=0;i