add: tools to shell directory includes

feat: refactor repo organization that include corresponding changes to load script, module library plus shell.env
add: python based yaml parser
add:  functions isDir isFile to corresponding modules
master
Kebler Network System Administrator 2022-02-20 12:44:29 -08:00
parent ec601fbb15
commit 784e6711a8
11 changed files with 147 additions and 175 deletions

View File

@ -1,6 +1,6 @@
# hidden files and directories are ignored by default
# i.e. beginning with .
# no ignores here may begin with .
# so NO ignores here may begin with .
*.off
*.example
*.tmpl

View File

@ -4,3 +4,4 @@ env
misc
lang
app
tools

View File

@ -67,3 +67,6 @@ alias follow="readlink -f"
# Will scrub all and reload only aliases in .bash_aliaes
alias reloada="unalias -a && source ~/.bash_aliases && compgen -a"
# use this when there are transport errors after unmounting or killing a mount
alias umountf="sudo umount -l"

11
env/modules.env vendored
View File

@ -1,7 +1,6 @@
# All the directories which contain based modules/libraries
# module libraries will be look up in this order
# first set any additional modules directories to search (if any)
# MODULE_DIRS=()
# Uncomment and add directories to this variable for
# searching custom directories for shell modules
# MODULE_DIRS=(some/dir another/dir)
# those will be searched first then /modules under the shell directories
# user,host,network,base in that order
# this provide a way to override a module with a more specific version
# based on $BASH_SHELL_DIRS
# this ordering provides a way to override a module with a more specific version

122
load.sh
View File

@ -1,33 +1,13 @@
#!/bin/bash
# [[ ! $- == *i* ]] && exec >> ~/logs/load.log && exec 2>&1
# export BASH_DEBUG=true
function nilog () {
[[ ! $- == *i* ]] && echo -e "-----\n $USER $*" &>> ~/logs/load.log
}
nilog "called load.sh $(date)"
# nilog caller: $(caller)
# nilog pid: $(ps -o comm= $PPID)
# PRIMARY SHELL LOAD SCRIPT - RUN FOR ALL INTERACTIVE SHELLS
# don't bother if bash is not the shell
[ ! "$SHELL" = "/bin/bash" ] && return 1
DIR=${1:-$(dirname ${BASH_SOURCE[0]})}
# ************ be sure module_load function is availABLE ***************************
[[ $(declare -F | grep module_load) ]] || source "$BASH_SHELL_BASE/module.lib"
# uncomment for debuggin
# echo $USER running load script in $DIR
# echo callers
# caller
# echo $(ps -o comm= $PPID)
# echo -----
# echo $BASH_SHELL_DIRS
module_load file
[[ $? -ne 0 ]] && echo unable to access the file module, aboarting load && return 1
module_load debug
if [[ $? -ne 0 ]]; then
echo "unable to load a 'debug' module using a noop for debug function"
@ -39,102 +19,12 @@ fi
([[ $BASH_ENV ]] && [[ ! $- == *i* ]] && [[ $BASH_USE_ALIAS ]]) && shopt -s expand_aliases
function shell_process_directory () {
local SUBDIRS
local DIR
DIR=${1:-$BASH_SHELL_BASE}
# echo soucring directory $DIR
# if [[ $DIR = "$BASH_SHELL_BASE" ]]; then
# BASH_SHELL_IGNORE=$(shell_get_ignores)
# excludes=$BASH_SHELL_IGNORE
# else
# excludes=$(shell_get_ignores $DIR)
# [[ $? -ne 0 ]] && excludes=$BASH_SHELL_IGNORE
# fi
if [ -d "$DIR" ]; then
if [ "$DIR" = "$BASH_SHELL_BASE" ] && [ "$BASH_SHELL_BASE_LOADED" = "true" ]; then
if [ -v PS1 ]; then
echo base directory already sourced
read -p "do you want to re-source the base (not recommended)? " -n 1 -r
echo
[[ ! $REPLY =~ ^[Yy]$ ]] && return 1
# else
# echo non-interactive shell
# return 1
fi
fi
if [[ $DIR != "$BASH_SHELL_BASE" ]] && [[ -f $DIR/load.sh ]]; then
# echo "$DIR/load.sh" found, running instead of default load
source "$DIR/load.sh"
else
# default processing
local SUBDIRS
local IGNORE_FILE
SUBDIRS=$([[ -f "$DIR/.bash-shell-include" ]] && cat "$DIR/.bash-shell-include" || echo $BASH_SHELL_BASE_SUBDIRS)
[[ $SSH_SESSION ]] && SUBDIRS+=" ssh/session"
IGNORE_FILE="$([[ -f "$DIR/.bash-shell-ignore" ]] && echo "$DIR" || echo "$BASH_SHELL_BASE")/.bash-shell-ignore"
for SUBDIR in $SUBDIRS; do
if [ -e "$DIR/$SUBDIR" ]; then
# echo processing subdirectory $DIR/$SUBDIR
# must quote all globs to avoid bash glob expansion which is likely on
# TODO have default set of ignores plus check for shell-ignore
# source_dir -p "archive" -x '"*.off" "*.md" "*TODO*" "LICENSE" "*.sample"' -d 0 $DIR/$SUBDIR
# source_dir -p "archive" -x "$excludes" -d 0 $DIR/$SUBDIR
source_dir -f "$IGNORE_FILE" -d 0 $DIR/$SUBDIR
# else
# echo no subdirectory $DIR/$SUBDIR to process, ignorning
fi
done
fi
# echo "done sourcing directory $DIR"
# else
# echo $DIR does not exist nothing to source
fi
}
# process the base directory by default
unset BASH_SHELL_BASE_LOADED
unset BASH_SHELL_LOADED
# add in any network directories
declare networks
source "$HOME/.bashrc"
networks="$HOME/${BASH_SHELL_USER:-"shell"}/.networks"
NETWORK_DOMAINS="$([[ -f $networks ]] && cat $networks) "
# These will be loaded for all users
networks="$BASH_SHELL_HOST/$(hostname)/.networks"
NETWORK_DOMAINS+="$([[ -f $networks ]] && cat $networks)"
NETWORK_DOMAINS=(${NETWORK_DOMAINS})
if [[ ! $NETWORK_DOMAINS = "" ]]; then
declare domain
BASH_SHELL_NETWORK_DIRS=""
for domain in "${NETWORK_DOMAINS[@]}"; do
# echo processing ${BASH_SHELL_NETWORK}/${domain}
if [[ ${domain} = !* ]]; then
domain=${domain/#!/}
export NETWORK_HOME=$domain
fi
if [[ -d ${BASH_SHELL_NETWORK}/${domain} ]]; then
BASH_SHELL_NETWORK_DIRS+="${BASH_SHELL_NETWORK}/${domain} "
fi
done
unset domain
export BASH_SHELL_NETWORK_DIRS
BASH_SHELL_DIRS+=" ${BASH_SHELL_NETWORK_DIRS}"
fi
# echo bash shell dirs: $BASH_SHELL_DIRS
dirs=${1:-$BASH_SHELL_DIRS}
BASH_SHELL_BASE_SUBDIRS=$(cat "$BASH_SHELL_BASE/.bash-shell-include")
# echo subdir includes: $BASH_SHELL_BASE_SUBDIRS
for dir in $dirs; do
module_load shell-process-directory
for dir in ${1:-$BASH_SHELL_DIRS}; do
# echo $dir
shell_process_directory $dir
[[ $dir == "$BASH_SHELL_BASE" ]] && BASH_SHELL_BASE_LOADED=true
@ -142,6 +32,4 @@ done
export BASH_SHELL_LOADED=true
# process user
# Note: $HOME/shell or $HOME/BASH_SHELL_USER are sourced via $HOME/.bashrc
# echo $(envg BASH) | xargs -n 1
# Note: $HOME/shell or $HOME/BASH_SHELL_USER are processed via $HOME/.bashrc

View File

@ -34,36 +34,37 @@ local MDIRS
local MDIR
local DIRS
local MODULE=$1
# Precidence is user. current hosts, all hosts, current network, all networks, base
DIRS=( \
$([ $BASH_SHELL_USER ] && echo $HOME/$BASH_SHELL_USER || echo $HOME/shell) \
$BASH_SHELL_HOST/$(hostname)
$BASH_SHELL_HOST/all \
$BASH_SHELL_NETWORK/$NETWORKNAME \
$BASH_SHELL_NETWORK/all \
$BASH_SHELL_BASE \
)
# echo ${DIRS[@]}
MDIRS=()
j=0
cnt=${#DIRS[@]}
for ((i=0;i<cnt;i++)); do
DIR="${DIRS[i]}/modules"
[ -d $DIR ] && MDIRS[j]=$DIR && j+=1
done
# MODULE_DIRS is array set in the environment and is an array of additional directories to
# search for modules. This makes is easy to include a custom module libraries outside
# the shell system. These take precedence over any modules found in shell directories below
[ $MODULE_DIRS ] && MDIRS=("$MODULE_DIRS" "${MDIRS[@]}")
# echo Module Directories ${MDIRS[@]}
for MDIR in "${MDIRS[@]}"
if [[ $MODULE_DIRS ]]; then
for DIR in "${MODULE_DIRS[@]}"
do
# echo mdir $MDIR name $MODULE
RES=$(module_confirm "$MDIR" "$MODULE")
if [[ -d $DIR ]]; then
RES=$(module_confirm "$DIR" "$MODULE")
[ $? -eq 0 ] && echo $RES && return 0
fi
done
fi
# BASH_SHELL_DIRS holds shell directories to search for modules. If will be searched
# in revsere order, user or network repos over host over base
DIRS=( ${BASH_SHELL_DIRS:-$BASH_SHELL_BASE} )
[ -d $HOME/$BASH_SHELL_USER ] && DIRS=("${DIRS[@]}" "$HOME/$BASH_SHELL_USER")
cnt=${#DIRS[@]}
for ((i=1;i<=cnt;i++)); do
# find modules in reverse order so more specific ones override
DIR="${DIRS[cnt-i]}/modules"
if [[ -d $DIR ]]; then
RES=$(module_confirm "$DIR" "$MODULE")
[ $? -eq 0 ] && echo $RES && return 0
fi
done
# no module found anywhere
return 1
}

View File

@ -6,3 +6,37 @@ function debug () {
# alias debug_on="sudo -i uncomment BASH_DEBUG /etc/bash.bashrc"
# alias debug_off="sudo -i comment BASH_DEBUG /etc/bash.bashrc"
# *************** DEBUGGING ***********************
# module_load debug
# if [[ $? -ne 0 ]]; then
# echo "unable to load a 'debug' module using a noop for debug function"
# # noop
# function debug () {
# :
# }
# fi
# [[ ! $- == *i* ]] && exec >> ~/logs/load.log && exec 2>&1
# export BASH_DEBUG=true
# uncomment for debugging
# echo $USER running load script in $DIR
# echo callers
# caller
# echo $(ps -o comm= $PPID)
# echo -----
# echo $BASH_SHELL_DIRS
# ******************END DEBUGGING *******************************
# ***************** LOGGING *****************
# function nilog () {
# [[ ! $- == *i* ]] && echo -e "-----\n $USER $*" &>> ~/logs/load.log
# }
# nilog "called load.sh $(date)"
# nilog caller: $(caller)
# nilog pid: $(ps -o comm= $PPID)
# *

View File

@ -31,6 +31,16 @@
# Usage message
isDir() {
if [[ -d $1 ]]
then
echo "true"
return 0
else
return 1
fi
}
chmodr () {
usage()

View File

@ -3,6 +3,16 @@
# export BASH_DEBUG=true
module_load debug
isFile() {
if [[ -f $1 ]]
then
echo "true"
return 0
else
return 1
fi
}
build_file () {
[[ -f "$2" ]] || (echo "output file $2 does not exist";return 1)
if [[ -f "$1" ]]; then

View File

@ -18,3 +18,8 @@ parse_yaml() {
}'
}
yaml() {
python3 -c "import yaml;print(yaml.safe_load(open('$1'))$2)"
}

View File

@ -1,13 +1,16 @@
#!/bin/bash
# Shell Repos Environment
# Customize Bash Shell setup here
# if bash is not installed then don't bother to continue
! command -v bash >/dev/null 2>&1 && echo no bash command && return 1
export SHELL=$(command -v bash )
# sourced for non-login interactive shells
# sourced via /etc/bash.bashrc so for all machine users
# $BASH_SHELL_BASE # this MUST be set in /etc/profile
# $BASH_SHELL_HOST=$BASH_SHELL_BASE
# $BASH_SHELL_NETWORK/all
# $BASH_SHELL_NETWORK/(array of Domains)
# $BASH_NETWORK_DIR # extra directory to look for networks, $BASH_SHELL_NETWORK is always checked
# $BASH_SHELL_HOST/all
# $BASH_SHELL_HOST/<hostname>
# also
@ -25,22 +28,23 @@ export SHELL=$(command -v bash )
# Using base shell setup repository
# https://git.kebler.net/kebler.net/bash-shell-base.git
# use these two if you want a common directory for all shell repos
# BASH_SHELL_BASE is set in /etc/profile
# load script in base repo to run
# BASH_SHELL_BASE is set in /etc/profile do not change here!
export BASH_SHELL_LOAD=$BASH_SHELL_BASE/load.sh # load.sh is default
export BASH_SHELL_STARTUP=$BASH_SHELL_BASE/startup.sh # strtup.sh is default
# if uncommented next lines sets up implicit sourcing for non-interactive shells
# echo ----NON_INTERACTIVE SHELL INFO-----
# echo ----NON_INTERACTIVE SHELL-----
# echo enabling bash shell repos for non-inactive shell
# export BASH_ENV=$BASH_SHELL_LOAD # same as interactive shell, beware usually too much
export BASH_ENV=$BASH_SHELL_BASE/module.lib # only load module loading functions
export BASH_ENV=$BASH_SHELL_BASE/module.lib # load module loading functions
# echo enabling aliases with non-interactive shell
export BASH_USE_ALIAS=true # will source aliases for non-interactive
# echo see $BASH_SHELL_BASE/shell.env
# echo ---------
# if not using implicit sourcing for non-interactive shells then on can do this per script
# if not using implicit sourcing for non-interactive shells then one can do this per script
##################
# expanding aliases is optional
# shopt -s expand_aliases
@ -53,33 +57,51 @@ export BASH_USE_ALIAS=true # will source aliases for non-interactive
([ -n "$SSH_CONNECTION" ] || [ -n "$SSH_CLIENT" ] || [ -n "$SSH_TTY" ]) && export SSH_SESSION=true
BASH_SHELL_DIRS="$BASH_SHELL_BASE "
declare parent
parent=$(dirname $BASH_SHELL_BASE)
export BASH_SHELL_HOST=$([[ -d $parent/host ]] && echo $parent/host || echo /opt/bash/shell/host)
# or comment above and set explictly
# Any Host
BASH_SHELL_ANY_HOST=${parent}/any/host # or set to
# export BASH_SHELL_ALL_HOSTS=<some directory>
# Host
BASH_SHELL_HOST=$parent/$(hostname)
# export BASH_SHELL_HOST=<some directory>
# identify a network name that this host resides on
# make a directory of the same name
export BASH_SHELL_NETWORK=$([[ -d $parent/network ]] && echo $parent/network || echo /opt/bash/shell/network)
# or comment above and set explictly
# export BASH_SHELL_NETWORK=<some directory>
BASH_SHELL_ANY_NETWORK=$parent/any/network
# export BASH_SHELL_ALL_NETWORKS=<some directory>
# by default USER shell sources will be looked for under $HOME/shell
# but the default directory for all users can be set manually below
# export BASH_SHELL_USER=<some directory under $HOME>
# add this export to .bashrc for custom directory. This can be done during or after using the user setup script
temp="
$BASH_SHELL_BASE
$BASH_SHELL_NETWORK/all
$BASH_SHELL_HOST/all
$BASH_SHELL_HOST/$(hostname)
"
[[ -d $BASH_SHELL_ANY_HOST ]] && BASH_SHELL_DIRS+="$BASH_SHELL_ANY_HOST " && export BASH_SHELL_ANY_HOST
[[ -d $BASH_SHELL_ANY_NETWORK ]] && BASH_SHELL_DIRS+="$BASH_SHELL_ANY_NETWORK " && export BASH_SHELL_ANY_NETWORK
export BASH_SHELL_DIRS=$(echo $temp)
# now load and export module loading functions library
source $BASH_SHELL_BASE/module.lib
module_load network-dirs
BASH_NETWORKS_DIR=$parent/networks
# BASH_NETWORKS_DIR=<some directory> # alt directory to look for networks
export BASH_NETWORKS_DIR
# These will be loaded for all users on a host
network_dirs "$BASH_SHELL_HOST/.networks"
[[ -d $BASH_SHELL_HOST ]] && BASH_SHELL_DIRS+="$BASH_SHELL_HOST " && export BASH_SHELL_HOST
# these loaded only for specific user on a host
network_dirs "$HOME/${BASH_SHELL_USER:-"shell"}/.networks"
export BASH_SHELL_DIRS
# echo ALL DIRS: $BASH_SHELL_DIRS
# cat $HOME/.bashrc
# source "$HOME/.bashrc"
# env | grep BASH
# env | grep NETWORK_DIRS
@ -87,7 +109,6 @@ export BASH_SHELL_DIRS=$(echo $temp)
# echo $BASH_SHELL_DIRS
# echo ---------------------------------
# now load and export module loading functions library
source $BASH_SHELL_BASE/module.lib
# echo end shell env