fixing bundler

move some module code to new modules

install/scripts/copy-user.sh

@@ -27,8 +27,9 @@ if [[ ! $subdir == "shell" ]]; then
 fi
 
 sdir=$user_home/$subdir
-echo -e "***********  create UCI BASH Shell directories for user in $dir *******"
+echo -e "***********  create UCI BASH Shell directories for user in $sdir *******"
 # subdirs="alias env ssh/config ssh/mounts /startup"
+mkdir -p $sdir
 for dir in $(cat $BASH_SHELL_BASE/.bash-shell-include); do
  mkdir -p $sdir/$dir
 done

load/01-basic

@@ -50,3 +50,6 @@ get_shopt () {
 # echo set the system editor using EDITOR environment variable
 edit() { ${EDITOR:-nano} $@; }
 gedit() { ${GEDITOR:-xed} $@; }
+
+
+rsynchg() { rsync --help | grep "\-$1"; }

load/alias/00-basic

@@ -64,3 +64,10 @@ alias follow="readlink -f"
 alias umountf="sudo umount -l"
 
 alias loginust="loginctl show-user $USER" 
+
+
+# alias to add for live user
+if [ -v PS1 ]; then
+ alias dfind="_find -t d -d 0 -n "
+ alias sdfind="_find -s -t d -d 0 -n "
+fi

modules/install-shell-repos.lib

@@ -11,8 +11,8 @@ install_shell_repo () {
     [[ ! $EUID -eq 0 ]] && { echo ERROR script must be run as root; return; }
     local user
     [[ $1 == "-u" ]] && user=$2 && shift 2
-    local dir=${2:-/shell/any}
+    local dir=${2:-/shell}
-    local repo=${1:-host} 
+    local repo=${1:-base} 
     local rpath=$dir/$repo
     mkdir -p $rpath
     # TODO set git host via option or env var

modules/scripting/bundle.func

@@ -7,48 +7,94 @@
 # or
 # scpt=$(bundle myscript.sh)
 # source <(echo -e "$scpt")
+module_load debug
+module_load file
 
 bundle () {
 
     local file
-
+    local code
-    if [[ $1 == "-m" ]]; then 
+    local compact
+    if [[ $1 == "-c" ]]; then
+    module_load helpers
+    compact=true
     shift 1
-    file=$(module_find $1)
-    else
-    file=$1
     fi
     
-    [[ ! -f $file ]] && return 1
+    # todo add -c for compacting the bundle, change this to options parsing.
-    module_load file 
+    
+    if [[ "$1" ]]; then
+        if [[ -f $1 ]]; then
+          file=$1
+         else 
+          if [[ $1 == "-m" ]]; then
+                file=$(module_find $2)
+                shift 1
+             else 
+                if [[ $1 == "-mf" ]]; then
+                    module_load $2
+                    code="$(declare -f $3)"
+                    shift 1
+                 else
+                    code="$1"
+                    shift 1
+                fi 
+          fi  
+        fi
+     else
+        echo "aborting, no file, module or function passed for bundling"
+        return 1
+    fi    
 
-    if [[ ! $2 == "__recurse__" ]]; then
     tmp_file=$( mktemp -t TEMP_FILE_bundle.XXXXXXXX )
     chmod 600 "$tmp_file"
+    # if [[ $2 == "__recurse__" ]] || [[ $file ]]; then
+    if [[ $file ]]; then
       \cp $file $tmp_file
     else 
-    tmp_file=$file
+     echo "$code" > $tmp_file
     fi 
-    # echo current temp file: $tmp_file
+
-    modules=$(sed -n -e 's/^module_load //p' < $tmp_file)
+
-    # echo found: $modules
+    if [[ ! $2 == "__recurse__" ]]; then
-    # return
+    _modules=""
+    output=$2
+    debug outputing bundle to: $output
+    debug "bash code to bundle \n $(cat $tmp_file)"
+    fi
+  
+    modules=$(sed -n -e 's/^\s*module_load //p' < $tmp_file)
+
     if [[ $modules ]]; then
-        # echo Modules: $modules
+        # remove found module_load before continuing
-        sed -i '/module_load/d' $tmp_file
+        sed -i '/^\s*module_load/d' $tmp_file
+        [[ $_modules ]] && debug "already bundled modules: $_modules"
         for module in $(printf '%s\n' "${modules[@]}"|tac);
             do 
-            # echo module: $module  
+                if [[ $_modules == " $module "* ]]; then 
-            # echo module path: $(module_find $module) 
+                    [[ $_modules ]] && debug module $module already bundled  
-            prepend_file $(module_find $module) $tmp_file
+                else 
+                    if modp=$(module_find ${module}); then 
+                        debug bundling module ${module} at: $modp 
+                        prepend_file $modp $tmp_file
+                    fi
+                fi 
             done
+        _modules="$_modules $modules"
         bundle $tmp_file __recurse__
     fi
     
+    # the recursion is done
     if [[ ! $2 == "__recurse__" ]]; then
+       [[ $compact ]] && compact_file $tmp_file    
+        if [[ $BASH_DEBUG ]]; then
+            debug "\n ------------code as bundled--------------\n \
+            $(cat $tmp_file) \
+            \n ------------------------------------------------------------ \n" 
+            debug "bundled modules were $_modules"
+       fi     
        if [[ $2 ]]; then
-            \cp $tmp_file $2 
+            \cp $tmp_file $output 
-            # echo $2
           else 
             echo -e "$(cat $tmp_file)"
         fi

modules/scripting/helpers.lib

@@ -70,7 +70,8 @@ parse_option () {
 }
 
 remove_empty_lines () {
-if [[ -f $1 ]]; then cat $1; else echo "$1"; fi | sed -rz 's/^\n+//; s/\n+$/\n/g'
+if [[ -f $1 ]]; then cat $1; else echo "$1"; fi | sed -n '/^\s*$/!p'
+# sed -rz 's/^\n+//; s/\n+$/\n/g'
 }
 
 remove_end_spaces () {
@@ -93,6 +94,16 @@ sed_ignore_comments () {
     fi
 }
 
+compact_file () {
+  # removes trailing spaces, blank lines and comments
+#   echo -e "$1"
+# grep -v '^\s*#' $1
+[[ -f $1 ]] && sed -i -e '/^\s*#/d' -e '/^\s*$/d'  "$1"
+#   if [[ -f "$1" ]]; then cat "$1"; else echo -e "$@"; fi | sed 's/\\s*$//g'  | sed -r '/^\s*$/d' | grep -v '^\s*#' 
+}
+
+
+
 #rows between matched rows
 # sed '1,/firstmatch/d;/secondmatch/,$d'
 

modules/scripting/source.lib

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+source_dir () {
+  # BASH_DEBUG=source_dir
+  module_load find
+  debug source_dir passed arguments: $*
+  debug function: source_dir
+  local FILES
+  FILES=$(_find -l "$@") # find function
+  # echo $FILES >&2
+  [[ $? -ne 0 ]] && return 1
+    for f in $FILES; do
+    debug sourcing: $f >&2
+    source "$f"
+  done
+
+}

modules/shell-process-directory.mod

@@ -24,7 +24,7 @@ function shell_process_directory () {
     local SUBDIRS
     local DIR
 
-    module_load file
+    module_load source
     [[ $? -ne 0 ]] && echo unable to access the file module, aborting load && return 1
 
     DIR=${1:-$BASH_SHELL_BASE}

modules/utility/dir.lib

@@ -12,8 +12,7 @@ isDir() {
     fi
 }
 
-rsynchc() { rsync --help | grep "\-$1"; }
+
-alias rsynch="rsynchc"
 
 function dir_copy () {
 

modules/utility/file.lib

@@ -53,243 +53,42 @@ function lines_2_str () {
   echo $str
 }
 
-
-
-# find, superceeds find use `command find` to get the super
-function _find () {
-
-# BASH_DEBUG=find
-
-# USAGE
-# all option arguments that contain globs/wildcards must be quoted to avoid expansion
-# f sets path and file excludes from a supplied file path
-# all lines ending in /  will be treated as directory names to ignore, otherwise files
-# p option explictly excludes paths(directories)
-# d option sets the directory depth which is current directy by default, 0 is all
-# n inclucdes only file globs otherwise it's all except .files
-# if no directory is given it will attempt to source the present working directory
-# example:
-# source_dir -p "archive" -e '"*.off" "*.md"'  -d 0 # $DIR/$SUBDIR
-local EXCLUDE_FILE
-local PATHS
-local NAMES
-local ENAMES
-local DEPTH=1
-local HIDDEN
-local raw
-local mounted
-local usesudo
-
-mounted="-mount"
-
-declare OPTION
-declare OPTARG
-declare OPTIND
-
-while getopts 'mrst:p:d:e:n:f:hl' OPTION; do
-case "$OPTION" in
-  s)
-    usesudo=sudo
-    ;;
-  r)
-    raw=true
-    ;;
-  t)
-    TYPE=$OPTARG
-    # echo "TYPE $TYPE"
-    ;;
-  f)
-    EXCLUDE_FILE=$OPTARG
-    # echo EXCLUDE FILE $EXCLUDE_FILE >&2
-    ;;
-  p)
-    # PATHS=("$OPTARG")
-    IFS=',' read -r -a PATHS <<< "$OPTARG"
-    # echo EXCLUDING THESE PATHS ${PATHS[*]}
-    ;;
-  e)
-    IFS=',' read -r -a ENAMES <<< "${OPTARG}"
-    # echo EXCLUDING THESE FILE NAMES ${ENAMES[*]}
-    ;;
-  n)
-    # NAMES=("$OPTARG")
-    IFS=',' read -r -a NAMES <<< "${OPTARG}"
-    # NAMES=$OPTARG
-    ;;
-  d)
-    DEPTH=$OPTARG
-    # echo "SOURCING TO DEPTH (0=any)" "$DEPTH"
-    ;;
-  h)
-    HIDDEN=true
-    # echo "SOURCING TO DEPTH (0=any)" "$DEPTH"
-    ;;
-
-  l)
-    LINKS=-L
-    ;;
-  m)
-   # include mounted directories
-    mounted=""
-    ;;   
-  *)
-    echo unknown option $OPTION
-    ;;
-esac
-done
-
-shift $(( OPTIND - 1 ))
-
-if [[ $raw ]]; then
-  echo executing raw find command
-  $usesudo $(which find) "$@"
-fi
-
-local DIR
-DIR="$*"
-if [ ! "$DIR" ]; then
-  if [ -v PS1 ]; then
-  echo no directory provided to search
-  echo searching present working directory $(pwd)
-  read -p "Do you want to continue? " -n 1 -r
-  [[ $REPLY =~ ^[Yy]$ ]] && DIR=$(pwd) || return 1
-  else
-    return 1
-  fi
-fi
-
-[ ! -d "$DIR" ] && echo " directory $DIR does not exist, aborting" && return 1
-
-# echo dir $DIR
-
-local FIND
-FIND="$usesudo $(which find) $LINKS $DIR"
-FIND+=$([ ! $DEPTH == 0 ] && echo " -maxdepth $DEPTH ")
-# FIND+=" -type $([ $TYPE ] && echo "$TYPE" || echo "f")"
-TYPE=${TYPE:-f}
-debug exclude mounts: $mounted
-FIND+=" $mounted -type $TYPE "
-# include HIDDEN files and directories IS FALSE BY DEFULT
-[[ ! $HIDDEN ]] && FIND+="! -path \"*/.*/*\" ! -name \".*\" "
-
-local name
-local path
-
-if [[ -f $EXCLUDE_FILE ]]; then
-  local ignores=$(lines_2_str "$EXCLUDE_FILE")
-  # echo ignores: $ignores >&2
-  for exclude in $ignores ; do
-    # echo exclude: ${exclude} >&2
-    [[ "$exclude" == */ ]] && PATHS+=("${exclude::-1}") || ENAMES+=("$exclude")
-  done
-fi
-
-# echo paths ${PATHS[@]} >&2
-# echo exclude names ${ENAMES[@]} >&2
-
-set -o noglob
-
-if [[ ${PATHS[0]} ]]; then
-  for path in ${PATHS[@]}; do
-     # echo excluding $path
-     FIND+=$(echo ' ! -path "*/'$path'/*"')
-  done
-fi
-
-if [[ ${ENAMES[0]} ]]; then
-  for name in ${ENAMES[@]}; do
-     debug excluding name "$name"
-     FIND+=$(echo ' ! -name '$name'')
-  done
-fi
-
-debug "INCLUDING ONLY THESE FILE NAMES ${NAMES[*]}"
-if [[ ${NAMES[0]} ]]; then
-  for name in "${NAMES[@]}"; do
-      debug only finding $name
-     FIND+=$(echo " -name '${name}'")
-  done
-fi
-
-# if [[ $NAMES ]]; then
-#   debug names for find command $NAMES
-#   for name in $NAMES; do
-#       debug "xxonly finding '$name'"
-#      FIND+=$(set -o noglob;echo " -name '${name}'")
-#   done
-# fi
-
-# echo
-# echo find dir: $DIR >&2
-debug "find command: $FIND"
-set +o noglob
-local FILES
-FILES=$(eval $FIND | sort)
-[[ $FILES ]] && echo $FILES
-return 0
-}
-
-source_dir () {
-  # BASH_DEBUG=source_dir
-
-  debug source_dir passed arguments: $*
-  debug function: source_dir
-  local FILES
-  FILES=$(_find -l "$@") # find function
-  # echo $FILES >&2
-  [[ $? -ne 0 ]] && return 1
-    for f in $FILES; do
-    debug sourcing: $f >&2
-    source "$f"
-  done
-
-}
-
-
 prepend_file () {
-# ---------------------------------------------------------------------------------------------------------------------- 
+  local tmp_fn
-# usage prepend_file <somefile> <anotherfile>
+  # ---------------------------------------------------------------------------------------------------------------------- 
-# ---------------------------------------------------------------------------------------------------------------------- 
+  # usage prepend_file <somefile> <anotherfile>
-# Prepend the contents of <somefile> [$1], to <anotherfile> [$2], leaving the result in <anotherfile> [$2].
+  # ---------------------------------------------------------------------------------------------------------------------- 
-# insert a newline at the end of <somefile> [$1] if necessary
+  # Prepend the contents of <somefile> [$1], to <anotherfile> [$2], leaving the result in <anotherfile> [$2].
-# ---------------------------------------------------------------------------------------------------------------------- 
+  # insert a newline at the end of <somefile> [$1] if necessary
+  # ---------------------------------------------------------------------------------------------------------------------- 
+  debug prepend $1 to $2
+  [[ -f $1 ]] || return 1
+  [[ -f $2 ]] || return 2 
 
-# check # echo $1 $2
+  #todo remove comments with flag
-[[ -f $1 ]] || return 1
+
-[[ -f $2 ]] || return 2 
+  tmp_fn=$( mktemp -t TEMP_FILE_prepend.XXXXXXXX )
-# init
+  chmod 600 "$tmp_fn"
-tmp_fn=$( mktemp -t TEMP_FILE_prepend.XXXXXXXX )
+  awk '{print}' $1 $2 > $tmp_fn
-chmod 600 "$tmp_fn"
+  \mv "$tmp_fn" "$2"
-\cp $1 $tmp_fn
+  # remove temporary file
-sed -i '$a\' $tmp_fn
+  rm -f "$tmp_fn"
-cat $2 >> $tmp_fn
+  return 0
-\mv "$tmp_fn" "$2"
-# cleanup
-rm -f "$tmp_fn"
-return 0
 
-# [End]
 }
 
+# change-ext() {
+# local depth
+# local d
+# d=${3:-0}
+# let d+=1
+# [ $d -lt 1 ] && depth="" || depth=" -maxdepth $d "
+# find . $depth -name '*.'$1 -exec rename -n 's/\.'$1'/.'$2'/' \{} \;
+# read -p "do REALLY want to change the extensions? " -n 1 -r
+# echo
+# [[ ! $REPLY =~ ^[Yy]$ ]] && return 1
+# find . $depth -name '*.'$1 -exec rename 's/\.'$1'/.'$2'/' \{} \;
+# }
 
-# alias to add for live user
+# alias chext=change-ext
-if [ -v PS1 ]; then
- alias dfind="_find -t d -d 0 -n "
- alias sdfind="_find -s -t d -d 0 -n "
-fi
-
-change-ext() {
-local depth
-local d
-d=${3:-0}
-let d+=1
-[ $d -lt 1 ] && depth="" || depth=" -maxdepth $d "
-find . $depth -name '*.'$1 -exec rename -n 's/\.'$1'/.'$2'/' \{} \;
-read -p "do REALLY want to change the extensions? " -n 1 -r
-echo
-[[ ! $REPLY =~ ^[Yy]$ ]] && return 1
-find . $depth -name '*.'$1 -exec rename 's/\.'$1'/.'$2'/' \{} \;
-}
-
-alias chext=change-ext
 

modules/utility/find.lib

@@ -0,0 +1,196 @@
+#!/bin/bash
+
+# export BASH_DEBUG=true
+module_load debug
+module_load iecho
+module_load file
+
+# find, superceeds find use `command find` to get the super
+function _find () {
+
+# BASH_DEBUG=find
+
+# USAGE
+# all option arguments that contain globs/wildcards must be quoted to avoid expansion
+# f sets path and file excludes from a supplied file path
+# all lines ending in /  will be treated as directory names to ignore, otherwise files
+# p option explictly excludes paths(directories)
+# d option sets the directory depth which is current directy by default, 0 is all
+# n inclucdes only file globs otherwise it's all except .files
+# if no directory is given it will attempt to source the present working directory
+# example:
+# source_dir -p "archive" -e '"*.off" "*.md"'  -d 0 # $DIR/$SUBDIR
+local EXCLUDE_FILE
+local PATHS
+local NAMES
+local ENAMES
+local DEPTH=1
+local HIDDEN
+local raw
+local mounted
+local usesudo
+
+mounted="-mount"
+
+declare OPTION
+declare OPTARG
+declare OPTIND
+
+while getopts 'mrst:p:d:e:n:f:hl' OPTION; do
+case "$OPTION" in
+  s)
+    usesudo=sudo
+    ;;
+  r)
+    raw=true
+    ;;
+  t)
+    TYPE=$OPTARG
+    # echo "TYPE $TYPE"
+    ;;
+  f)
+    EXCLUDE_FILE=$OPTARG
+    # echo EXCLUDE FILE $EXCLUDE_FILE >&2
+    ;;
+  p)
+    # PATHS=("$OPTARG")
+    IFS=',' read -r -a PATHS <<< "$OPTARG"
+    # echo EXCLUDING THESE PATHS ${PATHS[*]}
+    ;;
+  e)
+    IFS=',' read -r -a ENAMES <<< "${OPTARG}"
+    # echo EXCLUDING THESE FILE NAMES ${ENAMES[*]}
+    ;;
+  n)
+    # NAMES=("$OPTARG")
+    IFS=',' read -r -a NAMES <<< "${OPTARG}"
+    # NAMES=$OPTARG
+    ;;
+  d)
+    DEPTH=$OPTARG
+    # echo "SOURCING TO DEPTH (0=any)" "$DEPTH"
+    ;;
+  h)
+    HIDDEN=true
+    # echo "SOURCING TO DEPTH (0=any)" "$DEPTH"
+    ;;
+
+  l)
+    LINKS=-L
+    ;;
+  m)
+   # include mounted directories
+    mounted=""
+    ;;   
+  *)
+    echo unknown option $OPTION
+    ;;
+esac
+done
+
+shift $(( OPTIND - 1 ))
+
+if [[ $raw ]]; then
+  echo executing raw find command
+  $usesudo $(which find) "$@"
+fi
+
+local DIR
+DIR="$*"
+if [ ! "$DIR" ]; then
+  if [ -v PS1 ]; then
+  echo no directory provided to search
+  echo searching present working directory $(pwd)
+  read -p "Do you want to continue? " -n 1 -r
+  [[ $REPLY =~ ^[Yy]$ ]] && DIR=$(pwd) || return 1
+  else
+    return 1
+  fi
+fi
+
+[ ! -d "$DIR" ] && echo " directory $DIR does not exist, aborting" && return 1
+
+# echo dir $DIR
+
+local FIND
+FIND="$usesudo $(which find) $LINKS $DIR"
+FIND+=$([ ! $DEPTH == 0 ] && echo " -maxdepth $DEPTH ")
+# FIND+=" -type $([ $TYPE ] && echo "$TYPE" || echo "f")"
+TYPE=${TYPE:-f}
+debug exclude mounts: $mounted
+FIND+=" $mounted -type $TYPE "
+# include HIDDEN files and directories IS FALSE BY DEFULT
+[[ ! $HIDDEN ]] && FIND+="! -path \"*/.*/*\" ! -name \".*\" "
+
+local name
+local path
+
+if [[ -f $EXCLUDE_FILE ]]; then
+  local ignores=$(lines_2_str "$EXCLUDE_FILE")
+  # echo ignores: $ignores >&2
+  for exclude in $ignores ; do
+    # echo exclude: ${exclude} >&2
+    [[ "$exclude" == */ ]] && PATHS+=("${exclude::-1}") || ENAMES+=("$exclude")
+  done
+fi
+
+# echo paths ${PATHS[@]} >&2
+# echo exclude names ${ENAMES[@]} >&2
+
+set -o noglob
+
+if [[ ${PATHS[0]} ]]; then
+  for path in ${PATHS[@]}; do
+     # echo excluding $path
+     FIND+=$(echo ' ! -path "*/'$path'/*"')
+  done
+fi
+
+if [[ ${ENAMES[0]} ]]; then
+  for name in ${ENAMES[@]}; do
+     debug excluding name "$name"
+     FIND+=$(echo ' ! -name '$name'')
+  done
+fi
+
+debug "INCLUDING ONLY THESE FILE NAMES ${NAMES[*]}"
+if [[ ${NAMES[0]} ]]; then
+  for name in "${NAMES[@]}"; do
+      debug only finding $name
+     FIND+=$(echo " -name '${name}'")
+  done
+fi
+
+# if [[ $NAMES ]]; then
+#   debug names for find command $NAMES
+#   for name in $NAMES; do
+#       debug "xxonly finding '$name'"
+#      FIND+=$(set -o noglob;echo " -name '${name}'")
+#   done
+# fi
+
+# echo
+# echo find dir: $DIR >&2
+debug "find command: $FIND"
+set +o noglob
+local FILES
+FILES=$(eval $FIND | sort)
+[[ $FILES ]] && echo $FILES
+return 0
+}
+
+# change-ext() {
+# local depth
+# local d
+# d=${3:-0}
+# let d+=1
+# [ $d -lt 1 ] && depth="" || depth=" -maxdepth $d "
+# find . $depth -name '*.'$1 -exec rename -n 's/\.'$1'/.'$2'/' \{} \;
+# read -p "do REALLY want to change the extensions? " -n 1 -r
+# echo
+# [[ ! $REPLY =~ ^[Yy]$ ]] && return 1
+# find . $depth -name '*.'$1 -exec rename 's/\.'$1'/.'$2'/' \{} \;
+# }
+
+# alias chext=change-ext
+