From 398ae39a075e60c576942f03674f8a5376f04ff6 Mon Sep 17 00:00:00 2001 From: David Kebler Date: Fri, 16 Feb 2024 23:49:08 -0800 Subject: [PATCH] fixing bundler move some module code to new modules --- load/01-basic | 3 + load/alias/00-basic | 7 + modules/install-shell-repos.lib | 4 +- modules/scripting/bundle.func | 100 ++++++++--- modules/scripting/helpers.lib | 13 +- modules/scripting/source.lib | 17 ++ modules/shell-process-directory.mod | 2 +- modules/utility/dir.lib | 3 +- modules/utility/file.lib | 265 ++++------------------------ modules/utility/find.lib | 196 ++++++++++++++++++++ 10 files changed, 344 insertions(+), 266 deletions(-) create mode 100644 modules/scripting/source.lib create mode 100644 modules/utility/find.lib diff --git a/load/01-basic b/load/01-basic index 4bf43fa..231b727 100644 --- a/load/01-basic +++ b/load/01-basic @@ -50,3 +50,6 @@ get_shopt () { # echo set the system editor using EDITOR environment variable edit() { ${EDITOR:-nano} $@; } gedit() { ${GEDITOR:-xed} $@; } + + +rsynchg() { rsync --help | grep "\-$1"; } diff --git a/load/alias/00-basic b/load/alias/00-basic index b0e39f5..feba75a 100644 --- a/load/alias/00-basic +++ b/load/alias/00-basic @@ -64,3 +64,10 @@ alias follow="readlink -f" alias umountf="sudo umount -l" alias loginust="loginctl show-user $USER" + + +# alias to add for live user +if [ -v PS1 ]; then + alias dfind="_find -t d -d 0 -n " + alias sdfind="_find -s -t d -d 0 -n " +fi \ No newline at end of file diff --git a/modules/install-shell-repos.lib b/modules/install-shell-repos.lib index d348570..fe285c3 100644 --- a/modules/install-shell-repos.lib +++ b/modules/install-shell-repos.lib @@ -11,8 +11,8 @@ install_shell_repo () { [[ ! $EUID -eq 0 ]] && { echo ERROR script must be run as root; return; } local user [[ $1 == "-u" ]] && user=$2 && shift 2 - local dir=${2:-/shell/any} - local repo=${1:-host} + local dir=${2:-/shell} + local repo=${1:-base} local rpath=$dir/$repo mkdir -p $rpath # TODO set git host via option or env var diff --git a/modules/scripting/bundle.func b/modules/scripting/bundle.func index a8166f3..763d884 100644 --- a/modules/scripting/bundle.func +++ b/modules/scripting/bundle.func @@ -7,49 +7,95 @@ # or # scpt=$(bundle myscript.sh) # source <(echo -e "$scpt") +module_load debug +module_load file bundle () { local file - - if [[ $1 == "-m" ]]; then + local code + local compact + if [[ $1 == "-c" ]]; then + module_load helpers + compact=true shift 1 - file=$(module_find $1) - else - file=$1 - fi + fi + + # todo add -c for compacting the bundle, change this to options parsing. + + if [[ "$1" ]]; then + if [[ -f $1 ]]; then + file=$1 + else + if [[ $1 == "-m" ]]; then + file=$(module_find $2) + shift 1 + else + if [[ $1 == "-mf" ]]; then + module_load $2 + code="$(declare -f $3)" + shift 1 + else + code="$1" + shift 1 + fi + fi + fi + else + echo "aborting, no file, module or function passed for bundling" + return 1 + fi - [[ ! -f $file ]] && return 1 - module_load file - - if [[ ! $2 == "__recurse__" ]]; then tmp_file=$( mktemp -t TEMP_FILE_bundle.XXXXXXXX ) chmod 600 "$tmp_file" - \cp $file $tmp_file + # if [[ $2 == "__recurse__" ]] || [[ $file ]]; then + if [[ $file ]]; then + \cp $file $tmp_file else - tmp_file=$file + echo "$code" > $tmp_file + fi + + + if [[ ! $2 == "__recurse__" ]]; then + _modules="" + output=$2 + debug outputing bundle to: $output + debug "bash code to bundle \n $(cat $tmp_file)" fi - # echo current temp file: $tmp_file - modules=$(sed -n -e 's/^module_load //p' < $tmp_file) - # echo found: $modules - # return + + modules=$(sed -n -e 's/^\s*module_load //p' < $tmp_file) + if [[ $modules ]]; then - # echo Modules: $modules - sed -i '/module_load/d' $tmp_file + # remove found module_load before continuing + sed -i '/^\s*module_load/d' $tmp_file + [[ $_modules ]] && debug "already bundled modules: $_modules" for module in $(printf '%s\n' "${modules[@]}"|tac); - do - # echo module: $module - # echo module path: $(module_find $module) - prepend_file $(module_find $module) $tmp_file - done + do + if [[ $_modules == " $module "* ]]; then + [[ $_modules ]] && debug module $module already bundled + else + if modp=$(module_find ${module}); then + debug bundling module ${module} at: $modp + prepend_file $modp $tmp_file + fi + fi + done + _modules="$_modules $modules" bundle $tmp_file __recurse__ fi + # the recursion is done if [[ ! $2 == "__recurse__" ]]; then - if [[ $2 ]]; then - \cp $tmp_file $2 - # echo $2 - else + [[ $compact ]] && compact_file $tmp_file + if [[ $BASH_DEBUG ]]; then + debug "\n ------------code as bundled--------------\n \ + $(cat $tmp_file) \ + \n ------------------------------------------------------------ \n" + debug "bundled modules were $_modules" + fi + if [[ $2 ]]; then + \cp $tmp_file $output + else echo -e "$(cat $tmp_file)" fi rm -f $tmp_file; diff --git a/modules/scripting/helpers.lib b/modules/scripting/helpers.lib index a45a213..3e9bba8 100644 --- a/modules/scripting/helpers.lib +++ b/modules/scripting/helpers.lib @@ -70,7 +70,8 @@ parse_option () { } remove_empty_lines () { -if [[ -f $1 ]]; then cat $1; else echo "$1"; fi | sed -rz 's/^\n+//; s/\n+$/\n/g' +if [[ -f $1 ]]; then cat $1; else echo "$1"; fi | sed -n '/^\s*$/!p' +# sed -rz 's/^\n+//; s/\n+$/\n/g' } remove_end_spaces () { @@ -93,6 +94,16 @@ sed_ignore_comments () { fi } +compact_file () { + # removes trailing spaces, blank lines and comments +# echo -e "$1" +# grep -v '^\s*#' $1 +[[ -f $1 ]] && sed -i -e '/^\s*#/d' -e '/^\s*$/d' "$1" +# if [[ -f "$1" ]]; then cat "$1"; else echo -e "$@"; fi | sed 's/\\s*$//g' | sed -r '/^\s*$/d' | grep -v '^\s*#' +} + + + #rows between matched rows # sed '1,/firstmatch/d;/secondmatch/,$d' diff --git a/modules/scripting/source.lib b/modules/scripting/source.lib new file mode 100644 index 0000000..db14299 --- /dev/null +++ b/modules/scripting/source.lib @@ -0,0 +1,17 @@ +#!/bin/bash + +source_dir () { + # BASH_DEBUG=source_dir + module_load find + debug source_dir passed arguments: $* + debug function: source_dir + local FILES + FILES=$(_find -l "$@") # find function + # echo $FILES >&2 + [[ $? -ne 0 ]] && return 1 + for f in $FILES; do + debug sourcing: $f >&2 + source "$f" + done + +} \ No newline at end of file diff --git a/modules/shell-process-directory.mod b/modules/shell-process-directory.mod index d83d115..8a2a23e 100644 --- a/modules/shell-process-directory.mod +++ b/modules/shell-process-directory.mod @@ -24,7 +24,7 @@ function shell_process_directory () { local SUBDIRS local DIR - module_load file + module_load source [[ $? -ne 0 ]] && echo unable to access the file module, aborting load && return 1 DIR=${1:-$BASH_SHELL_BASE} diff --git a/modules/utility/dir.lib b/modules/utility/dir.lib index d321976..9ed5639 100644 --- a/modules/utility/dir.lib +++ b/modules/utility/dir.lib @@ -12,8 +12,7 @@ isDir() { fi } -rsynchc() { rsync --help | grep "\-$1"; } -alias rsynch="rsynchc" + function dir_copy () { diff --git a/modules/utility/file.lib b/modules/utility/file.lib index 38ffa29..0f5a2d0 100644 --- a/modules/utility/file.lib +++ b/modules/utility/file.lib @@ -53,243 +53,42 @@ function lines_2_str () { echo $str } - - -# find, superceeds find use `command find` to get the super -function _find () { - -# BASH_DEBUG=find - -# USAGE -# all option arguments that contain globs/wildcards must be quoted to avoid expansion -# f sets path and file excludes from a supplied file path -# all lines ending in / will be treated as directory names to ignore, otherwise files -# p option explictly excludes paths(directories) -# d option sets the directory depth which is current directy by default, 0 is all -# n inclucdes only file globs otherwise it's all except .files -# if no directory is given it will attempt to source the present working directory -# example: -# source_dir -p "archive" -e '"*.off" "*.md"' -d 0 # $DIR/$SUBDIR -local EXCLUDE_FILE -local PATHS -local NAMES -local ENAMES -local DEPTH=1 -local HIDDEN -local raw -local mounted -local usesudo - -mounted="-mount" - -declare OPTION -declare OPTARG -declare OPTIND - -while getopts 'mrst:p:d:e:n:f:hl' OPTION; do -case "$OPTION" in - s) - usesudo=sudo - ;; - r) - raw=true - ;; - t) - TYPE=$OPTARG - # echo "TYPE $TYPE" - ;; - f) - EXCLUDE_FILE=$OPTARG - # echo EXCLUDE FILE $EXCLUDE_FILE >&2 - ;; - p) - # PATHS=("$OPTARG") - IFS=',' read -r -a PATHS <<< "$OPTARG" - # echo EXCLUDING THESE PATHS ${PATHS[*]} - ;; - e) - IFS=',' read -r -a ENAMES <<< "${OPTARG}" - # echo EXCLUDING THESE FILE NAMES ${ENAMES[*]} - ;; - n) - # NAMES=("$OPTARG") - IFS=',' read -r -a NAMES <<< "${OPTARG}" - # NAMES=$OPTARG - ;; - d) - DEPTH=$OPTARG - # echo "SOURCING TO DEPTH (0=any)" "$DEPTH" - ;; - h) - HIDDEN=true - # echo "SOURCING TO DEPTH (0=any)" "$DEPTH" - ;; - - l) - LINKS=-L - ;; - m) - # include mounted directories - mounted="" - ;; - *) - echo unknown option $OPTION - ;; -esac -done - -shift $(( OPTIND - 1 )) - -if [[ $raw ]]; then - echo executing raw find command - $usesudo $(which find) "$@" -fi - -local DIR -DIR="$*" -if [ ! "$DIR" ]; then - if [ -v PS1 ]; then - echo no directory provided to search - echo searching present working directory $(pwd) - read -p "Do you want to continue? " -n 1 -r - [[ $REPLY =~ ^[Yy]$ ]] && DIR=$(pwd) || return 1 - else - return 1 - fi -fi - -[ ! -d "$DIR" ] && echo " directory $DIR does not exist, aborting" && return 1 - -# echo dir $DIR - -local FIND -FIND="$usesudo $(which find) $LINKS $DIR" -FIND+=$([ ! $DEPTH == 0 ] && echo " -maxdepth $DEPTH ") -# FIND+=" -type $([ $TYPE ] && echo "$TYPE" || echo "f")" -TYPE=${TYPE:-f} -debug exclude mounts: $mounted -FIND+=" $mounted -type $TYPE " -# include HIDDEN files and directories IS FALSE BY DEFULT -[[ ! $HIDDEN ]] && FIND+="! -path \"*/.*/*\" ! -name \".*\" " - -local name -local path - -if [[ -f $EXCLUDE_FILE ]]; then - local ignores=$(lines_2_str "$EXCLUDE_FILE") - # echo ignores: $ignores >&2 - for exclude in $ignores ; do - # echo exclude: ${exclude} >&2 - [[ "$exclude" == */ ]] && PATHS+=("${exclude::-1}") || ENAMES+=("$exclude") - done -fi - -# echo paths ${PATHS[@]} >&2 -# echo exclude names ${ENAMES[@]} >&2 - -set -o noglob - -if [[ ${PATHS[0]} ]]; then - for path in ${PATHS[@]}; do - # echo excluding $path - FIND+=$(echo ' ! -path "*/'$path'/*"') - done -fi - -if [[ ${ENAMES[0]} ]]; then - for name in ${ENAMES[@]}; do - debug excluding name "$name" - FIND+=$(echo ' ! -name '$name'') - done -fi - -debug "INCLUDING ONLY THESE FILE NAMES ${NAMES[*]}" -if [[ ${NAMES[0]} ]]; then - for name in "${NAMES[@]}"; do - debug only finding $name - FIND+=$(echo " -name '${name}'") - done -fi - -# if [[ $NAMES ]]; then -# debug names for find command $NAMES -# for name in $NAMES; do -# debug "xxonly finding '$name'" -# FIND+=$(set -o noglob;echo " -name '${name}'") -# done -# fi - -# echo -# echo find dir: $DIR >&2 -debug "find command: $FIND" -set +o noglob -local FILES -FILES=$(eval $FIND | sort) -[[ $FILES ]] && echo $FILES -return 0 -} - -source_dir () { - # BASH_DEBUG=source_dir - - debug source_dir passed arguments: $* - debug function: source_dir - local FILES - FILES=$(_find -l "$@") # find function - # echo $FILES >&2 - [[ $? -ne 0 ]] && return 1 - for f in $FILES; do - debug sourcing: $f >&2 - source "$f" - done - -} - - prepend_file () { -# ---------------------------------------------------------------------------------------------------------------------- -# usage prepend_file -# ---------------------------------------------------------------------------------------------------------------------- -# Prepend the contents of [$1], to [$2], leaving the result in [$2]. -# insert a newline at the end of [$1] if necessary -# ---------------------------------------------------------------------------------------------------------------------- + local tmp_fn + # ---------------------------------------------------------------------------------------------------------------------- + # usage prepend_file + # ---------------------------------------------------------------------------------------------------------------------- + # Prepend the contents of [$1], to [$2], leaving the result in [$2]. + # insert a newline at the end of [$1] if necessary + # ---------------------------------------------------------------------------------------------------------------------- + debug prepend $1 to $2 + [[ -f $1 ]] || return 1 + [[ -f $2 ]] || return 2 -# check # echo $1 $2 -[[ -f $1 ]] || return 1 -[[ -f $2 ]] || return 2 -# init -tmp_fn=$( mktemp -t TEMP_FILE_prepend.XXXXXXXX ) -chmod 600 "$tmp_fn" -\cp $1 $tmp_fn -sed -i '$a\' $tmp_fn -cat $2 >> $tmp_fn -\mv "$tmp_fn" "$2" -# cleanup -rm -f "$tmp_fn" -return 0 + #todo remove comments with flag + + tmp_fn=$( mktemp -t TEMP_FILE_prepend.XXXXXXXX ) + chmod 600 "$tmp_fn" + awk '{print}' $1 $2 > $tmp_fn + \mv "$tmp_fn" "$2" + # remove temporary file + rm -f "$tmp_fn" + return 0 -# [End] } +# change-ext() { +# local depth +# local d +# d=${3:-0} +# let d+=1 +# [ $d -lt 1 ] && depth="" || depth=" -maxdepth $d " +# find . $depth -name '*.'$1 -exec rename -n 's/\.'$1'/.'$2'/' \{} \; +# read -p "do REALLY want to change the extensions? " -n 1 -r +# echo +# [[ ! $REPLY =~ ^[Yy]$ ]] && return 1 +# find . $depth -name '*.'$1 -exec rename 's/\.'$1'/.'$2'/' \{} \; +# } -# alias to add for live user -if [ -v PS1 ]; then - alias dfind="_find -t d -d 0 -n " - alias sdfind="_find -s -t d -d 0 -n " -fi - -change-ext() { -local depth -local d -d=${3:-0} -let d+=1 -[ $d -lt 1 ] && depth="" || depth=" -maxdepth $d " -find . $depth -name '*.'$1 -exec rename -n 's/\.'$1'/.'$2'/' \{} \; -read -p "do REALLY want to change the extensions? " -n 1 -r -echo -[[ ! $REPLY =~ ^[Yy]$ ]] && return 1 -find . $depth -name '*.'$1 -exec rename 's/\.'$1'/.'$2'/' \{} \; -} - -alias chext=change-ext +# alias chext=change-ext diff --git a/modules/utility/find.lib b/modules/utility/find.lib new file mode 100644 index 0000000..1aa441b --- /dev/null +++ b/modules/utility/find.lib @@ -0,0 +1,196 @@ +#!/bin/bash + +# export BASH_DEBUG=true +module_load debug +module_load iecho +module_load file + +# find, superceeds find use `command find` to get the super +function _find () { + +# BASH_DEBUG=find + +# USAGE +# all option arguments that contain globs/wildcards must be quoted to avoid expansion +# f sets path and file excludes from a supplied file path +# all lines ending in / will be treated as directory names to ignore, otherwise files +# p option explictly excludes paths(directories) +# d option sets the directory depth which is current directy by default, 0 is all +# n inclucdes only file globs otherwise it's all except .files +# if no directory is given it will attempt to source the present working directory +# example: +# source_dir -p "archive" -e '"*.off" "*.md"' -d 0 # $DIR/$SUBDIR +local EXCLUDE_FILE +local PATHS +local NAMES +local ENAMES +local DEPTH=1 +local HIDDEN +local raw +local mounted +local usesudo + +mounted="-mount" + +declare OPTION +declare OPTARG +declare OPTIND + +while getopts 'mrst:p:d:e:n:f:hl' OPTION; do +case "$OPTION" in + s) + usesudo=sudo + ;; + r) + raw=true + ;; + t) + TYPE=$OPTARG + # echo "TYPE $TYPE" + ;; + f) + EXCLUDE_FILE=$OPTARG + # echo EXCLUDE FILE $EXCLUDE_FILE >&2 + ;; + p) + # PATHS=("$OPTARG") + IFS=',' read -r -a PATHS <<< "$OPTARG" + # echo EXCLUDING THESE PATHS ${PATHS[*]} + ;; + e) + IFS=',' read -r -a ENAMES <<< "${OPTARG}" + # echo EXCLUDING THESE FILE NAMES ${ENAMES[*]} + ;; + n) + # NAMES=("$OPTARG") + IFS=',' read -r -a NAMES <<< "${OPTARG}" + # NAMES=$OPTARG + ;; + d) + DEPTH=$OPTARG + # echo "SOURCING TO DEPTH (0=any)" "$DEPTH" + ;; + h) + HIDDEN=true + # echo "SOURCING TO DEPTH (0=any)" "$DEPTH" + ;; + + l) + LINKS=-L + ;; + m) + # include mounted directories + mounted="" + ;; + *) + echo unknown option $OPTION + ;; +esac +done + +shift $(( OPTIND - 1 )) + +if [[ $raw ]]; then + echo executing raw find command + $usesudo $(which find) "$@" +fi + +local DIR +DIR="$*" +if [ ! "$DIR" ]; then + if [ -v PS1 ]; then + echo no directory provided to search + echo searching present working directory $(pwd) + read -p "Do you want to continue? " -n 1 -r + [[ $REPLY =~ ^[Yy]$ ]] && DIR=$(pwd) || return 1 + else + return 1 + fi +fi + +[ ! -d "$DIR" ] && echo " directory $DIR does not exist, aborting" && return 1 + +# echo dir $DIR + +local FIND +FIND="$usesudo $(which find) $LINKS $DIR" +FIND+=$([ ! $DEPTH == 0 ] && echo " -maxdepth $DEPTH ") +# FIND+=" -type $([ $TYPE ] && echo "$TYPE" || echo "f")" +TYPE=${TYPE:-f} +debug exclude mounts: $mounted +FIND+=" $mounted -type $TYPE " +# include HIDDEN files and directories IS FALSE BY DEFULT +[[ ! $HIDDEN ]] && FIND+="! -path \"*/.*/*\" ! -name \".*\" " + +local name +local path + +if [[ -f $EXCLUDE_FILE ]]; then + local ignores=$(lines_2_str "$EXCLUDE_FILE") + # echo ignores: $ignores >&2 + for exclude in $ignores ; do + # echo exclude: ${exclude} >&2 + [[ "$exclude" == */ ]] && PATHS+=("${exclude::-1}") || ENAMES+=("$exclude") + done +fi + +# echo paths ${PATHS[@]} >&2 +# echo exclude names ${ENAMES[@]} >&2 + +set -o noglob + +if [[ ${PATHS[0]} ]]; then + for path in ${PATHS[@]}; do + # echo excluding $path + FIND+=$(echo ' ! -path "*/'$path'/*"') + done +fi + +if [[ ${ENAMES[0]} ]]; then + for name in ${ENAMES[@]}; do + debug excluding name "$name" + FIND+=$(echo ' ! -name '$name'') + done +fi + +debug "INCLUDING ONLY THESE FILE NAMES ${NAMES[*]}" +if [[ ${NAMES[0]} ]]; then + for name in "${NAMES[@]}"; do + debug only finding $name + FIND+=$(echo " -name '${name}'") + done +fi + +# if [[ $NAMES ]]; then +# debug names for find command $NAMES +# for name in $NAMES; do +# debug "xxonly finding '$name'" +# FIND+=$(set -o noglob;echo " -name '${name}'") +# done +# fi + +# echo +# echo find dir: $DIR >&2 +debug "find command: $FIND" +set +o noglob +local FILES +FILES=$(eval $FIND | sort) +[[ $FILES ]] && echo $FILES +return 0 +} + +# change-ext() { +# local depth +# local d +# d=${3:-0} +# let d+=1 +# [ $d -lt 1 ] && depth="" || depth=" -maxdepth $d " +# find . $depth -name '*.'$1 -exec rename -n 's/\.'$1'/.'$2'/' \{} \; +# read -p "do REALLY want to change the extensions? " -n 1 -r +# echo +# [[ ! $REPLY =~ ^[Yy]$ ]] && return 1 +# find . $depth -name '*.'$1 -exec rename 's/\.'$1'/.'$2'/' \{} \; +# } + +# alias chext=change-ext +