#!/bin/bash export COPY_MODULE_PATH="$(dirname $(realpath "${BASH_SOURCE:-$0}"))" module_load debug module_load confirm module_load helpers copy () { if ! which rsync &> /dev/null; then echo rsync must be installed to use this function; return 6; fi local SHOST; local DHOST; local sshargs; local args local SRC; local DEST; local mirror; local quiet; local Mirror local SPATH; local DPATH; local exec; local list; local verbose local usesudo;local cmd;local noconfirm;local insert; local sargs args=() help() { cat < source destination -- --- Available Script Options -r, run the command by this script uses --dry-run. When ready pass this flag. -s, run command as sudo -l, list the rsync command that will be run instead of running it. removes --dry-run. great for generating a rsync command to be used elsewhere -p, make path at destination if need be. careful will creating missing subdirectories -e, rsync --exclude , as many of these as desired -i, insert source inside destination instead of making folder at destination, good for changing destination folder name -m, copy as "mirror" uses options. -a --numeric-ids --delete --force -M, copy as "mirror" above -m plus '--delete-excluded' added. A perfect mirror -f, rsync --exclude-from , can have multiple -F, copy a filesystem, uses -M and -m mirror plus -xHAXS, and uses sudo (i.e. -s script option) -n, no confirm, will always confirm command unless this is set -q, quiet, suppress progress -v, verbose, same as -v for rsync option, unset if -q is set -a, archive, same as -a for rsync option, basic identical copy with recursion -c, clean tranfer meaning will not transfer folder with trash, cache, log, nodemodules -C, show excludes for clean transfer -h, this help text ssh arguments may be added at the end after --. any option with multiple arguments must be enclosed in literal double quotes, (.e.g) -o \"ProxyCommand nohup ssh firewall nc -w1 %h %p\" source or destination may include a : prefix but not both use 'fcopy ' for fast copy uses -r -q -m -n -a EOF } # parse sshcp options local OPTION local OPTARG local OPTIND while getopts 'avMqpf:le:rcCmd:shnFi' OPTION; do # echo OPTION $OPTION ARG $OPTARG INDX $OPTIND case "$OPTION" in # TODO add D and S for destion and source path s) usesudo="sudo -E" ;; c) # clean out unneeded folders like cache, trash, log args+=("--exclude-from=$COPY_MODULE_PATH/cache-trash-log.exc") ;; C) echo "## clean transfer exclusions found in $COPY_MODULE_PATH/cache-trash-log.exc ##" cat "$COPY_MODULE_PATH/cache-trash-log.exc" echo -e "\n##############################################" return 0 ;; q) quiet=true ;; n) noconfirm=true ;; e) args+=("--exclude=$OPTARG") ;; f) args+=("--exclude-from=$OPTARG") ;; F) args+=("-xHAXS") args+=("--exclude-from=$COPY_MODULE_PATH/cache-trash-log.exc") Mirror=true mirror=true usesudo="sudo -E" ;; p) args+=("--mkpath") ;; i) insert=true ;; m) mirror=true ;; M) Mirror=true mirror=true ;; l) list=true ;; v) verbose="-v" ;; r) exec=true ;; a) args+=("-a") ;; h) help return 0 ;; *) >&2 echo fatal: unknown remote script option $OPTION, aborting help return 1 ;; esac done shift $((OPTIND - 1)) [ $# -lt 2 ] && ( echo Error: both a source and destination need to be passed; help; ) && return 1 # source parse SRC=$1; shift if [[ $SRC =~ ":" ]]; then # echo source is remote SHOST=${SHOST:-$(sed 's/\(.*\):.*/\1/' <<< "$SRC")} SPATH=$(sed 's/.*:\(.*\)/\1/' <<< "$SRC") else # echo source is local SPATH=$SRC fi # destination parse DEST=$1;shift if [[ $DEST =~ ":" ]]; then # echo destination is remote DHOST=${DHOST:-$(sed 's/\(.*\):.*/\1/' <<< "$DEST")} DPATH=$(sed 's/.*:\(.*\)/\1/' <<< "$DEST") else # echo destination is local DPATH=$DEST fi [[ $DHOST && $SHOST ]] && echo this script can not process remote host to remote host transfer && return 5 [[ ! $SHOST ]] && SPATH=$(realpath $SPATH) [[ ! $DHOST ]] && DPATH=$(realpath $DPATH) # additional argument processing debug $( ( IFS=$','; echo all remaining arguments: "$*" ) ) value="++" sci=$(( $(printf "%s\n" "$@" | sed -n '/^'$value'$/{=;q}') - 1 )) value="--" si=$(( $(printf "%s\n" "$@" | sed -n '/^'$value'$/{=;q}') - 1 )) debug $sci $si local sca=() local sa=() if [[ $sci -ge 0 ]]; then args+=("${@:1:$sci}") if [[ $si -ge 0 ]]; then debug both ssh args sca=("${@:$sci+2:$#-$si+1}") sa=("${@:$si+1:$#}") else debug only script args sca=("${@:$sci+2:$#}") fi else if [[ $si -ge 0 ]]; then debug only ssh run args args+=("${@:1:$si}") sa=("${@:$si+1:$#}") else debug in args only args+=("${@:1:$#+1}") fi fi debug args: up before ssh: ${args[*]} # is source or destination is remote, create -e ssh sync option if [[ $DHOST || $SHOST ]]; then debug remote copy, loading ssh debug ssh script args: ${sca[*]} debug ssh run args: ${sa[*]} debug remote host: ${DHOST:-$SHOST} if ! module_load ssh &>/dev/null; then echo unable to load ssh module echo likely the uci network repo has not been installed echo to do a remote copy this repo which contains the ssh module echo "must be installed. Try \"module_load uci-shell-install; install_shell_network\"" return 6 fi if [[ ! $(get_user_host ${DHOST:-$SHOST}) ]]; then >&2 echo fatal: the host ${DHOST:-$SHOST} is not valid, aborting remote copy return 1 fi local sshcmd # sshcmd="ssh -l "${ssshargs[*]}" "${sshargs[*]}"" # echo $sshcmd # if ! sshcmd=$(ssh -l "${ssshargs[*]}" ${DHOST:-$SHOST}); then # double quotes on array expansion not good if ! sshcmd=$(ssh -l ${sca[*]} ${DHOST:-$SHOST} ${sa[*]}); then echo failed to build ssh command for rsync, aborting debug sshcmd: ${sshcmd} return 5 fi debug sshcmd: ${sshcmd} args+=("-e '$sshcmd'") debug args: inclcuding ssh: ${args[*]} fi # assemble remaining arguments from options [[ ! $exec ]] && args+=(--dry-run -v) [[ ! $quiet ]] && args+=(--info=progress2 $verbose) [[ $mirror ]] && args+=(-a --numeric-ids --delete --force) [[ $Mirror ]] && args+=(--delete-excluded) [[ ! -v PS1 ]] && noconfirm=true cmd="$usesudo $(which rsync) ${args[*]} $SRC$([[ $insert ]] && echo "/") $DEST" [[ $list ]] && echo "$cmd" if [[ ! $noconfirm ]]; then if [[ $insert ]]; then # todo mirror option confirm The contents within $([[ $SHOST ]] && echo $SHOST:)$SPATH will be \ $([[ $mirror ]] && echo mirrored || echo placed ) inside $([[ $DHOST ]] && echo $DHOST:)$DPATH || return 0 else confirm the directory $(basename $SPATH) of $([[ $SHOST ]] && echo $SHOST:)$SPATH will be \ $([[ $mirror ]] && echo mirrored || echo put ) at destination $([[ $DHOST ]] && echo $DHOST:)$DPATH/$(basename $SPATH) || return 0 fi fi if [[ $list && ! $noconfirm ]]; then confirm list command during dry run, do you want to continue with dry run || return 0 fi if [[ ! $quiet ]]; then [[ $exec ]] && echo executing the copy command || echo dry run of command, use -r to execute; fi if eval $cmd; then [[ ! $quiet ]] && echo copy success! debug $cmd else >&2 echo remote copy failed >&2 echo $cmd return 1 fi } alias fcpy="copy -r -q -i -n -a " # alias mirror="copy -i -a -M" # custom copy commands as functions mirror_shell () { local dir; local dest #TODO don't overwrite hostname [ $# -lt 2 ] && echo "both a directory and remote host need to be passed" && return 2 dir=$1 [[ ! -d $dir ]] && echo no directory $dir to mirror && return 1 dest=$2 shift 2 copy $@ -i -a -c -M "$dir" "$dest:$dir" --exclude=archive* } rpush () { local dir; local dest #TODO don't overwrite hostname [ $# -lt 2 ] && echo "both a directory and remote host need to be passed" && return 2 dir=$1 [[ ! -d $dir ]] && echo no directory $dir to mirror && return 1 dest=$2 shift 2 copy "$@" -i -a "$dir" "$dest:$dir" } rpull () { local dir; local src; local dest #TODO don't overwrite hostname [ $# -lt 2 ] && echo "remote pull, both a directory and remote host need to be passed" && return 2 dir=$1 src=$2 shift 2 if [[ ! $1 == "-"* ]]; then dest=$1 shift echo pulling to alternate directory $dest fi copy "$@" -l -i "$src:$dir" "${dest:-dir}" copy "$@" -i "$src:$dir" "${dest:-dir}" } rpushmirror () { rpush "$@" -M -c } rpullmirror () { rpull "$@" -M -c } rbrowser_copy () { local browser; local dest [ $# -lt 2 ] && echo "both a browser directory (within /opt/chromium) and remote host need to be passed" && return 2 browser=$1 [[ ! -d /opt/chromium/$browser ]] && echo no browser directory /opt/chromium/$browser && return 1 dest=$2 shift 2 copy $@ -a -c -M "/opt/chromium/$browser" "$dest:/opt/chromium" --exclude=Singleton* }