shell-base/modules/utility/copy.lib

#!/bin/bash

export COPY_MODULE_PATH="$(dirname $(realpath "${BASH_SOURCE:-$0}"))"
module_load debug
module_load confirm
module_load helpers

copy () {

  if ! which rsync &> /dev/null; then echo rsync must be installed to use this function; return 6; fi    

  local SHOST; local DHOST; local sshargs; local args
  local SRC; local DEST; local mirror; local quiet; local Mirror 
  local SPATH; local DPATH; local exec; local list; local verbose
  local usesudo;local cmd;local noconfirm;local insert; local sargs

  args=()

  help() {

  cat <<EOF  

usage:
rsync <script options> source destination <rsync options> -- <ssh script options> --- <ssh options>

Available Script Options
-r, run the command by this script uses --dry-run.  When ready pass this flag.  
-s, run command as sudo
-l, list the rsync command that will be run instead of running it.  removes --dry-run. 
    great for generating a rsync command to be used elsewhere 
-p, make path at destination if need be.  careful  will creating missing subdirectories    
-e, rsync --exclude , as many of these as desired
-i, insert source inside destination instead of making folder at destination, good for changing destination folder name
-m, copy as "mirror"  uses options.   -a --numeric-ids --delete --force
-M, copy as "mirror" above -m plus '--delete-excluded' added.  A perfect mirror
-f, rsync --exclude-from <path>, can have multiple 
-F, copy a filesystem, uses -M and -m mirror plus  -xHAXS, and uses sudo (i.e. -s script option)
-n, no confirm, will always confirm command unless this is set
-q, quiet, suppress progress
-v, verbose, same as -v for rsync option, unset if -q is set
-a, archive, same as -a for rsync option, basic identical copy with recursion
-c, clean tranfer meaning will not transfer folder with trash, cache, log, nodemodules
-C, show excludes for clean transfer
-h, this help text  

ssh arguments may be added at the end after --.  any option with multiple arguments
   must be enclosed in literal double quotes, (.e.g) -o \"ProxyCommand nohup ssh firewall nc -w1 %h %p\" 

source or destination may include a <host>: prefix but not both

use 'fcopy <src> <dest>' for fast copy  uses  -r -q -m -n -a
EOF

  }
  
  # parse sshcp options
  local OPTION
  local OPTARG
  local OPTIND
  while getopts 'avMqpf:le:rcCmd:shnFi' OPTION; do
    # echo OPTION $OPTION ARG $OPTARG INDX $OPTIND
    case "$OPTION" in
    # TODO add D and S for destion and source path
      s)
        usesudo="sudo -E"
        ;;
      c) 
      # clean out unneeded folders like cache, trash, log
           args+=("--exclude-from=$COPY_MODULE_PATH/cache-trash-log.exc")
        ;;
      C) 
        echo "## clean transfer exclusions found in $COPY_MODULE_PATH/cache-trash-log.exc ##"
        cat "$COPY_MODULE_PATH/cache-trash-log.exc"
        echo -e "\n##############################################"
        return 0
        ;;
      q)
       quiet=true 
      ;; 
      n)
        noconfirm=true
        ;;
      e)
        args+=("--exclude=$OPTARG")
        ;;
      f)
        args+=("--exclude-from=$OPTARG")
        ;;  
      F)
        args+=("-xHAXS")
        args+=("--exclude-from=$COPY_MODULE_PATH/cache-trash-log.exc")
        Mirror=true
        mirror=true
        usesudo="sudo -E"
        ;;            
      p)
        args+=("--mkpath")
        ;;   
      i)
        insert=true
        ;;          
      m)
        mirror=true
        ;;
      M)
        Mirror=true
        mirror=true
        ;;
      l)
        list=true
        ;;
      v)
        verbose="-v"
        ;;  
      r)
        exec=true
        ;;        
      a)
        args+=("-a")
        ;;  
      h) 
        help
        return 0  
      ;;  
      *)
        >&2 echo fatal: unknown remote script option $OPTION, aborting 
        help
        return 1
        ;;
      esac
  done

  shift $((OPTIND - 1))

  [ $# -lt 2 ] && ( echo Error: both a source and destination need to be passed; help; ) && return 1

 # source parse
  SRC=$1; shift
  if [[ $SRC =~ ":" ]]; then
  #  echo source is remote
   SHOST=${SHOST:-$(sed 's/\(.*\):.*/\1/' <<< "$SRC")}
   SPATH=$(sed 's/.*:\(.*\)/\1/' <<< "$SRC") 
   else
  #  echo source is local
   SPATH=$SRC
   fi
  # destination parse
  DEST=$1;shift
   if [[ $DEST =~ ":" ]]; then
  #  echo destination is remote
   DHOST=${DHOST:-$(sed 's/\(.*\):.*/\1/' <<< "$DEST")} 
   DPATH=$(sed 's/.*:\(.*\)/\1/' <<< "$DEST") 
   else
  #  echo destination is local
   DPATH=$DEST
   fi
   [[ $DHOST && $SHOST ]] && echo this script can not process remote host to remote host transfer && return 5

  [[ ! $SHOST ]] && SPATH=$(realpath $SPATH) 
  [[ ! $DHOST ]] && DPATH=$(realpath $DPATH) 

  # additional argument processing
  debug $( ( IFS=$','; echo all remaining arguments: "$*" ) )
  value="++"
  sci=$(( $(printf "%s\n" "$@" | sed -n '/^'$value'$/{=;q}') - 1 ))
  value="--"
  si=$(( $(printf "%s\n" "$@" | sed -n '/^'$value'$/{=;q}') - 1 ))
  debug $sci $si

  local sca=()
  local sa=()
  if [[ $sci -ge 0 ]]; then
    args+=("${@:1:$sci}")
    if [[ $si -ge 0 ]]; then
    debug both ssh args 
      sca=("${@:$sci+2:$#-$si+1}") 
      sa=("${@:$si+1:$#}") 
    else 
      debug only script args
      sca=("${@:$sci+2:$#}")   
    fi  
   else 
    if [[ $si -ge 0 ]]; then
     debug only ssh run args
     args+=("${@:1:$si}")
     sa=("${@:$si+1:$#}") 
     else 
     debug in args only
     args+=("${@:1:$#+1}")
    fi
  fi

  debug args: up before ssh:  ${args[*]}

  # is source or destination is remote, create -e ssh sync option
  if [[ $DHOST || $SHOST ]]; then
    debug remote copy, loading ssh
    debug ssh script args: ${sca[*]}
    debug ssh run args: ${sa[*]}
    debug remote host: ${DHOST:-$SHOST} 

    if ! module_load ssh &>/dev/null; then 
      echo unable to load ssh module
      echo likely the uci network repo has not been installed
      echo to do a remote copy this repo which contains the ssh module
      echo "must be installed.  Try \"module_load uci-shell-install; install_shell_network\""
      return 6
    fi  
    if [[ ! $(get_user_host ${DHOST:-$SHOST}) ]]; then
      >&2 echo fatal: the host ${DHOST:-$SHOST} is not valid, aborting remote copy
      return 1
    fi 

    local sshcmd
    # sshcmd="ssh -l "${ssshargs[*]}"  "${sshargs[*]}""
    # echo $sshcmd
        # if ! sshcmd=$(ssh -l "${ssshargs[*]}" ${DHOST:-$SHOST}); then
    # double quotes on array expansion not good
    if ! sshcmd=$(ssh -l ${sca[*]} ${DHOST:-$SHOST} ${sa[*]}); then
     echo failed to build ssh command for rsync, aborting
     debug sshcmd: ${sshcmd}
     return 5
    fi 
    debug sshcmd: ${sshcmd}
    args+=("-e '$sshcmd'") 
    debug args: inclcuding ssh:  ${args[*]}
  fi  

  # assemble remaining arguments from options
  [[ ! $exec ]] && args+=(--dry-run -v)
  [[ ! $quiet ]] && args+=(--info=progress2 $verbose)
  [[ $mirror ]] && args+=(-a --numeric-ids --delete --force)
  [[ $Mirror ]] && args+=(--delete-excluded)

  [[ ! -v PS1 ]] && noconfirm=true

  cmd="$usesudo $(which rsync) ${args[*]} $SRC$([[ $insert ]] && echo "/") $DEST" 
  [[ $list ]] && echo "$cmd" 

  if [[ ! $noconfirm ]]; then
    if [[ $insert ]]; then
     # todo mirror option
     confirm The contents within $([[ $SHOST ]] && echo $SHOST:)$SPATH will be \
     $([[ $mirror ]] && echo mirrored || echo placed ) inside $([[ $DHOST ]] && echo $DHOST:)$DPATH || return 0
     else
     confirm the directory $(basename $SPATH) of $([[ $SHOST ]] && echo $SHOST:)$SPATH will be \
     $([[ $mirror ]] && echo mirrored || echo put ) at destination $([[ $DHOST ]] && echo $DHOST:)$DPATH/$(basename $SPATH) || return 0
    fi
  fi  

  if [[ $list && ! $noconfirm ]]; then
    confirm list command during dry run, do you want to continue with dry run || return 0
  fi
     
  if [[ ! $quiet ]]; then  [[ $exec ]] && echo executing the copy command || echo dry run of command, use -r to execute; fi
  if eval $cmd; then
      [[ ! $quiet ]] && echo copy success! 
      debug $cmd
    else 
      >&2 echo remote copy failed
      >&2 echo $cmd
      return 1
    fi
  }

  alias fcpy="copy -r -q -i -n -a "

  # alias mirror="copy -i -a -M" 


  # custom copy commands as functions

  mirror_shell () {

    local dir; local dest
    #TODO don't overwrite hostname

    [ $# -lt 2 ] && echo "both a directory and remote host need to be passed" && return 2    
    dir=$1
    [[ ! -d $dir ]] && echo no directory $dir to mirror && return 1
    dest=$2
    shift 2
    copy $@ -i -a -c -M "$dir" "$dest:$dir" --exclude=archive*

  }

  rpush () {

    local dir; local dest
    #TODO don't overwrite hostname

    [ $# -lt 2 ] && echo "both a directory and remote host need to be passed" && return 2    
    dir=$1
    [[ ! -d $dir ]] && echo no directory $dir to mirror && return 1
    dest=$2
    shift 2
    copy "$@" -i -a "$dir" "$dest:$dir"

  }

  rpull () {

    local dir; local src; local dest
    #TODO don't overwrite hostname

    [ $# -lt 2 ] && echo "remote pull, both a directory and remote host need to be passed" && return 2    
    dir=$1
    src=$2
    shift 2
    if [[ ! $1 == "-"* ]]; then
     dest=$1
     shift
     echo pulling to alternate directory $dest
    fi 
    copy "$@" -l -i "$src:$dir" "${dest:-dir}"
    copy "$@" -i "$src:$dir" "${dest:-dir}"
  }

  rpushmirror () {
    rpush "$@" -M -c
  }

  rpullmirror () {
    rpull "$@" -M -c
  }
  
  rbrowser_copy () {

    local browser; local dest

    [ $# -lt 2 ] && echo "both a browser directory (within /opt/chromium) and remote host need to be passed" && return 2    
    browser=$1
    [[ ! -d /opt/chromium/$browser ]] && echo no browser directory /opt/chromium/$browser && return 1
    dest=$2
    shift 2
    copy $@ -a -c -M "/opt/chromium/$browser" "$dest:/opt/chromium" --exclude=Singleton*

  }