backup script refactored to use restic

master
Kebler Network System Administrator 2021-08-29 08:01:40 -07:00
parent b5901d1d6f
commit b1fd4a6dc6
6 changed files with 787 additions and 71 deletions

View File

@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
backup () { backup () {
DIR=$(dirname $(cd "$(dirname "$BASH_SOURCE")" >/dev/null 2>&1 ; pwd -P )) DIR=$(dirname $(cd "$(dirname "$BASH_SOURCE")" >/dev/null 2>&1 ; pwd -P ))
$DIR/dev/backup/backup "$@" $DIR/dev/backup/dbackup "$@"
#module_load backup #module_load backup
#run "$@" #run "$@"
} }

603
all/dev/backup/dbackup Executable file
View File

@ -0,0 +1,603 @@
#!/usr/bin/env bash
# This script was generated by bashly (https://github.com/DannyBen/bashly)
# Modifying it manually is not recommended
# :command.root_command
root_command() {
# :src/root_command.sh
#!/bin/bash
echo "running root command"
inspect_args
# module_load ssh
module_load confirm
module_load path
local settings=${args[--settings]}
if [[ -f $settings ]]; then
echo loading settings file $settings
module_load yaml
eval $(parse_yaml $settings "s_")
echo $s_source
echo $s_target
fi
if [[ $s_server_host ]]; then
s_server="rest:http$([[ $s_server_secure ]] && echo "s")://${s_server_host}$([[ $s_server_port ]] && echo :${s_server_port})"
fi
local password=${args[--password]:-$BACKUP_PASSWORD}
password=${password:-$s_password}
[[ ! $password ]] && echo restic requires a backup repository password, exiting && return 2
password="RESTIC_PASSWORD=${password}"
local server=${args[--server]:-$BACKUP_SERVER}
server=${server:-$s_server}
local backup_dir=${args[--backup_dir]:-$BACKUP_DIR}
backup_dir=${backup_dir:-$s_backup_dir}
backup_dir=${backup_dir:-"/backup"}
local source="${args[source]:-$s_source}"
source=$(echo "${source:-$PWD}" | tr -s /)
local target=${args[target]:-$s_target}
target="$(echo $target | tr -s /)"
if [[ ${args[--dir]:-$s_dir} ]]; then
target="${target}$(echo $source | tr -s / | sed -e "s#^[.]##")"
fi
if [[ $(isAbsPath $target) ]]; then
echo absolute target directory
else
target="/${HOSTNAME}/${target}"
if [[ $server ]]; then
target="${server}${target}"
else
target=${backup_dir}${target}
fi
fi
local exclude=${args[--exclude_file]:-$BACKUP_EXCLUDE}
exclude=${exclude:-$s_exclude}
exclude=${exclude:-"$source/exclude.bac"}
local shost=$([[ ${args[--shost]} ]] && echo ${args[--shost]}::)
local suser=$([[ ${args[--suser]} ]] && echo ${args[--suser]}@)
local thost=$([[ ${args[--host]} ]] && echo ${args[--host]}::)
local tuser=$([[ ${args[--user]} ]] && echo ${args[--user]}@)
local options=$(echo ${args[--options]} | awk '{gsub(/\\/," ")}1')
local bin=$(command -v restic)
local cmd=${args[cmd]:-"backup"}
echo before exists exclude: $exclude
exclude=$([[ -f $exclude ]] && echo "--iexclude-file $exclude" || echo "")
echo source: $source
echo target $target
echo exclude: $exclude
# local ssh="--remote-schema \"ssh -C %s /home/sysadmin/.local/bin/rdiff-backup --server\""
#cmd="$sudo rdiff-backup $options $exclude $ssh ${suser}${shost}$source ${tuser}${thost}$target"
cmd="sudo $password $bin -r $target backup $source $exclude"
if [[ ${args[--init]} ]]; then cmd="sudo $password $bin -r $target init"; fi
if [[ ${args[--snap]} ]]; then cmd="sudo $password $bin -r $target snapshots"; fi
if [[ ${args[--prune]} ]]; then cmd="sudo password $bin -r $target prune"; fi
if [[ ${args[--view]} || ${args[--view-path]} ]]; then
mount=${args[--view-path]:-$BACKUP_MOUNT}
mount=${mount:-"/opt/backup/view/"}
echo view mount point $mount
if [[ -e ${mount} ]]; then
cmd="$password $bin -r $target mount $mount";
echo browse files at $mount/snapshots/latest${source}
else
echo $mount: directory for mounting snapshot for viewing does not exist. Create and try again
return 3
fi
fi
echo $cmd
confirm run this command? || return 1
eval $cmd
# sudo chown -R $USER:$USER $target
# sudo chown -R $USER:$USER /home/$USER/.cache/restic
# sudo chmod -R g+rwX /home/$USER/.cache/restic
}
# :command.version_command
version_command() {
echo "$version"
}
# :command.usage
dbackup_usage() {
if [[ -n $long_usage ]]; then
printf "dbackup - differential backup using restic\n"
echo
else
printf "dbackup - differential backup using restic\n"
echo
fi
printf "Usage:\n"
printf " dbackup [SOURCE] [TARGET] [options]\n"
printf " dbackup --help\n"
printf " dbackup --version\n"
echo
if [[ -n $long_usage ]]; then
printf "Options:\n"
# :command.usage_fixed_flags
echo " --help"
printf " Show this help\n"
echo
echo " --version"
printf " Show version number\n"
echo
# :command.usage_flags
# :flag.usage
echo " --password, -p PASSWORD"
printf " repo password (or file path) for backup repository\n"
echo
# :flag.usage
echo " --remote, -r"
printf " backup to a remote machine\n"
echo
# :flag.usage
echo " --server URL"
printf " url of restic rest server\n"
echo
# :flag.usage
echo " --init"
printf " initialize repo (default is backup)\n"
echo
# :flag.usage
echo " --snap"
printf " list repo snapshots\n"
echo
# :flag.usage
echo " --view, -v"
printf " mount snapshot for viewing (default is BACKUP_MOUNT or /opt/backup/view)\n"
echo
# :flag.usage
echo " --view-path PATH"
printf " set custom mount path for viewing of snapshot, --view not required if set\n"
echo
# :flag.usage
echo " --prune PRUNE"
printf " prune repo (default is backup). true for default prune or path to prune\n settings\n"
echo
# :flag.usage
echo " --password, -p PASSWORD"
printf " repo password (or file path) for backup repository\n"
echo
# :flag.usage
echo " --settings, -s SYAML"
printf " path to settings file (yaml). Keys are same as long\n"
echo
# :flag.usage
echo " --host, -h THOST"
printf " host on remote to target to receive backup\n"
echo
# :flag.usage
echo " --shost SHOST"
printf " remote to host of source\n"
echo
# :flag.usage
echo " --user, -u TUSER"
printf " user on remote host\n"
echo
# :flag.usage
echo " --suser SUSER"
printf " remote user on source host\n"
echo
# :flag.usage
echo " --sshcfg SSHCFG"
printf " path to sshcfg file\n"
echo
# :flag.usage
echo " --options, -o OPTIONS"
printf " additional options (restic or rsync)\n"
echo
# :flag.usage
echo " --include_file, -i INCLUDE"
printf " include file\n"
echo
# :flag.usage
echo " --exclude_file, -e EXCLUDE"
printf " exclude file\n"
echo
# :flag.usage
echo " --dir, -d"
printf " append source directory path to target directory\n"
echo
# :command.usage_args
printf "Arguments:\n"
# :argument.usage
echo " SOURCE"
printf " source directory to be backed up, default is $PWD\n"
echo
# :argument.usage
echo " TARGET"
printf " target directory for backup\n"
echo
# :command.usage_environment_variables
printf "Environment Variables:\n"
# :environment_variable.usage
echo " BACKUP_EXCLUDE"
printf " path to file of excludes\n"
echo
# :environment_variable.usage
echo " BACKUP_INCLUDE"
printf " path to directory of includes\n"
echo
# :environment_variable.usage
echo " BACKUP_SETTINGS"
printf " path to default settings file\n"
echo
# :environment_variable.usage
echo " BACKUP_PASSWORD"
printf " path to default settings file\n"
echo
# :environment_variable.usage
echo " BACKUP_SERVER"
printf " URL of Restic rest server\n"
echo
# :environment_variable.usage
echo " BACKUP_DIR"
printf " Backup Directory\n"
echo
# :command.usage_examples
printf "Examples:\n"
printf " backup -p password . /target/dir\n"
printf " backup -s /path/to/settings/yaml/file\n"
echo
fi
}
# :command.inspect_args
inspect_args() {
readarray -t sorted_keys < <(printf '%s\n' "${!args[@]}" | sort)
if (( ${#args[@]} )); then
echo args:
for k in "${sorted_keys[@]}"; do echo "- \${args[$k]} = ${args[$k]}"; done
else
echo args: none
fi
if (( ${#other_args[@]} )); then
echo
echo other_args:
echo "- \${other_args[*]} = ${other_args[*]}"
for i in "${!other_args[@]}"; do
echo "- \${other_args[$i]} = ${other_args[$i]}"
done
fi
}
# :command.command_functions
# :command.parse_requirements
parse_requirements() {
# :command.fixed_flag_filter
case "$1" in
--version )
version_command
exit
;;
--help )
long_usage=yes
dbackup_usage
exit 1
;;
esac
# :command.environment_variables_filter
# :command.dependencies_filter
# :command.command_filter
action="root"
# :command.required_args_filter
# :command.required_flags_filter
# :command.parse_requirements_while
while [[ $# -gt 0 ]]; do
key="$1"
case "$key" in
# :flag.case
--password | -p )
if [[ $2 ]]; then
args[--password]="$2"
shift
shift
else
printf "%s\n" "--password requires an argument: --password, -p PASSWORD"
exit 1
fi
;;
# :flag.case
--remote | -r )
args[--remote]=1
shift
;;
# :flag.case
--server )
if [[ $2 ]]; then
args[--server]="$2"
shift
shift
else
printf "%s\n" "--server requires an argument: --server URL"
exit 1
fi
;;
# :flag.case
--init )
args[--init]=1
shift
;;
# :flag.case
--snap )
args[--snap]=1
shift
;;
# :flag.case
--view | -v )
args[--view]=1
shift
;;
# :flag.case
--view-path )
if [[ $2 ]]; then
args[--view-path]="$2"
shift
shift
else
printf "%s\n" "--view-path requires an argument: --view-path PATH"
exit 1
fi
;;
# :flag.case
--prune )
if [[ $2 ]]; then
args[--prune]="$2"
shift
shift
else
printf "%s\n" "--prune requires an argument: --prune PRUNE"
exit 1
fi
;;
# :flag.case
--password | -p )
if [[ $2 ]]; then
args[--password]="$2"
shift
shift
else
printf "%s\n" "--password requires an argument: --password, -p PASSWORD"
exit 1
fi
;;
# :flag.case
--settings | -s )
if [[ $2 ]]; then
args[--settings]="$2"
shift
shift
else
printf "%s\n" "--settings requires an argument: --settings, -s SYAML"
exit 1
fi
;;
# :flag.case
--host | -h )
if [[ $2 ]]; then
args[--host]="$2"
shift
shift
else
printf "%s\n" "--host requires an argument: --host, -h THOST"
exit 1
fi
;;
# :flag.case
--shost )
if [[ $2 ]]; then
args[--shost]="$2"
shift
shift
else
printf "%s\n" "--shost requires an argument: --shost SHOST"
exit 1
fi
;;
# :flag.case
--user | -u )
if [[ $2 ]]; then
args[--user]="$2"
shift
shift
else
printf "%s\n" "--user requires an argument: --user, -u TUSER"
exit 1
fi
;;
# :flag.case
--suser )
if [[ $2 ]]; then
args[--suser]="$2"
shift
shift
else
printf "%s\n" "--suser requires an argument: --suser SUSER"
exit 1
fi
;;
# :flag.case
--sshcfg )
if [[ $2 ]]; then
args[--sshcfg]="$2"
shift
shift
else
printf "%s\n" "--sshcfg requires an argument: --sshcfg SSHCFG"
exit 1
fi
;;
# :flag.case
--options | -o )
if [[ $2 ]]; then
args[--options]="$2"
shift
shift
else
printf "%s\n" "--options requires an argument: --options, -o OPTIONS"
exit 1
fi
;;
# :flag.case
--include_file | -i )
if [[ $2 ]]; then
args[--include_file]="$2"
shift
shift
else
printf "%s\n" "--include_file requires an argument: --include_file, -i INCLUDE"
exit 1
fi
;;
# :flag.case
--exclude_file | -e )
if [[ $2 ]]; then
args[--exclude_file]="$2"
shift
shift
else
printf "%s\n" "--exclude_file requires an argument: --exclude_file, -e EXCLUDE"
exit 1
fi
;;
# :flag.case
--dir | -d )
args[--dir]=1
shift
;;
-* )
printf "invalid option: %s\n" "$key"
exit 1
;;
* )
# :command.parse_requirements_case
if [[ ! ${args[source]} ]]; then
args[source]=$1
shift
elif [[ ! ${args[target]} ]]; then
args[target]=$1
shift
else
printf "invalid argument: %s\n" "$key"
exit 1
fi
;;
esac
done
# :command.default_assignments
# :command.whitelist_filter
}
# :command.initialize
initialize() {
version="0.1.0"
long_usage=''
set -e
# :src/initialize.sh
# Code here runs inside the initialize() function
# Use it for anything that you need to run before any other function, like
# setting environment vairables:
# CONFIG_FILE=settings.ini
#
# Feel free to empty (but not delete) this file.
echo running initialize function
}
# :command.run
run() {
declare -A args
declare -a other_args
parse_requirements "$@"
if [[ $action == "root" ]]; then
root_command
fi
}
initialize
run "$@"

View File

@ -1,59 +1,92 @@
name: backup name: dbackup
help: backup/mirror a directory using rdiff or rsync help: differential backup using restic
version: 0.1.0 version: 0.1.0
environment_variables: environment_variables:
- name: BACKUP_EXCLUDE - name: BACKUP_EXCLUDE
help: path to file of excludes help: path to file of excludes
- name: BACKUP_INCLUDE - name: BACKUP_INCLUDE
help: path to directory of includes help: path to directory of includes
- name: BACKUP_SETTINGS
help: path to default settings file
- name: BACKUP_PASSWORD
help: path to default settings file
- name: BACKUP_SERVER
help: URL of Restic rest server
- name: BACKUP_DIR
help: Backup Directory
args: args:
- name: source - name: source
# required: 1 help: source directory to be backed up, default is $PWD
help: source directory to be backed up, default is $PWD - name: target
- name: target help: "target directory for backup"
help: "target directory for backup"
flags: flags:
- long: --host - long: --password
short: -h short: -p
arg: thost arg: password
help: host on remote to target to receive backup help: repo password (or file path) for backup repository
- long: --shost - long: --remote
arg: shost short: -r
help: remote to host of source help: backup to a remote machine
- long: --user - long: --server
short: -u arg: url
arg: tuser help: url of restic rest server
help: user on remote host - long: --init
- long: --suser help: initialize repo (default is backup)
arg: suser - long: --snap
help: remote user on source host help: list repo snapshots
- long: --sshcfg - long: --view
arg: sshcfg short: -v
help: path to sshcfg file help: mount snapshot for viewing (default is BACKUP_MOUNT or /opt/backup/view)
- long: --options - long: --view-path
short : -o help: set custom mount path for viewing of snapshot, --view not required if set
arg: options arg: path
help: rdiff options - long: --prune
- long: --include_file arg: prune
short : -i help: prune repo (default is backup). true for default prune or path to prune settings
arg: options - long: --password
help: rdiff options short: -p
- long: --exclude_file arg: password
short : -e help: repo password (or file path) for backup repository
arg: options - long: --settings
help: rdiff options short: -s
- long: --mirror arg: syaml
short: -m help: path to settings file (yaml). Keys are same as long
help: make a mirror copy instead of a differential snapshot - long: --host
- long: --sudo short: -h
short: -s arg: thost
help: run the backup as sudo help: host on remote to target to receive backup
- long: --no-dir - long: --shost
short: -n arg: shost
help: don't make a directory of same name within the target, merge sub directories help: remote to host of source
- long: --user
short: -u
arg: tuser
help: user on remote host
- long: --suser
arg: suser
help: remote user on source host
- long: --sshcfg
arg: sshcfg
help: path to sshcfg file
- long: --options
short: -o
arg: options
help: additional options (restic or rsync)
- long: --include_file
short: -i
arg: include
help: include file
- long: --exclude_file
short: -e
arg: exclude
help: exclude file
- long: --dir
short: -d
help: append source directory path to target directory
examples: examples:
- backup -m -n -h thost --shost shost -u tuser --suser suser . /target/dir - backup -p password . /target/dir
- backup -s /path/to/settings/yaml/file

View File

@ -4,3 +4,4 @@
# CONFIG_FILE=settings.ini # CONFIG_FILE=settings.ini
# #
# Feel free to empty (but not delete) this file. # Feel free to empty (but not delete) this file.
echo running initialize function

View File

@ -1,35 +1,114 @@
echo Running Backup Command #!/bin/bash
echo "running root command"
inspect_args inspect_args
# module_load ssh # module_load ssh
module_load confirm module_load confirm
module_load path
local settings=${args[--settings]}
if [[ -f $settings ]]; then
echo loading settings file $settings
module_load yaml
eval $(parse_yaml $settings "s_")
echo $s_source
echo $s_target
fi
if [[ $s_server_host ]]; then
s_server="rest:http$([[ $s_server_secure ]] && echo "s")://${s_server_host}$([[ $s_server_port ]] && echo :${s_server_port})"
fi
local password=${args[--password]:-$BACKUP_PASSWORD}
password=${password:-$s_password}
[[ ! $password ]] && echo restic requires a backup repository password, exiting && return 2
password="RESTIC_PASSWORD=${password}"
local server=${args[--server]:-$BACKUP_SERVER}
server=${server:-$s_server}
local backup_dir=${args[--backup_dir]:-$BACKUP_DIR}
backup_dir=${backup_dir:-$s_backup_dir}
backup_dir=${backup_dir:-"/backup"}
local source="${args[source]:-$s_source}"
source=$(echo "${source:-$PWD}" | tr -s /)
local target=${args[target]:-$s_target}
target="$(echo $target | tr -s /)"
if [[ ${args[--dir]:-$s_dir} ]]; then
target="${target}$(echo $source | tr -s / | sed -e "s#^[.]##")"
fi
if [[ $(isAbsPath $target) ]]; then
echo absolute target directory
else
target="/${HOSTNAME}/${target}"
if [[ $server ]]; then
target="${server}${target}"
else
target=${backup_dir}${target}
fi
fi
local exclude=${args[--exclude_file]:-$BACKUP_EXCLUDE}
exclude=${exclude:-$s_exclude}
exclude=${exclude:-"$source/exclude.bac"}
local source=${args[source]}
local target=${args[target]}
local shost=$([[ ${args[--shost]} ]] && echo ${args[--shost]}::) local shost=$([[ ${args[--shost]} ]] && echo ${args[--shost]}::)
local suser=$([[ ${args[--suser]} ]] && echo ${args[--suser]}@) local suser=$([[ ${args[--suser]} ]] && echo ${args[--suser]}@)
local thost=$([[ ${args[--host]} ]] && echo ${args[--host]}::) local thost=$([[ ${args[--host]} ]] && echo ${args[--host]}::)
local tuser=$([[ ${args[--user]} ]] && echo ${args[--user]}@) local tuser=$([[ ${args[--user]} ]] && echo ${args[--user]}@)
local mirror=${args[--mirror]}
local options=$(echo ${args[--options]} | awk '{gsub(/\\/," ")}1') local options=$(echo ${args[--options]} | awk '{gsub(/\\/," ")}1')
local sudo=$([[ ${args[--sudo]} ]] && echo sudo)
local exclude=${args[--exclude_file]}
if [[ ! $exclude ]]; then local bin=$(command -v restic)
echo checking local excludes
[[ -f $source/exclude.bac ]] && exclude="$source/exclude.bac"
# ls $source
# [[ $EXCLUDE_BACKUP ]] && exclude="$BACKUP_EXCLUDE"
fi
exclude=$([[ $exclude ]] && echo --exclude-globbing-filelist $exclude)
echo exclude file: $exclude local cmd=${args[cmd]:-"backup"}
echo before exists exclude: $exclude
exclude=$([[ -f $exclude ]] && echo "--iexclude-file $exclude" || echo "")
echo source: $source
echo target $target
echo exclude: $exclude
# local ssh="--remote-schema \"ssh -C %s /home/sysadmin/.local/bin/rdiff-backup --server\"" # local ssh="--remote-schema \"ssh -C %s /home/sysadmin/.local/bin/rdiff-backup --server\""
cmd="$sudo rdiff-backup $options $exclude $ssh ${suser}${shost}$source ${tuser}${thost}$target" #cmd="$sudo rdiff-backup $options $exclude $ssh ${suser}${shost}$source ${tuser}${thost}$target"
cmd="sudo $password $bin -r $target backup $source $exclude"
if [[ ${args[--init]} ]]; then cmd="sudo $password $bin -r $target init"; fi
if [[ ${args[--snap]} ]]; then cmd="sudo $password $bin -r $target snapshots"; fi
if [[ ${args[--prune]} ]]; then cmd="sudo password $bin -r $target prune"; fi
if [[ ${args[--view]} || ${args[--view-path]} ]]; then
mount=${args[--view-path]:-$BACKUP_MOUNT}
mount=${mount:-"/opt/backup/view/"}
echo view mount point $mount
if [[ -e ${mount} ]]; then
cmd="$password $bin -r $target mount $mount";
echo browse files at $mount/snapshots/latest${source}
else
echo $mount: directory for mounting snapshot for viewing does not exist. Create and try again
return 3
fi
fi
echo $cmd echo $cmd
confirm run this command? || return 1 confirm run this command? || return 1
eval $cmd eval $cmd
# sudo chown -R $USER:$USER $target
# sudo chown -R $USER:$USER /home/$USER/.cache/restic
# sudo chmod -R g+rwX /home/$USER/.cache/restic

View File

@ -1,4 +1,4 @@
shopt -s expand_aliases shopt -s expand_aliases
source ../../app/bashly source ../../app/bashly
bashly generate bashly generate
./backup "$@" ./dbackup -s /backup/local/jobs/remote/hacking.yml $1