Added/edited/removed files for testing new release

setup.sh

@@ -1,8 +1,15 @@
 #!/bin/bash
 
-cp ./templates/.resticPasswordTemplate ./.resticPassword
 cp ./templates/backupsTemplate.sh ./backups.sh
+cp ./templates/resticCleanTemplate.sh ./resticClean.sh
 cp ./templates/resticExcludesTemplate ./resticExcludes
+cp ./templates/resticTemplate.sh ./restic.sh
 cp ./templates/rsyncManifestTemplate ./rsyncManifest
-chmod 600 ./.resticPassword
+cp ./templates/rsyncTemplate.sh ./rsync.sh
+cp ./templates/variablesTemplate.sh ./variables.sh
 chmod +x ./backups.sh
+chmod +x ./resticClean.sh
+chmod +x ./restic.sh
+chmod +x ./rsync.sh
+chmod +x ./variables.sh
+chmod 600 ./variables.sh

templates/.resticPasswordTemplate

@@ -1 +0,0 @@
-# Delete all text in this file and replace with only the password

templates/backupsTemplate.sh

@@ -1,131 +1,93 @@
 #!/bin/bash
 
+###############
+# SCRIPT PREP #
+###############
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+. "${SCRIPT_DIR}"/variables.sh
+
 ############################
 # LOGGING & ERROR HANDLING #
 ############################
 
-# Ensure you set the SCRIPT_DIR variable correctly as the error handling will not catch it
+# Setup for logging
-# Change the LOG_RETENTION if you wish for more or less.
-readonly SCRIPT_DIR="/path/to/script/dir"
-readonly LOG_DIR="${SCRIPT_DIR}/backupLogs"
-readonly DATETIME="$(date '+%Y-%m-%d_%H:%M:%S')"
-readonly BACKUP_LOG="${LOG_DIR}/backupLog_"${DATETIME}".log"
-readonly LOG_RETENTION="14"
-
 exec 3<&1 4<&2
 trap "exec 2<&4 1<&3" 0 1 2 3
 exec > >(tee >(ts "%Y-%m-%d_%H:%M:%S" > "${BACKUP_LOG}")) 2>&1
-
 set -eEuo pipefail
-# Uncomment the below to debug the script
+
-# set -x
+# Set logging to debug
-
+if [ $DEBUG = 'yes' ]; then
-# trap 'err_report' ERR
+  set -x
-
+fi
-# function err_report() {
+
-#     sleep 5
+# If script fails, send notification to NTFY
-#     curl \
+if [ $NTFY_NOTIFICATIONS = 'yes' ]; then
-#       -T "${BACKUP_LOG}" \
+  trap 'err_report' ERR
-#       -H "Filename: backupLog_"${DATETIME}".log" \
+
-#       -H prio:high \
+  function err_report() {
-#       -H "Title: Backup Failed on ${HOSTNAME}" \
+      sleep 5
-#       ntfyUser:ntfyPassword@ntfyDomain/ntfyTopic
+      curl \
-# }
+        -T "${BACKUP_LOG}" \
-
+        -H "Filename: backupLog_"${DATETIME}".log" \
-################
+        -H prio:high \
-# RSYNC SCRIPT #
+        -H "Title: Backup Failed on ${HOSTNAME}" \
-################
+        ntfyUser:ntfyPassword@ntfyDomain/ntfyTopic
-
+  }
-# Configure variables from here...
+fi
-readonly RSYNC_SOURCE_01="/path/to/dir/to/backup-01"
+
-readonly RSYNC_DEST_01="/path/to/dir/to/backup/to-01"
+##################
-readonly RSYNC_MANIFEST_01="${SCRIPT_DIR}/rsyncManifest"
+# BACKUP SCRIPTS #
-readonly RSYNC_RETENTION_DAYS_01="9"
+##################
-# ...to here
+
-readonly RSYNC_DEST_PATH_01="${RSYNC_DEST_01}/${DATETIME}"
+# If selected, run an rsync backup
-readonly RSYNC_LATEST_LINK_01="${RSYNC_DEST_01}/latest"
+if [ $RSYNC_BACKUP = 'yes' ]; then
-
+  . "${SCRIPT_DIR}"/rsync.sh
-# Creates the backup directory
+fi
-mkdir -p "${RSYNC_DEST_01}"
+
-
+# If selected, run a local restic backup
-# -avP will tell rsync to run in archive mode, be verbose, keep partial files if interrupted, and show progress
+if [ $LOCAL_RESTIC_BACKUP = 'yes' ]; then
-rsync -avP --delete --prune-empty-dirs --include-from="${RSYNC_MANIFEST_01}" \
+  . "${SCRIPT_DIR}"/rsync.sh
-  "${RSYNC_SOURCE_01}/" \
+fi
-  --link-dest "${RSYNC_LATEST_LINK_01}" \
+
-  "${RSYNC_DEST_PATH_01}"
+# If selected, run a forget, prune, and check on the local restic repo after a local restic backup
-
+if [ $LOCAL_RESTIC_BACKUP_CLEAN = 'yes' ]; then
-# This will update the latest hardlink
+  . "${SCRIPT_DIR}"/rsync.sh
-rm -rf "${RSYNC_LATEST_LINK_01}"
+fi
-ln -s "${RSYNC_DEST_PATH_01}" "${RSYNC_LATEST_LINK_01}"
+
-
+# If selected, run a forget, prune, and check on the local restic repo before a remote restic backup
-# The hacky fix for the NFS destination timestamp bug
+if [ $REMOTE_RESTIC_BACKUP_CLEAN_LOCAL = 'yes' ]; then
-touch "${RSYNC_DEST_PATH_01}"/timestamp.fix
+  . "${SCRIPT_DIR}"/rsync.sh
-
+fi
-# This will prune excess version folders.
+
-cd "${RSYNC_DEST_01}"
+# If selected, run a remote restic backup
-rm -rf `ls -t | tail -n +"${RSYNC_RETENTION_DAYS_01}"`
+if [ $REMOTE_RESTIC_BACKUP = 'yes' ]; then
-
+  . "${SCRIPT_DIR}"/rsync.sh
-#################
+fi
-# RESTIC SCRIPT #
+
-#################
+# If selected, run a forget, prune, and check on the remote restic repo after a remote restic backup
-
+if [ $REMOTE_RESTIC_BACKUP_CLEAN = 'yes' ]; then
-# Configure all but first and last accordingly.
+  . "${SCRIPT_DIR}"/rsync.sh
-readonly RESTIC_PASSWORD_01="${SCRIPT_DIR}/.resticPassword"
+fi
-readonly RESTIC_SOURCE_01="/path/to/dir/to/backup-01"
-readonly RESTIC_REPO_01="/path/to/restic/repo-01"
-readonly RESTIC_RETENTION_DAYS_01="7"
-readonly RESTIC_RETENTION_WEEKS_01="4"
-readonly RESTIC_RETENTION_MONTHS_01="6"
-readonly RESTIC_RETENTION_YEARS_01="1"
-# If you prefer a keep last retention policy, comment out the above 4 and uncomment the below and configure
-# readonly RESTIC_RETENTION_KEEP_LAST_01="2"
-readonly RESTIC_TAG_01="tag01"
-readonly RESTIC_TAG_02="tag02"
-readonly RESTIC_EXCLUDES_01="${SCRIPT_DIR}/resticExcludes"
-
-# --p points to the password file, -r points to the restic repo path
-restic backup --verbose \
-  -p "${RESTIC_PASSWORD_01}" \
-  -r "${RESTIC_REPO_01}" \
-  --tag "${RESTIC_TAG_01}" --tag "${RESTIC_TAG_02}" \
-  --exclude-caches \
-  --exclude-file="${RESTIC_EXCLUDES_01}" \
-  "${RESTIC_SOURCE_01}"
-
-# Now we forget snapshots and prune data for the same tags in the repo
-restic forget --prune --verbose --tag "${RESTIC_TAG_01}","${RESTIC_TAG_02}" \
-  -p "${RESTIC_PASSWORD_01}" \
-  -r "${RESTIC_REPO_01}" \
-  --keep-daily "${RESTIC_RETENTION_DAYS_01}" \
-  --keep-weekly "${RESTIC_RETENTION_WEEKS_01}" \
-  --keep-monthly "${RESTIC_RETENTION_MONTHS_01}" \
-  --keep-yearly "${RESTIC_RETENTION_YEARS_01}"
-
-# If using a keep last retention policy, comment out the above forget command and uncomment the below
-# restic forget --prune --verbose --tag "${RESTIC_TAG_01}","${RESTIC_TAG_02}" \
-#   -p "${RESTIC_PASSWORD_01}" \
-#   -r "${RESTIC_REPO_01}" \
-#   --keep-last "${RESTIC_RETENTION_KEEP_LAST_01}"
-
-# Finally, we verify the integrity of the repo
-restic check \
-  -p "${RESTIC_PASSWORD_01}" \
-  -r "${RESTIC_REPO_01}"
 
 ##############
 # TIDYING UP #
 ##############
 
-# Clean up log files older than 14 days
+# Clean up log files older than log retention days
-# find "${LOG_DIR}" -mtime +"${LOG_RETENTION}" -type f -delete
 find "${LOG_DIR}"/*.log -mtime +"${LOG_RETENTION}" -type f -delete
 
 # End of script message in log
-echo > >(tee >(echo "$(ts "%Y-%m-%d_%H:%M:%S") Backup Script Complete" >> "${BACKUP_LOG}"))
+echo > >(tee >(echo "$(ts "%Y-%m-%d_%H:%M:%S") Backup Script Complete on ${HOSTNAME}" >> "${BACKUP_LOG}"))
-# sleep 5
+
-# curl \
+# If script succeeds, send notification to NTFY
-#   -T "${BACKUP_LOG}" \
+if [ $NTFY_NOTIFICATIONS = 'yes' ]; then
-#   -H "Filename: backupLog_"${DATETIME}".log" \
+  sleep 5
-#   -H prio:low \
+  curl \
-#   -H "Title: Backup Succeeded on ${HOSTNAME}" \
+    -T "${BACKUP_LOG}" \
-#   ntfyUser:ntfyPassword@ntfyDomain/ntfyTopic
+    -H "Filename: backupLog_"${DATETIME}".log" \
+    -H prio:low \
+    -H "Title: Backup Succeeded on ${HOSTNAME}" \
+    ntfyUser:ntfyPassword@ntfyDomain/ntfyTopic
+fi

templates/resticCleanTemplate.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Run retention policy
+
+if [ $RESTIC_CALENDAR_RETENTION = 'yes' ]; then
+  restic forget --prune --verbose \
+    --tag "${RESTIC_TAG_01}","${RESTIC_TAG_02}" \
+    --keep-daily "${RESTIC_RETENTION_DAYS}" \
+    --keep-weekly "${RESTIC_RETENTION_WEEKS}" \
+    --keep-monthly "${RESTIC_RETENTION_MONTHS}" \
+    --keep-yearly "${RESTIC_RETENTION_YEARS}"
+fi
+
+if [ $RESTIC_CALENDAR_RETENTION = 'no' ]; then
+  restic forget --prune --verbose \
+    --tag "${RESTIC_TAG_01}","${RESTIC_TAG_02}" \
+    --keep-last "${RESTIC_RETENTION_KEEP_LAST}"
+fi
+
+# Verify the repo's integrity
+restic check --verbose

templates/resticExcludesTemplate

@@ -1,2 +1,2 @@
-# Exclude the .resticPassword file
+# Exclude the Variables file
-/path/to/restic/password/.resticPassword
+/path/to/variables.sh

templates/resticTemplate.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+restic backup --verbose \
+  --tag "${RESTIC_TAG_01}" --tag "${RESTIC_TAG_02}" \
+  --exclude-caches \
+  --exclude-file="${RESTIC_EXCLUDES}" \
+  "${RESTIC_SOURCE}"

templates/rsyncTemplate.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Create the backup directory if it doesn't exist
+mkdir -p "${RSYNC_DEST}"
+
+# Run the rsync backup
+rsync -avP --delete --prune-empty-dirs --include-from="${RSYNC_MANIFEST}" \
+  "${RSYNC_SOURCE}/" \
+  --link-dest "${RSYNC_LATEST_LINK}" \
+  "${RSYNC_DEST_PATH}"
+
+# Update the latest hardlink
+rm -rf "${RSYNC_LATEST_LINK}"
+ln -s "${RSYNC_DEST_PATH}" "${RSYNC_LATEST_LINK}"
+
+# A hacky fix for the NFS timestamp
+touch "${RSYNC_DEST_PATH}"/timestamp.fix
+
+# Run retention policy
+cd "${RSYNC_DEST}"
+rm -rf `ls -t | tail -n +$(("${RSYNC_RETENTION_DAYS}"+2))`

templates/variablesTemplate.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+#####################
+# FUNCTION SWITCHES #
+#####################
+
+# For any function that you want to run, change from "no" to "yes"
+readonly DEBUG='no'
+readonly NTFY_NOTIFICATIONS='no'
+readonly RSYNC_BACKUP='no'
+readonly LOCAL_RESTIC_BACKUP='no'
+readonly LOCAL_RESTIC_BACKUP_CLEAN='no'
+readonly REMOTE_RESTIC_BACKUP_CLEAN_LOCAL='no'
+readonly REMOTE_RESTIC_BACKUP='no'
+readonly REMOTE_RESTIC_BACKUP_CLEAN='no'
+# Set the below 'no' if using keep last instead
+readonly RESTIC_CALENDAR_RETENTION='yes'
+
+############################
+# LOGGING & ERROR HANDLING #
+############################
+
+# Ensure you set the SCRIPT_DIR variable correctly as the error handling will not catch it
+# Change the LOG_RETENTION if you wish for more or less.
+readonly LOG_DIR="${SCRIPT_DIR}/backupLogs"
+readonly DATETIME="$(date '+%Y-%m-%d_%H:%M:%S')"
+readonly BACKUP_LOG="${LOG_DIR}/backupLog_"${DATETIME}".log"
+readonly LOG_RETENTION="14"
+
+################
+# RSYNC SCRIPT #
+################
+
+## Configure variables from here...
+# readonly RSYNC_SOURCE="/path/to/dir/to/backup"
+# readonly RSYNC_DEST="/path/to/dir/to/backup/to"
+# readonly RSYNC_MANIFEST="${SCRIPT_DIR}/rsyncManifest"
+# readonly RSYNC_RETENTION_DAYS="7"
+## ...to here
+# readonly RSYNC_DEST_PATH="${RSYNC_DEST}/${DATETIME}"
+# readonly RSYNC_LATEST_LINK="${RSYNC_DEST}/latest"
+
+#################
+# RESTIC SCRIPT #
+#################
+
+## Only use the following two AWS vars if backing up to a compatible repo
+# export AWS_ACCESS_KEY_ID="KEY-IID"
+# export AWS_SECRET_ACCESS_KEY="SECRET-KEY"
+## Set all the remaining variables except for the last
+# export RESTIC_PASSWORD="REPOSITORY-PASSWORD"
+# export RESTIC_REPOSITORY="PATH-TO-REPOSITORY"
+# readonly RESTIC_SOURCE="/path/to/dir/to/backup"
+# readonly RESTIC_RETENTION_DAYS="7"
+# readonly RESTIC_RETENTION_WEEKS="4"
+# readonly RESTIC_RETENTION_MONTHS="6"
+# readonly RESTIC_RETENTION_YEARS="1"
+## If you prefer a keep last retention policy, comment out the above 4 and uncomment the below and configure
+# readonly RESTIC_RETENTION_KEEP_LAST="2"
+# readonly RESTIC_TAG_01="${HOSTNAME}"
+# readonly RESTIC_TAG_02="TAG-02"
+# readonly RESTIC_EXCLUDES="${SCRIPT_DIR}/resticExcludes"