diff --git a/setup.sh b/setup.sh index 99493bb..7204558 100644 --- a/setup.sh +++ b/setup.sh @@ -1,8 +1,15 @@ #!/bin/bash -cp ./templates/.resticPasswordTemplate ./.resticPassword cp ./templates/backupsTemplate.sh ./backups.sh +cp ./templates/resticCleanTemplate.sh ./resticClean.sh cp ./templates/resticExcludesTemplate ./resticExcludes +cp ./templates/resticTemplate.sh ./restic.sh cp ./templates/rsyncManifestTemplate ./rsyncManifest -chmod 600 ./.resticPassword -chmod +x ./backups.sh \ No newline at end of file +cp ./templates/rsyncTemplate.sh ./rsync.sh +cp ./templates/variablesTemplate.sh ./variables.sh +chmod +x ./backups.sh +chmod +x ./resticClean.sh +chmod +x ./restic.sh +chmod +x ./rsync.sh +chmod +x ./variables.sh +chmod 600 ./variables.sh \ No newline at end of file diff --git a/templates/.resticPasswordTemplate b/templates/.resticPasswordTemplate deleted file mode 100644 index 34666e1..0000000 --- a/templates/.resticPasswordTemplate +++ /dev/null @@ -1 +0,0 @@ -# Delete all text in this file and replace with only the password \ No newline at end of file diff --git a/templates/backupsTemplate.sh b/templates/backupsTemplate.sh index 1ecbd39..5947697 100644 --- a/templates/backupsTemplate.sh +++ b/templates/backupsTemplate.sh @@ -1,131 +1,93 @@ #!/bin/bash +############### +# SCRIPT PREP # +############### + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +. "${SCRIPT_DIR}"/variables.sh + ############################ # LOGGING & ERROR HANDLING # ############################ -# Ensure you set the SCRIPT_DIR variable correctly as the error handling will not catch it -# Change the LOG_RETENTION if you wish for more or less. -readonly SCRIPT_DIR="/path/to/script/dir" -readonly LOG_DIR="${SCRIPT_DIR}/backupLogs" -readonly DATETIME="$(date '+%Y-%m-%d_%H:%M:%S')" -readonly BACKUP_LOG="${LOG_DIR}/backupLog_"${DATETIME}".log" -readonly LOG_RETENTION="14" - +# Setup for logging exec 3<&1 4<&2 trap "exec 2<&4 1<&3" 0 1 2 3 exec > >(tee >(ts "%Y-%m-%d_%H:%M:%S" > "${BACKUP_LOG}")) 2>&1 - set -eEuo pipefail -# Uncomment the below to debug the script -# set -x - -# trap 'err_report' ERR - -# function err_report() { -# sleep 5 -# curl \ -# -T "${BACKUP_LOG}" \ -# -H "Filename: backupLog_"${DATETIME}".log" \ -# -H prio:high \ -# -H "Title: Backup Failed on ${HOSTNAME}" \ -# ntfyUser:ntfyPassword@ntfyDomain/ntfyTopic -# } - -################ -# RSYNC SCRIPT # -################ - -# Configure variables from here... -readonly RSYNC_SOURCE_01="/path/to/dir/to/backup-01" -readonly RSYNC_DEST_01="/path/to/dir/to/backup/to-01" -readonly RSYNC_MANIFEST_01="${SCRIPT_DIR}/rsyncManifest" -readonly RSYNC_RETENTION_DAYS_01="9" -# ...to here -readonly RSYNC_DEST_PATH_01="${RSYNC_DEST_01}/${DATETIME}" -readonly RSYNC_LATEST_LINK_01="${RSYNC_DEST_01}/latest" - -# Creates the backup directory -mkdir -p "${RSYNC_DEST_01}" - -# -avP will tell rsync to run in archive mode, be verbose, keep partial files if interrupted, and show progress -rsync -avP --delete --prune-empty-dirs --include-from="${RSYNC_MANIFEST_01}" \ - "${RSYNC_SOURCE_01}/" \ - --link-dest "${RSYNC_LATEST_LINK_01}" \ - "${RSYNC_DEST_PATH_01}" - -# This will update the latest hardlink -rm -rf "${RSYNC_LATEST_LINK_01}" -ln -s "${RSYNC_DEST_PATH_01}" "${RSYNC_LATEST_LINK_01}" - -# The hacky fix for the NFS destination timestamp bug -touch "${RSYNC_DEST_PATH_01}"/timestamp.fix - -# This will prune excess version folders. -cd "${RSYNC_DEST_01}" -rm -rf `ls -t | tail -n +"${RSYNC_RETENTION_DAYS_01}"` - -################# -# RESTIC SCRIPT # -################# - -# Configure all but first and last accordingly. -readonly RESTIC_PASSWORD_01="${SCRIPT_DIR}/.resticPassword" -readonly RESTIC_SOURCE_01="/path/to/dir/to/backup-01" -readonly RESTIC_REPO_01="/path/to/restic/repo-01" -readonly RESTIC_RETENTION_DAYS_01="7" -readonly RESTIC_RETENTION_WEEKS_01="4" -readonly RESTIC_RETENTION_MONTHS_01="6" -readonly RESTIC_RETENTION_YEARS_01="1" -# If you prefer a keep last retention policy, comment out the above 4 and uncomment the below and configure -# readonly RESTIC_RETENTION_KEEP_LAST_01="2" -readonly RESTIC_TAG_01="tag01" -readonly RESTIC_TAG_02="tag02" -readonly RESTIC_EXCLUDES_01="${SCRIPT_DIR}/resticExcludes" - -# --p points to the password file, -r points to the restic repo path -restic backup --verbose \ - -p "${RESTIC_PASSWORD_01}" \ - -r "${RESTIC_REPO_01}" \ - --tag "${RESTIC_TAG_01}" --tag "${RESTIC_TAG_02}" \ - --exclude-caches \ - --exclude-file="${RESTIC_EXCLUDES_01}" \ - "${RESTIC_SOURCE_01}" - -# Now we forget snapshots and prune data for the same tags in the repo -restic forget --prune --verbose --tag "${RESTIC_TAG_01}","${RESTIC_TAG_02}" \ - -p "${RESTIC_PASSWORD_01}" \ - -r "${RESTIC_REPO_01}" \ - --keep-daily "${RESTIC_RETENTION_DAYS_01}" \ - --keep-weekly "${RESTIC_RETENTION_WEEKS_01}" \ - --keep-monthly "${RESTIC_RETENTION_MONTHS_01}" \ - --keep-yearly "${RESTIC_RETENTION_YEARS_01}" - -# If using a keep last retention policy, comment out the above forget command and uncomment the below -# restic forget --prune --verbose --tag "${RESTIC_TAG_01}","${RESTIC_TAG_02}" \ -# -p "${RESTIC_PASSWORD_01}" \ -# -r "${RESTIC_REPO_01}" \ -# --keep-last "${RESTIC_RETENTION_KEEP_LAST_01}" - -# Finally, we verify the integrity of the repo -restic check \ - -p "${RESTIC_PASSWORD_01}" \ - -r "${RESTIC_REPO_01}" + +# Set logging to debug +if [ $DEBUG = 'yes' ]; then + set -x +fi + +# If script fails, send notification to NTFY +if [ $NTFY_NOTIFICATIONS = 'yes' ]; then + trap 'err_report' ERR + + function err_report() { + sleep 5 + curl \ + -T "${BACKUP_LOG}" \ + -H "Filename: backupLog_"${DATETIME}".log" \ + -H prio:high \ + -H "Title: Backup Failed on ${HOSTNAME}" \ + ntfyUser:ntfyPassword@ntfyDomain/ntfyTopic + } +fi + +################## +# BACKUP SCRIPTS # +################## + +# If selected, run an rsync backup +if [ $RSYNC_BACKUP = 'yes' ]; then + . "${SCRIPT_DIR}"/rsync.sh +fi + +# If selected, run a local restic backup +if [ $LOCAL_RESTIC_BACKUP = 'yes' ]; then + . "${SCRIPT_DIR}"/rsync.sh +fi + +# If selected, run a forget, prune, and check on the local restic repo after a local restic backup +if [ $LOCAL_RESTIC_BACKUP_CLEAN = 'yes' ]; then + . "${SCRIPT_DIR}"/rsync.sh +fi + +# If selected, run a forget, prune, and check on the local restic repo before a remote restic backup +if [ $REMOTE_RESTIC_BACKUP_CLEAN_LOCAL = 'yes' ]; then + . "${SCRIPT_DIR}"/rsync.sh +fi + +# If selected, run a remote restic backup +if [ $REMOTE_RESTIC_BACKUP = 'yes' ]; then + . "${SCRIPT_DIR}"/rsync.sh +fi + +# If selected, run a forget, prune, and check on the remote restic repo after a remote restic backup +if [ $REMOTE_RESTIC_BACKUP_CLEAN = 'yes' ]; then + . "${SCRIPT_DIR}"/rsync.sh +fi ############## # TIDYING UP # ############## -# Clean up log files older than 14 days -# find "${LOG_DIR}" -mtime +"${LOG_RETENTION}" -type f -delete +# Clean up log files older than log retention days find "${LOG_DIR}"/*.log -mtime +"${LOG_RETENTION}" -type f -delete # End of script message in log -echo > >(tee >(echo "$(ts "%Y-%m-%d_%H:%M:%S") Backup Script Complete" >> "${BACKUP_LOG}")) -# sleep 5 -# curl \ -# -T "${BACKUP_LOG}" \ -# -H "Filename: backupLog_"${DATETIME}".log" \ -# -H prio:low \ -# -H "Title: Backup Succeeded on ${HOSTNAME}" \ -# ntfyUser:ntfyPassword@ntfyDomain/ntfyTopic \ No newline at end of file +echo > >(tee >(echo "$(ts "%Y-%m-%d_%H:%M:%S") Backup Script Complete on ${HOSTNAME}" >> "${BACKUP_LOG}")) + +# If script succeeds, send notification to NTFY +if [ $NTFY_NOTIFICATIONS = 'yes' ]; then + sleep 5 + curl \ + -T "${BACKUP_LOG}" \ + -H "Filename: backupLog_"${DATETIME}".log" \ + -H prio:low \ + -H "Title: Backup Succeeded on ${HOSTNAME}" \ + ntfyUser:ntfyPassword@ntfyDomain/ntfyTopic +fi \ No newline at end of file diff --git a/templates/resticCleanTemplate.sh b/templates/resticCleanTemplate.sh new file mode 100644 index 0000000..8e54bc0 --- /dev/null +++ b/templates/resticCleanTemplate.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Run retention policy + +if [ $RESTIC_CALENDAR_RETENTION = 'yes' ]; then + restic forget --prune --verbose \ + --tag "${RESTIC_TAG_01}","${RESTIC_TAG_02}" \ + --keep-daily "${RESTIC_RETENTION_DAYS}" \ + --keep-weekly "${RESTIC_RETENTION_WEEKS}" \ + --keep-monthly "${RESTIC_RETENTION_MONTHS}" \ + --keep-yearly "${RESTIC_RETENTION_YEARS}" +fi + +if [ $RESTIC_CALENDAR_RETENTION = 'no' ]; then + restic forget --prune --verbose \ + --tag "${RESTIC_TAG_01}","${RESTIC_TAG_02}" \ + --keep-last "${RESTIC_RETENTION_KEEP_LAST}" +fi + +# Verify the repo's integrity +restic check --verbose \ No newline at end of file diff --git a/templates/resticExcludesTemplate b/templates/resticExcludesTemplate index 5e0074b..f8afc96 100644 --- a/templates/resticExcludesTemplate +++ b/templates/resticExcludesTemplate @@ -1,2 +1,2 @@ -# Exclude the .resticPassword file -/path/to/restic/password/.resticPassword \ No newline at end of file +# Exclude the Variables file +/path/to/variables.sh \ No newline at end of file diff --git a/templates/resticTemplate.sh b/templates/resticTemplate.sh new file mode 100644 index 0000000..ec1a539 --- /dev/null +++ b/templates/resticTemplate.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +restic backup --verbose \ + --tag "${RESTIC_TAG_01}" --tag "${RESTIC_TAG_02}" \ + --exclude-caches \ + --exclude-file="${RESTIC_EXCLUDES}" \ + "${RESTIC_SOURCE}" \ No newline at end of file diff --git a/templates/rsyncTemplate.sh b/templates/rsyncTemplate.sh new file mode 100644 index 0000000..c0ec8d9 --- /dev/null +++ b/templates/rsyncTemplate.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Create the backup directory if it doesn't exist +mkdir -p "${RSYNC_DEST}" + +# Run the rsync backup +rsync -avP --delete --prune-empty-dirs --include-from="${RSYNC_MANIFEST}" \ + "${RSYNC_SOURCE}/" \ + --link-dest "${RSYNC_LATEST_LINK}" \ + "${RSYNC_DEST_PATH}" + +# Update the latest hardlink +rm -rf "${RSYNC_LATEST_LINK}" +ln -s "${RSYNC_DEST_PATH}" "${RSYNC_LATEST_LINK}" + +# A hacky fix for the NFS timestamp +touch "${RSYNC_DEST_PATH}"/timestamp.fix + +# Run retention policy +cd "${RSYNC_DEST}" +rm -rf `ls -t | tail -n +$(("${RSYNC_RETENTION_DAYS}"+2))` \ No newline at end of file diff --git a/templates/variablesTemplate.sh b/templates/variablesTemplate.sh new file mode 100644 index 0000000..ae2ed6e --- /dev/null +++ b/templates/variablesTemplate.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +##################### +# FUNCTION SWITCHES # +##################### + +# For any function that you want to run, change from "no" to "yes" +readonly DEBUG='no' +readonly NTFY_NOTIFICATIONS='no' +readonly RSYNC_BACKUP='no' +readonly LOCAL_RESTIC_BACKUP='no' +readonly LOCAL_RESTIC_BACKUP_CLEAN='no' +readonly REMOTE_RESTIC_BACKUP_CLEAN_LOCAL='no' +readonly REMOTE_RESTIC_BACKUP='no' +readonly REMOTE_RESTIC_BACKUP_CLEAN='no' +# Set the below 'no' if using keep last instead +readonly RESTIC_CALENDAR_RETENTION='yes' + +############################ +# LOGGING & ERROR HANDLING # +############################ + +# Ensure you set the SCRIPT_DIR variable correctly as the error handling will not catch it +# Change the LOG_RETENTION if you wish for more or less. +readonly LOG_DIR="${SCRIPT_DIR}/backupLogs" +readonly DATETIME="$(date '+%Y-%m-%d_%H:%M:%S')" +readonly BACKUP_LOG="${LOG_DIR}/backupLog_"${DATETIME}".log" +readonly LOG_RETENTION="14" + +################ +# RSYNC SCRIPT # +################ + +## Configure variables from here... +# readonly RSYNC_SOURCE="/path/to/dir/to/backup" +# readonly RSYNC_DEST="/path/to/dir/to/backup/to" +# readonly RSYNC_MANIFEST="${SCRIPT_DIR}/rsyncManifest" +# readonly RSYNC_RETENTION_DAYS="7" +## ...to here +# readonly RSYNC_DEST_PATH="${RSYNC_DEST}/${DATETIME}" +# readonly RSYNC_LATEST_LINK="${RSYNC_DEST}/latest" + +################# +# RESTIC SCRIPT # +################# + +## Only use the following two AWS vars if backing up to a compatible repo +# export AWS_ACCESS_KEY_ID="KEY-IID" +# export AWS_SECRET_ACCESS_KEY="SECRET-KEY" +## Set all the remaining variables except for the last +# export RESTIC_PASSWORD="REPOSITORY-PASSWORD" +# export RESTIC_REPOSITORY="PATH-TO-REPOSITORY" +# readonly RESTIC_SOURCE="/path/to/dir/to/backup" +# readonly RESTIC_RETENTION_DAYS="7" +# readonly RESTIC_RETENTION_WEEKS="4" +# readonly RESTIC_RETENTION_MONTHS="6" +# readonly RESTIC_RETENTION_YEARS="1" +## If you prefer a keep last retention policy, comment out the above 4 and uncomment the below and configure +# readonly RESTIC_RETENTION_KEEP_LAST="2" +# readonly RESTIC_TAG_01="${HOSTNAME}" +# readonly RESTIC_TAG_02="TAG-02" +# readonly RESTIC_EXCLUDES="${SCRIPT_DIR}/resticExcludes" \ No newline at end of file