Rsync-and-Restic-Backup-Scripts/templates/backups-template.sh

#!/bin/bash

#######################
# backups-template.sh #
#######################

### A script to perform incremental backups using rsync and restic
### Why both? Rsync for quickly browsable and retrievable backups
### Restic for larger and longer term backups
# Throughout, take note of leading/trailing forward slashes
# Be sure to make script executable:
# chmod u+x ./script.sh
# If you add this script to your crontab, either use all absolute paths...
# Or add add a cd command into the script's dir...
# i.e.: 0 0 * * * cd /path/to/script/dir/ && ./script.sh
# If you run this as root, future runs must also be as root

# Script-wide variables and commands
readonly DATETIME="$(date '+%Y-%m-%d_%H:%M:%S')"
readonly LOG_DIR="/path/to/log/dir" # Configure this variable
readonly BACKUP_LOG="${LOG_DIR}/backup-log_"${DATETIME}".log"

set -o errexit
set -o nounset
set -o pipefail
exec 3<&1 4<&2
trap "exec 2<&4 1<&3" 0 1 2 3
# Needs moreutils installed for logging timestamps to work. Otherwise, comment next line and uncomment the one after
exec > >(tee >(ts "%Y-%m-%d_%H:%M:%S" > "${BACKUP_LOG}")) 2>&1
# exec > >(tee "${BACKUP_LOG}") 2>&1

# Rsync Manifest 01 Variables
readonly RSYNC_SOURCE_01="/path/to/dir/to/backup/01" # Configure this variable
readonly RSYNC_DEST_01="/path/to/dir/to/backup/to/01" # Configure this variable
readonly RSYNC_DEST_PATH_01="${RSYNC_DEST_01}/${DATETIME}"
readonly RSYNC_LATEST_LINK_01="${RSYNC_DEST_01}/latest"
readonly RSYNC_MANIFEST_01="/path/to/manfiest/01/file.conf" # Configure this variable
# Math is number of versions wanted + 1 for latest hardlink + 1
# So 9 would retain 7 days
readonly RSYNC_RETENTION_DAYS_01="9" # Configure this variable

# Rsync Manifest 02 Variables. Remove if only using one manifest. Duplicate and increment if using more.
readonly RSYNC_SOURCE_02="/path/to/dir/to/backup/02" # Configure this variable
readonly RSYNC_DEST_02="/path/to/dir/to/backup/to/02" # Configure this variable
readonly RSYNC_DEST_PATH_02="${RSYNC_DEST_02}/${DATETIME}"
readonly RSYNC_LATEST_LINK_02="${RSYNC_DEST_02}/latest"
readonly RSYNC_MANIFEST_02="/path/to/manfiest/02/file.conf" # Configure this variable
readonly RSYNC_RETENTION_DAYS_02="9" # Configure this variable

# Restic Backup 01 Variables.
readonly RESTIC_PASSWORD_01="/path/to/restic/password/01.file" # Configure this variable
readonly RESTIC_SOURCE_01="/path/to/restic/repo-01" # Configure this variable
readonly RESTIC_REPO_01="/path/to/backup/dest-01" # Configure this variable
readonly RESTIC_COMPRESSION_01="max"
# RE: Retention policy commands, please pay attention in the code, as you can restrict by tags
# Otherwise this script will apply the retention policy to ALL snapshots from all sources in the repo
readonly RESTIC_RETENTION_DAYS_01="7" # Configure this variable
readonly RESTIC_RETENTION_WEEKS_01="4" # Configure this variable
readonly RESTIC_RETENTION_MONTHS_01="6" # Configure this variable
readonly RESTIC_RETENTION_YEARS_01="1" # Configure this variable
# Tags can be used on backups from multiple sources, and more than one tag can be used on one source
# Mix and combine as desired
readonly RESTIC_TAG_01="tag01" # Configure this variable
readonly RESTIC_TAG_02="tag02" # Configure this variable

# Restic Backup 02 Variables. Note that you can use the same repo and password for multiple sources.
# Or separate ones. Configure to your own needs.
readonly RESTIC_PASSWORD_02="/path/to/restic/password/02.file" # Configure this variable
readonly RESTIC_SOURCE_02="/path/to/restic/repo-02" # Configure this variable
readonly RESTIC_REPO_02="/path/to/backup/dest-02" # Configure this variable
readonly RESTIC_COMPRESSION_02="max"
readonly RESTIC_RETENTION_DAYS_02="7" # Configure this variable
readonly RESTIC_RETENTION_WEEKS_02="4" # Configure this variable
readonly RESTIC_RETENTION_MONTHS_02="6" # Configure this variable
readonly RESTIC_RETENTION_YEARS_02="1" # Configure this variable
readonly RESTIC_TAG_03="tag03" # Configure this variable
readonly RESTIC_TAG_04="tag04" # Configure this variable

###################
# RSYNC SCRIPT(S) #
###################

# Creates the backup directory
mkdir -p "${RSYNC_DEST_01}"

# -avP will tell rsync to run in archive mode, be verbose, keep partial files if interrupted, and show progress
# --delete will delete any folders in latest that are no longer present in source
# --prune-empty-dirs will not sync any empty dirs, essential to work correctly with the manifests
# --include-from pulls what dirs/paths/files to sync from the manifest
rsync -avP --delete --prune-empty-dirs --include-from="${RSYNC_MANIFEST_01}" \
  "${RSYNC_SOURCE_01}/" \
  --link-dest "${RSYNC_LATEST_LINK_01}" \
  "${RSYNC_DEST_PATH_01}"

# This will update the latest hardlink
rm -rf "${RSYNC_LATEST_LINK_01}"
ln -s "${RSYNC_DEST_PATH_01}" "${RSYNC_LATEST_LINK_01}"

# A hacky fix for an issue where the -a switch is required for rsync to compare time stamps for incremental backups
# But the ${BACKUP_PATH} dir's time stamp gets messed up when doing this over NFS
# (will be patched out once a better fix is in place)
touch "${RSYNC_DEST_PATH_01}"/timestamp.fix

# This will prune excess version folders.
cd "${RSYNC_DEST_01}"
rm -rf `ls -t | tail -n +"${RSYNC_RETENTION_DAYS_01}"`

# CD backup to script dir to reset for next steps
cd -

# If you are only using one manifest, feel free to delete the next chunk
# Copy and paste it as many times as you need manifests, ensuring to increment variables
# First we cd back to the script's dir, then continue as normal (can remove if using absolute paths for all variables)

mkdir -p "${RSYNC_DEST_02}"

rsync -avP --delete --prune-empty-dirs --include-from="${RSYNC_MANIFEST_02}" \
  "${RSYNC_SOURCE_02}/" \
  --link-dest "${RSYNC_LATEST_LINK_02}" \
  "${RSYNC_DEST_PATH_02}"

rm -rf "${RSYNC_LATEST_LINK_02}"
ln -s "${RSYNC_DEST_PATH_02}" "${RSYNC_LATEST_LINK_02}"
touch "${RSYNC_DEST_PATH_02}"/timestamp.fix
cd "${RSYNC_DEST_02}"
rm -rf `ls -t | tail -n +"${RSYNC_RETENTION_DAYS_02}"`
cd -

####################
# RESTIC SCRIPT(S) #
####################

# --p points to the password file
# -r points to the restic repo path
# --tag will tag the snapshot, repeat as many times as necessary
# The final line outputs a log file
restic backup --verbose --compression "${RESTIC_COMPRESSION_01}" \
  -p "${RESTIC_PASSWORD_01}" \
  -r "${RESTIC_REPO_01}" \
  --tag "${RESTIC_TAG_01}" --tag "${RESTIC_TAG_02}" \
  --exclude-caches \
  "${RESTIC_SOURCE_01}"

# Now we forget snapshots and prune data for the same tags in the repo
restic forget --prune --tag tag1,tag2 \
  -p "${RESTIC_PASSWORD_01}" \
  -r "${RESTIC_REPO_01}" \
  --keep-daily "${RESTIC_RETENTION_DAYS_01}" \
  --keep-weekly "${RESTIC_RETENTION_WEEKS_01}" \
  --keep-monthly "${RESTIC_RETENTION_MONTHS_01}" \
  --keep-yearly "${RESTIC_RETENTION_YEARS_01}"

# Finally, we verify the integrity of the repo
restic check \
  -p "${RESTIC_PASSWORD_01}" \
  -r "${RESTIC_REPO_01}"

# If you are only backing up from one source, feel free to delete the next chunk
# Copy and paste it as many times as you have sources, ensuring to increment variables
restic backup --verbose --compression "${RESTIC_COMPRESSION_02}" \
  -p "${RESTIC_PASSWORD_02}" \
  -r "${RESTIC_REPO_02}" \
  --tag "${RESTIC_TAG_03}" --tag "${RESTIC_TAG_04}" \
  --exclude-caches \
  "${RESTIC_SOURCE_02}"

restic forget --prune --tag tag3,tag4 \
  -p "${RESTIC_PASSWORD_02}" \
  -r "${RESTIC_REPO_02}" \
  --keep-daily "${RESTIC_RETENTION_DAYS_02}" \
  --keep-weekly "${RESTIC_RETENTION_WEEKS_02}" \
  --keep-monthly "${RESTIC_RETENTION_MONTHS_02}" \
  --keep-yearly "${RESTIC_RETENTION_YEARS_02}"

restic check \
  -p "${RESTIC_PASSWORD_02}" \
  -r "${RESTIC_REPO_02}"

##############
# TIDYING UP #
##############

# End of script message in log
echo > >(tee >(echo "$(ts "%Y-%m-%d_%H:%M:%S") Backup Script Complete" >> "${BACKUP_LOG}"))

#############
# FOOTNOTES #
#############

# Inspiration for the base rsync script: https://linuxconfig.org/how-to-create-incremental-backups-using-rsync-on-linux
# Inspiration for how to format the include-from-file: https://stackoverflow.com/a/32527277
# Inspiration for command in the rsync script to delete all but the most recent directories: https://stackoverflow.com/a/4127056
# Inspiration for the base restic script: https://codeberg.org/Taffer/restic-scripts
# More Inspiration for the base restic script: https://forum.yunohost.org/t/daily-automated-backups-using-restic/16812
# Inspiration for the logging to console and file function: https://unix.stackexchange.com/a/574542
# Inspiration for adding timestamps to the logfile: https://stackoverflow.com/a/39239416
# Inspiration for adding timestamp to end of script line: https://www.baeldung.com/linux/prepend-timestamp-command-output