From edef6fcde7af188b76d5cde313a344f7547e3626 Mon Sep 17 00:00:00 2001 From: capntack Date: Sun, 2 Apr 2023 13:11:53 -0500 Subject: [PATCH] Initial Commit --- README.md | 0 templates/.restic-password-template | 2 + templates/backups-template.sh | 197 +++++++++++++++++++++++++ templates/rsync-manifest-template.conf | 11 ++ 4 files changed, 210 insertions(+) create mode 100644 README.md create mode 100644 templates/.restic-password-template create mode 100644 templates/backups-template.sh create mode 100644 templates/rsync-manifest-template.conf diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/templates/.restic-password-template b/templates/.restic-password-template new file mode 100644 index 0000000..4f4e687 --- /dev/null +++ b/templates/.restic-password-template @@ -0,0 +1,2 @@ +# Delete all text in this file and replace with only the password +# Be sure to secure this file by running chmod 600 on it \ No newline at end of file diff --git a/templates/backups-template.sh b/templates/backups-template.sh new file mode 100644 index 0000000..c7794f3 --- /dev/null +++ b/templates/backups-template.sh @@ -0,0 +1,197 @@ +#!/bin/bash + +####################### +# backups-template.sh # +####################### + +### A script to perform incremental backups using rsync and restic +### Why both? Rsync for quickly browsable and retrievable backups +### Restic for larger and longer term backups +# Throughout, take note of leading/trailing forward slashes +# Be sure to make script executable: +# chmod u+x ./script.sh +# If you add this script to your crontab, either use all absolute paths... +# Or add add a cd command into the script's dir... +# i.e.: 0 0 * * * cd /path/to/script/dir/ && ./script.sh +# If you run this as root, future runs must also be as root + +# Script-wide variables and commands +readonly DATETIME="$(date '+%Y-%m-%d_%H:%M:%S')" +readonly LOG_DIR="/path/to/log/dir" # Configure this variable +readonly BACKUP_LOG="${LOG_DIR}/backup-log_"${DATETIME}".log" + +set -o errexit +set -o nounset +set -o pipefail +exec 3<&1 4<&2 +trap "exec 2<&4 1<&3" 0 1 2 3 +# Needs moreutils installed for logging timestamps to work. Otherwise, comment next line and uncomment the one after +exec > >(tee >(ts "%Y-%m-%d_%H:%M:%S" > "${BACKUP_LOG}")) 2>&1 +# exec > >(tee "${BACKUP_LOG}") 2>&1 + +# Rsync Manifest 01 Variables +readonly RSYNC_SOURCE_01="/path/to/dir/to/backup/01" # Configure this variable +readonly RSYNC_DEST_01="/path/to/dir/to/backup/to/01" # Configure this variable +readonly RSYNC_DEST_PATH_01="${RSYNC_DEST_01}/${DATETIME}" +readonly RSYNC_LATEST_LINK_01="${RSYNC_DEST_01}/latest" +readonly RSYNC_MANIFEST_01="/path/to/manfiest/01/file.conf" # Configure this variable +# Math is number of versions wanted + 1 for latest hardlink + 1 +# So 9 would retain 7 days +readonly RSYNC_RETENTION_DAYS_01="9" # Configure this variable + +# Rsync Manifest 02 Variables. Remove if only using one manifest. Duplicate and increment if using more. +readonly RSYNC_SOURCE_02="/path/to/dir/to/backup/02" # Configure this variable +readonly RSYNC_DEST_02="/path/to/dir/to/backup/to/02" # Configure this variable +readonly RSYNC_DEST_PATH_02="${RSYNC_DEST_02}/${DATETIME}" +readonly RSYNC_LATEST_LINK_02="${RSYNC_DEST_02}/latest" +readonly RSYNC_MANIFEST_02="/path/to/manfiest/02/file.conf" # Configure this variable +readonly RSYNC_RETENTION_DAYS_02="9" # Configure this variable + +# Restic Backup 01 Variables. +readonly RESTIC_PASSWORD_01="/path/to/restic/password/01.file" # Configure this variable +readonly RESTIC_SOURCE_01="/path/to/restic/repo-01" # Configure this variable +readonly RESTIC_REPO_01="/path/to/backup/dest-01" # Configure this variable +readonly RESTIC_COMPRESSION_01="max" +# RE: Retention policy commands, please pay attention in the code, as you can restrict by tags +# Otherwise this script will apply the retention policy to ALL snapshots from all sources in the repo +readonly RESTIC_RETENTION_DAYS_01="7" # Configure this variable +readonly RESTIC_RETENTION_WEEKS_01="4" # Configure this variable +readonly RESTIC_RETENTION_MONTHS_01="6" # Configure this variable +readonly RESTIC_RETENTION_YEARS_01="1" # Configure this variable +# Tags can be used on backups from multiple sources, and more than one tag can be used on one source +# Mix and combine as desired +readonly RESTIC_TAG_01="tag01" # Configure this variable +readonly RESTIC_TAG_02="tag02" # Configure this variable + +# Restic Backup 02 Variables. Note that you can use the same repo and password for multiple sources. +# Or separate ones. Configure to your own needs. +readonly RESTIC_PASSWORD_02="/path/to/restic/password/02.file" # Configure this variable +readonly RESTIC_SOURCE_02="/path/to/restic/repo-02" # Configure this variable +readonly RESTIC_REPO_02="/path/to/backup/dest-02" # Configure this variable +readonly RESTIC_COMPRESSION_02="max" +readonly RESTIC_RETENTION_DAYS_02="7" # Configure this variable +readonly RESTIC_RETENTION_WEEKS_02="4" # Configure this variable +readonly RESTIC_RETENTION_MONTHS_02="6" # Configure this variable +readonly RESTIC_RETENTION_YEARS_02="1" # Configure this variable +readonly RESTIC_TAG_03="tag03" # Configure this variable +readonly RESTIC_TAG_04="tag04" # Configure this variable + +################### +# RSYNC SCRIPT(S) # +################### + +# Creates the backup directory +mkdir -p "${RSYNC_DEST_01}" + +# -avP will tell rsync to run in archive mode, be verbose, keep partial files if interrupted, and show progress +# --delete will delete any folders in latest that are no longer present in source +# --prune-empty-dirs will not sync any empty dirs, essential to work correctly with the manifests +# --include-from pulls what dirs/paths/files to sync from the manifest +rsync -avP --delete --prune-empty-dirs --include-from="${RSYNC_MANIFEST_01}" \ + "${RSYNC_SOURCE_01}/" \ + --link-dest "${RSYNC_LATEST_LINK_01}" \ + "${RSYNC_DEST_PATH_01}" + +# This will update the latest hardlink +rm -rf "${RSYNC_LATEST_LINK_01}" +ln -s "${RSYNC_DEST_PATH_01}" "${RSYNC_LATEST_LINK_01}" + +# A hacky fix for an issue where the -a switch is required for rsync to compare time stamps for incremental backups +# But the ${BACKUP_PATH} dir's time stamp gets messed up when doing this over NFS +# (will be patched out once a better fix is in place) +touch "${RSYNC_DEST_PATH_01}"/timestamp.fix + +# This will prune excess version folders. +cd "${RSYNC_DEST_01}" +rm -rf `ls -t | tail -n +"${RSYNC_RETENTION_DAYS_01}"` + +# CD backup to script dir to reset for next steps +cd - + +# If you are only using one manifest, feel free to delete the next chunk +# Copy and paste it as many times as you need manifests, ensuring to increment variables +# First we cd back to the script's dir, then continue as normal (can remove if using absolute paths for all variables) + +mkdir -p "${RSYNC_DEST_02}" + +rsync -avP --delete --prune-empty-dirs --include-from="${RSYNC_MANIFEST_02}" \ + "${RSYNC_SOURCE_02}/" \ + --link-dest "${RSYNC_LATEST_LINK_02}" \ + "${RSYNC_DEST_PATH_02}" + +rm -rf "${RSYNC_LATEST_LINK_02}" +ln -s "${RSYNC_DEST_PATH_02}" "${RSYNC_LATEST_LINK_02}" +touch "${RSYNC_DEST_PATH_02}"/timestamp.fix +cd "${RSYNC_DEST_02}" +rm -rf `ls -t | tail -n +"${RSYNC_RETENTION_DAYS_02}"` +cd - + +#################### +# RESTIC SCRIPT(S) # +#################### + +# --p points to the password file +# -r points to the restic repo path +# --tag will tag the snapshot, repeat as many times as necessary +# The final line outputs a log file +restic backup --verbose --compression "${RESTIC_COMPRESSION_01}" \ + -p "${RESTIC_PASSWORD_01}" \ + -r "${RESTIC_REPO_01}" \ + --tag "${RESTIC_TAG_01}" --tag "${RESTIC_TAG_02}" \ + --exclude-caches \ + "${RESTIC_SOURCE_01}" + +# Now we forget snapshots and prune data for the same tags in the repo +restic forget --prune --tag tag1,tag2 \ + -p "${RESTIC_PASSWORD_01}" \ + -r "${RESTIC_REPO_01}" \ + --keep-daily "${RESTIC_RETENTION_DAYS_01}" \ + --keep-weekly "${RESTIC_RETENTION_WEEKS_01}" \ + --keep-monthly "${RESTIC_RETENTION_MONTHS_01}" \ + --keep-yearly "${RESTIC_RETENTION_YEARS_01}" + +# Finally, we verify the integrity of the repo +restic check \ + -p "${RESTIC_PASSWORD_01}" \ + -r "${RESTIC_REPO_01}" + +# If you are only backing up from one source, feel free to delete the next chunk +# Copy and paste it as many times as you have sources, ensuring to increment variables +restic backup --verbose --compression "${RESTIC_COMPRESSION_02}" \ + -p "${RESTIC_PASSWORD_02}" \ + -r "${RESTIC_REPO_02}" \ + --tag "${RESTIC_TAG_03}" --tag "${RESTIC_TAG_04}" \ + --exclude-caches \ + "${RESTIC_SOURCE_02}" + +restic forget --prune --tag tag3,tag4 \ + -p "${RESTIC_PASSWORD_02}" \ + -r "${RESTIC_REPO_02}" \ + --keep-daily "${RESTIC_RETENTION_DAYS_02}" \ + --keep-weekly "${RESTIC_RETENTION_WEEKS_02}" \ + --keep-monthly "${RESTIC_RETENTION_MONTHS_02}" \ + --keep-yearly "${RESTIC_RETENTION_YEARS_02}" + +restic check \ + -p "${RESTIC_PASSWORD_02}" \ + -r "${RESTIC_REPO_02}" + +############## +# TIDYING UP # +############## + +# End of script message in log +echo > >(tee >(echo "$(ts "%Y-%m-%d_%H:%M:%S") Backup Script Complete" > "${BACKUP_LOG}")) + +############# +# FOOTNOTES # +############# + +# Inspiration for the base rsync script: https://linuxconfig.org/how-to-create-incremental-backups-using-rsync-on-linux +# Inspiration for how to format the include-from-file: https://stackoverflow.com/a/32527277 +# Inspiration for command in the rsync script to delete all but the most recent directories: https://stackoverflow.com/a/4127056 +# Inspiration for the base restic script: https://codeberg.org/Taffer/restic-scripts +# More Inspiration for the base restic script: https://forum.yunohost.org/t/daily-automated-backups-using-restic/16812 +# Inspiration for the logging to console and file function: https://unix.stackexchange.com/a/574542 +# Inspiration for adding timestamps to the logfile: https://stackoverflow.com/a/39239416 +# Inspiration for adding timestamp to end of script line: https://www.baeldung.com/linux/prepend-timestamp-command-output \ No newline at end of file diff --git a/templates/rsync-manifest-template.conf b/templates/rsync-manifest-template.conf new file mode 100644 index 0000000..360e8ee --- /dev/null +++ b/templates/rsync-manifest-template.conf @@ -0,0 +1,11 @@ +# manifest-template.conf + +# First, include all dirs ++ */ +# Then, include the dirs/paths/files you want to backup ++ /file.name ++ /*wildcard.possible ++ /Sample-Directory/*** ++ /some/deep/directory/oh/yeah/ +# Finally, exclude everything not explicitely included above +- * \ No newline at end of file