#!/bin/bash ####################### # backups-template.sh # ####################### ### A script to perform incremental backups using rsync and restic ### Why both? Rsync for quickly browsable and retrievable backups ### Restic for larger and longer term backups # Throughout, take note of leading/trailing forward slashes # Be sure to make script executable: # chmod u+x ./script.sh # If you add this script to your crontab, either use all absolute paths... # Or add add a cd command into the script's dir... # i.e.: 0 0 * * * cd /path/to/script/dir/ && ./script.sh # If you run this as root, future runs must also be as root # Script-wide variables and commands readonly DATETIME="$(date '+%Y-%m-%d_%H:%M:%S')" readonly LOG_DIR="/path/to/log/dir" # Configure this variable readonly BACKUP_LOG="${LOG_DIR}/backup-log_"${DATETIME}".log" set -o errexit set -o nounset set -o pipefail exec 3<&1 4<&2 trap "exec 2<&4 1<&3" 0 1 2 3 # Needs moreutils installed for logging timestamps to work. Otherwise, comment next line and uncomment the one after exec > >(tee >(ts "%Y-%m-%d_%H:%M:%S" > "${BACKUP_LOG}")) 2>&1 # exec > >(tee "${BACKUP_LOG}") 2>&1 # Rsync Manifest 01 Variables readonly RSYNC_SOURCE_01="/path/to/dir/to/backup/01" # Configure this variable readonly RSYNC_DEST_01="/path/to/dir/to/backup/to/01" # Configure this variable readonly RSYNC_DEST_PATH_01="${RSYNC_DEST_01}/${DATETIME}" readonly RSYNC_LATEST_LINK_01="${RSYNC_DEST_01}/latest" readonly RSYNC_MANIFEST_01="/path/to/manfiest/01/file.conf" # Configure this variable # Math is number of versions wanted + 1 for latest hardlink + 1 # So 9 would retain 7 days readonly RSYNC_RETENTION_DAYS_01="9" # Configure this variable # Rsync Manifest 02 Variables. Remove if only using one manifest. Duplicate and increment if using more. readonly RSYNC_SOURCE_02="/path/to/dir/to/backup/02" # Configure this variable readonly RSYNC_DEST_02="/path/to/dir/to/backup/to/02" # Configure this variable readonly RSYNC_DEST_PATH_02="${RSYNC_DEST_02}/${DATETIME}" readonly RSYNC_LATEST_LINK_02="${RSYNC_DEST_02}/latest" readonly RSYNC_MANIFEST_02="/path/to/manfiest/02/file.conf" # Configure this variable readonly RSYNC_RETENTION_DAYS_02="9" # Configure this variable # Restic Backup 01 Variables. readonly RESTIC_PASSWORD_01="/path/to/restic/password/01.file" # Configure this variable readonly RESTIC_SOURCE_01="/path/to/restic/repo-01" # Configure this variable readonly RESTIC_REPO_01="/path/to/backup/dest-01" # Configure this variable readonly RESTIC_COMPRESSION_01="max" # RE: Retention policy commands, please pay attention in the code, as you can restrict by tags # Otherwise this script will apply the retention policy to ALL snapshots from all sources in the repo readonly RESTIC_RETENTION_DAYS_01="7" # Configure this variable readonly RESTIC_RETENTION_WEEKS_01="4" # Configure this variable readonly RESTIC_RETENTION_MONTHS_01="6" # Configure this variable readonly RESTIC_RETENTION_YEARS_01="1" # Configure this variable # Tags can be used on backups from multiple sources, and more than one tag can be used on one source # Mix and combine as desired readonly RESTIC_TAG_01="tag01" # Configure this variable readonly RESTIC_TAG_02="tag02" # Configure this variable # Restic Backup 02 Variables. Note that you can use the same repo and password for multiple sources. # Or separate ones. Configure to your own needs. readonly RESTIC_PASSWORD_02="/path/to/restic/password/02.file" # Configure this variable readonly RESTIC_SOURCE_02="/path/to/restic/repo-02" # Configure this variable readonly RESTIC_REPO_02="/path/to/backup/dest-02" # Configure this variable readonly RESTIC_COMPRESSION_02="max" readonly RESTIC_RETENTION_DAYS_02="7" # Configure this variable readonly RESTIC_RETENTION_WEEKS_02="4" # Configure this variable readonly RESTIC_RETENTION_MONTHS_02="6" # Configure this variable readonly RESTIC_RETENTION_YEARS_02="1" # Configure this variable readonly RESTIC_TAG_03="tag03" # Configure this variable readonly RESTIC_TAG_04="tag04" # Configure this variable ################### # RSYNC SCRIPT(S) # ################### # Creates the backup directory mkdir -p "${RSYNC_DEST_01}" # -avP will tell rsync to run in archive mode, be verbose, keep partial files if interrupted, and show progress # --delete will delete any folders in latest that are no longer present in source # --prune-empty-dirs will not sync any empty dirs, essential to work correctly with the manifests # --include-from pulls what dirs/paths/files to sync from the manifest rsync -avP --delete --prune-empty-dirs --include-from="${RSYNC_MANIFEST_01}" \ "${RSYNC_SOURCE_01}/" \ --link-dest "${RSYNC_LATEST_LINK_01}" \ "${RSYNC_DEST_PATH_01}" # This will update the latest hardlink rm -rf "${RSYNC_LATEST_LINK_01}" ln -s "${RSYNC_DEST_PATH_01}" "${RSYNC_LATEST_LINK_01}" # A hacky fix for an issue where the -a switch is required for rsync to compare time stamps for incremental backups # But the ${BACKUP_PATH} dir's time stamp gets messed up when doing this over NFS # (will be patched out once a better fix is in place) touch "${RSYNC_DEST_PATH_01}"/timestamp.fix # This will prune excess version folders. cd "${RSYNC_DEST_01}" rm -rf `ls -t | tail -n +"${RSYNC_RETENTION_DAYS_01}"` # CD backup to script dir to reset for next steps cd - # If you are only using one manifest, feel free to delete the next chunk # Copy and paste it as many times as you need manifests, ensuring to increment variables # First we cd back to the script's dir, then continue as normal (can remove if using absolute paths for all variables) mkdir -p "${RSYNC_DEST_02}" rsync -avP --delete --prune-empty-dirs --include-from="${RSYNC_MANIFEST_02}" \ "${RSYNC_SOURCE_02}/" \ --link-dest "${RSYNC_LATEST_LINK_02}" \ "${RSYNC_DEST_PATH_02}" rm -rf "${RSYNC_LATEST_LINK_02}" ln -s "${RSYNC_DEST_PATH_02}" "${RSYNC_LATEST_LINK_02}" touch "${RSYNC_DEST_PATH_02}"/timestamp.fix cd "${RSYNC_DEST_02}" rm -rf `ls -t | tail -n +"${RSYNC_RETENTION_DAYS_02}"` cd - #################### # RESTIC SCRIPT(S) # #################### # --p points to the password file # -r points to the restic repo path # --tag will tag the snapshot, repeat as many times as necessary # The final line outputs a log file restic backup --verbose --compression "${RESTIC_COMPRESSION_01}" \ -p "${RESTIC_PASSWORD_01}" \ -r "${RESTIC_REPO_01}" \ --tag "${RESTIC_TAG_01}" --tag "${RESTIC_TAG_02}" \ --exclude-caches \ "${RESTIC_SOURCE_01}" # Now we forget snapshots and prune data for the same tags in the repo restic forget --prune --tag tag1,tag2 \ -p "${RESTIC_PASSWORD_01}" \ -r "${RESTIC_REPO_01}" \ --keep-daily "${RESTIC_RETENTION_DAYS_01}" \ --keep-weekly "${RESTIC_RETENTION_WEEKS_01}" \ --keep-monthly "${RESTIC_RETENTION_MONTHS_01}" \ --keep-yearly "${RESTIC_RETENTION_YEARS_01}" # Finally, we verify the integrity of the repo restic check \ -p "${RESTIC_PASSWORD_01}" \ -r "${RESTIC_REPO_01}" # If you are only backing up from one source, feel free to delete the next chunk # Copy and paste it as many times as you have sources, ensuring to increment variables restic backup --verbose --compression "${RESTIC_COMPRESSION_02}" \ -p "${RESTIC_PASSWORD_02}" \ -r "${RESTIC_REPO_02}" \ --tag "${RESTIC_TAG_03}" --tag "${RESTIC_TAG_04}" \ --exclude-caches \ "${RESTIC_SOURCE_02}" restic forget --prune --tag tag3,tag4 \ -p "${RESTIC_PASSWORD_02}" \ -r "${RESTIC_REPO_02}" \ --keep-daily "${RESTIC_RETENTION_DAYS_02}" \ --keep-weekly "${RESTIC_RETENTION_WEEKS_02}" \ --keep-monthly "${RESTIC_RETENTION_MONTHS_02}" \ --keep-yearly "${RESTIC_RETENTION_YEARS_02}" restic check \ -p "${RESTIC_PASSWORD_02}" \ -r "${RESTIC_REPO_02}" ############## # TIDYING UP # ############## # End of script message in log echo > >(tee >(echo "$(ts "%Y-%m-%d_%H:%M:%S") Backup Script Complete" >> "${BACKUP_LOG}")) ############# # FOOTNOTES # ############# # Inspiration for the base rsync script: https://linuxconfig.org/how-to-create-incremental-backups-using-rsync-on-linux # Inspiration for how to format the include-from-file: https://stackoverflow.com/a/32527277 # Inspiration for command in the rsync script to delete all but the most recent directories: https://stackoverflow.com/a/4127056 # Inspiration for the base restic script: https://codeberg.org/Taffer/restic-scripts # More Inspiration for the base restic script: https://forum.yunohost.org/t/daily-automated-backups-using-restic/16812 # Inspiration for the logging to console and file function: https://unix.stackexchange.com/a/574542 # Inspiration for adding timestamps to the logfile: https://stackoverflow.com/a/39239416 # Inspiration for adding timestamp to end of script line: https://www.baeldung.com/linux/prepend-timestamp-command-output