Added the ability to use an includes file with restic

.gitignore

@@ -1,6 +1,11 @@
-*.log
+*log*
-.resticPassword
+*test*
+.*
 backups.sh
+restic.sh
+resticClean.sh
 resticExcludes
+rsync.sh
 rsyncManifest
-testing
+backupsVariables.sh
+backupsVariablesAWS.sh

README.md

@@ -1,6 +1,6 @@
 # Rsync and Restic Backup Scripts
 
-> **Thank you for visiting!** If you are viewing this repo on GitHub or GitLab, please note that this is just a mirror. Please visit the [originating repo](https://tacksupport.net/git/capntack/FOSview) for any comments, issues, pull requests, etc. You can sign in with your GitHub or GitLab account via Oauth2.
+> **Thank you for visiting!** If you are viewing this repo on GitHub or GitLab, please note that this is just a mirror. Please visit the [originating repo](https://tacksupport.net/git/Tack-Support/Rsync-and-Restic-Backup-Scripts) for any comments, issues, pull requests, etc. You can sign in with your GitHub or GitLab account via Oauth2.
 
 <br>
 
@@ -103,25 +103,25 @@ restic backup --verbose --compression max \
 10. Verify your backup by first fetching the snapshot ID:
 
 ```bash
-restic -r /path/to/repo snapshots
+restic snapshots -r /path/to/repo
 ```
 
 Then list the files within to verify eveything is there:
 
 ```bash
-restic ls -r /path/to/repo --long $SNAPSHOT_ID
+restic ls --long -r /path/to/repo $SNAPSHOT_ID
 ```
 
 Then compare the backup size to the size of the source. This will retrieve the uncompressed size of the repo, and it won't perfectly align. But it should give you an idea.
 
 ```bash
-restic ls -r /path/to/repo stats $SNAPSHOT_ID
+restic stats -r /path/to/repo
 ```
 
 And finally, check the integrity of the repo:
 
 ```bash
-restic -r /path/to/repo check
+restic check -r /path/to/repo
 ```
 
 <br>

setup.sh

@@ -1,8 +1,27 @@
 #!/bin/bash
 
-cp ./templates/.resticPasswordTemplate ./.resticPassword
+# Install prereqs
+apt install rsync restic moreutils
+restic self-update
+
+# Copy templates to base directory and rename
 cp ./templates/backupsTemplate.sh ./backups.sh
+cp ./templates/resticCleanTemplate.sh ./resticClean.sh
 cp ./templates/resticExcludesTemplate ./resticExcludes
+cp ./templates/resticTemplate.sh ./restic.sh
 cp ./templates/rsyncManifestTemplate ./rsyncManifest
-chmod 600 ./.resticPassword
+cp ./templates/rsyncTemplate.sh ./rsync.sh
+cp ./templates/backupsVariablesAWSTemplate.sh ./backupsVariablesAWS.sh
+cp ./templates/backupsVariablesTemplate.sh ./backupsVariables.sh
+
+# Set scripts to executable
 chmod +x ./backups.sh
+chmod +x ./resticClean.sh
+chmod +x ./restic.sh
+chmod +x ./rsync.sh
+chmod +x ./backupsVariablesAWS.sh
+chmod +x ./backupsVariables.sh
+
+# Set variable files to r/w by owner only
+chmod 600 ./backupsVariablesAWS.sh
+chmod 600 ./backupsVariables.sh

templates/.resticPasswordTemplate

@@ -1 +0,0 @@
-# Delete all text in this file and replace with only the password

templates/backupsTemplate.sh

@@ -1,131 +1,100 @@
 #!/bin/bash
 
+###############
+# SCRIPT PREP #
+###############
+
+# Source base directory of script and source variables
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+. "${SCRIPT_DIR}"/backupsVariables.sh
+
+# Update restic
+if [ $UPDATE = 'yes' ]; then
+  restic self-update
+fi
+
 ############################
 # LOGGING & ERROR HANDLING #
 ############################
 
-# Ensure you set the SCRIPT_DIR variable correctly as the error handling will not catch it
+# Setup for logging
-# Change the LOG_RETENTION if you wish for more or less.
+mkdir -p "${SCRIPT_DIR}"/logs
-readonly SCRIPT_DIR="/path/to/script/dir"
-readonly LOG_DIR="${SCRIPT_DIR}/backupLogs"
-readonly DATETIME="$(date '+%Y-%m-%d_%H:%M:%S')"
-readonly BACKUP_LOG="${LOG_DIR}/backupLog_"${DATETIME}".log"
-readonly LOG_RETENTION="14"
-
 exec 3<&1 4<&2
 trap "exec 2<&4 1<&3" 0 1 2 3
 exec > >(tee >(ts "%Y-%m-%d_%H:%M:%S" > "${BACKUP_LOG}")) 2>&1
-
 set -eEuo pipefail
-# Uncomment the below to debug the script
+
-# set -x
+# Set logging to debug
-
+if [ $DEBUG = 'yes' ]; then
-# trap 'err_report' ERR
+  set -x
-
+fi
-# function err_report() {
+
-#     sleep 5
+# If script fails, send notification to NTFY
-#     curl \
+if [ $NTFY_NOTIFICATIONS = 'yes' ]; then
-#       -T "${BACKUP_LOG}" \
+  trap 'err_report' ERR
-#       -H "Filename: backupLog_"${DATETIME}".log" \
+
-#       -H prio:high \
+  function err_report() {
-#       -H "Title: Backup Failed on ${HOSTNAME}" \
+      sleep 5
-#       ntfyUser:ntfyPassword@ntfyDomain/ntfyTopic
+      curl \
-# }
+        -T "${BACKUP_LOG}" \
-
+        -H "Filename: backupLog_"${DATETIME}".log" \
-################
+        -H prio:high \
-# RSYNC SCRIPT #
+        -H "Title: Backup Failed on ${HOSTNAME}" \
-################
+        "${NTFY_URL}"
-
+  }
-# Configure variables from here...
+fi
-readonly RSYNC_SOURCE_01="/path/to/dir/to/backup-01"
+
-readonly RSYNC_DEST_01="/path/to/dir/to/backup/to-01"
+##################
-readonly RSYNC_MANIFEST_01="${SCRIPT_DIR}/rsyncManifest"
+# BACKUP SCRIPTS #
-readonly RSYNC_RETENTION_DAYS_01="9"
+##################
-# ...to here
+
-readonly RSYNC_DEST_PATH_01="${RSYNC_DEST_01}/${DATETIME}"
+# If selected, run an rsync backup
-readonly RSYNC_LATEST_LINK_01="${RSYNC_DEST_01}/latest"
+if [ $RSYNC_BACKUP = 'yes' ]; then
-
+  . "${SCRIPT_DIR}"/rsync.sh
-# Creates the backup directory
+fi
-mkdir -p "${RSYNC_DEST_01}"
+
-
+# If selected, run a local restic backup
-# -avP will tell rsync to run in archive mode, be verbose, keep partial files if interrupted, and show progress
+if [ $LAN_RESTIC_BACKUP = 'yes' ]; then
-rsync -avP --delete --prune-empty-dirs --include-from="${RSYNC_MANIFEST_01}" \
+  . "${SCRIPT_DIR}"/restic.sh
-  "${RSYNC_SOURCE_01}/" \
+fi
-  --link-dest "${RSYNC_LATEST_LINK_01}" \
+
-  "${RSYNC_DEST_PATH_01}"
+# If selected, run a forget, prune, and check on the local restic repo after a local restic backup
-
+if [ $LAN_RESTIC_CLEAN = 'yes' ]; then
-# This will update the latest hardlink
+  . "${SCRIPT_DIR}"/resticClean.sh
-rm -rf "${RSYNC_LATEST_LINK_01}"
+fi
-ln -s "${RSYNC_DEST_PATH_01}" "${RSYNC_LATEST_LINK_01}"
+
-
+# Overwrite LAN restic variables with the AWS restic variables
-# The hacky fix for the NFS destination timestamp bug
+if [ $LAN_AND_AWS_RESTIC = 'yes' ]; then
-touch "${RSYNC_DEST_PATH_01}"/timestamp.fix
+  . "${SCRIPT_DIR}"/backupsVariablesAWS.sh
-
+fi
-# This will prune excess version folders.
+
-cd "${RSYNC_DEST_01}"
+# If selected, run a remote restic backup
-rm -rf `ls -t | tail -n +"${RSYNC_RETENTION_DAYS_01}"`
+if [ $AWS_RESTIC_BACKUP = 'yes' ]; then
-
+  . "${SCRIPT_DIR}"/restic.sh
-#################
+fi
-# RESTIC SCRIPT #
+
-#################
+# If selected, run a forget, prune, and check on the remote restic repo after a remote restic backup
-
+if [ $AWS_RESTIC_CLEAN = 'yes' ]; then
-# Configure all but first and last accordingly.
+  . "${SCRIPT_DIR}"/resticClean.sh
-readonly RESTIC_PASSWORD_01="${SCRIPT_DIR}/.resticPassword"
+fi
-readonly RESTIC_SOURCE_01="/path/to/dir/to/backup-01"
-readonly RESTIC_REPO_01="/path/to/restic/repo-01"
-readonly RESTIC_RETENTION_DAYS_01="7"
-readonly RESTIC_RETENTION_WEEKS_01="4"
-readonly RESTIC_RETENTION_MONTHS_01="6"
-readonly RESTIC_RETENTION_YEARS_01="1"
-# If you prefer a keep last retention policy, comment out the above 4 and uncomment the below and configure
-# readonly RESTIC_RETENTION_KEEP_LAST_01="2"
-readonly RESTIC_TAG_01="tag01"
-readonly RESTIC_TAG_02="tag02"
-readonly RESTIC_EXCLUDES_01="${SCRIPT_DIR}/resticExcludes"
-
-# --p points to the password file, -r points to the restic repo path
-restic backup --verbose \
-  -p "${RESTIC_PASSWORD_01}" \
-  -r "${RESTIC_REPO_01}" \
-  --tag "${RESTIC_TAG_01}" --tag "${RESTIC_TAG_02}" \
-  --exclude-caches \
-  --exclude-file="${RESTIC_EXCLUDES_01}" \
-  "${RESTIC_SOURCE_01}"
-
-# Now we forget snapshots and prune data for the same tags in the repo
-restic forget --prune --verbose --tag "${RESTIC_TAG_01}","${RESTIC_TAG_02}" \
-  -p "${RESTIC_PASSWORD_01}" \
-  -r "${RESTIC_REPO_01}" \
-  --keep-daily "${RESTIC_RETENTION_DAYS_01}" \
-  --keep-weekly "${RESTIC_RETENTION_WEEKS_01}" \
-  --keep-monthly "${RESTIC_RETENTION_MONTHS_01}" \
-  --keep-yearly "${RESTIC_RETENTION_YEARS_01}"
-
-# If using a keep last retention policy, comment out the above forget command and uncomment the below
-# restic forget --prune --verbose --tag "${RESTIC_TAG_01}","${RESTIC_TAG_02}" \
-#   -p "${RESTIC_PASSWORD_01}" \
-#   -r "${RESTIC_REPO_01}" \
-#   --keep-last "${RESTIC_RETENTION_KEEP_LAST_01}"
-
-# Finally, we verify the integrity of the repo
-restic check \
-  -p "${RESTIC_PASSWORD_01}" \
-  -r "${RESTIC_REPO_01}"
 
 ##############
 # TIDYING UP #
 ##############
 
-# Clean up log files older than 14 days
+# Clean up log files older than log retention days
-# find "${LOG_DIR}" -mtime +"${LOG_RETENTION}" -type f -delete
 find "${LOG_DIR}"/*.log -mtime +"${LOG_RETENTION}" -type f -delete
 
 # End of script message in log
-echo > >(tee >(echo "$(ts "%Y-%m-%d_%H:%M:%S") Backup Script Complete" >> "${BACKUP_LOG}"))
+echo > >(tee >(echo "$(ts "%Y-%m-%d_%H:%M:%S") Backup Script Complete on ${HOSTNAME}" >> "${BACKUP_LOG}"))
-# sleep 5
+
-# curl \
+# If script succeeds, send notification to NTFY
-#   -T "${BACKUP_LOG}" \
+if [ $NTFY_NOTIFICATIONS = 'yes' ]; then
-#   -H "Filename: backupLog_"${DATETIME}".log" \
+  sleep 5
-#   -H prio:low \
+  curl \
-#   -H "Title: Backup Succeeded on ${HOSTNAME}" \
+    -T "${BACKUP_LOG}" \
-#   ntfyUser:ntfyPassword@ntfyDomain/ntfyTopic
+    -H "Filename: backupLog_"${DATETIME}".log" \
+    -H prio:low \
+    -H "Title: Backup Succeeded on ${HOSTNAME}" \
+    "${NTFY_URL}"
+fi

templates/backupsVariablesAWSTemplate.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+########################
+# AWS RESTIC REPO VARS #
+########################
+
+## Only set these if LAN_AND_AWS_RESTIC is set to 'yes' in variables.sh
+## Also ensure the restic vars in that file or for the LAN restic
+
+## Only uncomment and set this var if you want it different from variables.sh
+# readonly RESTIC_CALENDAR_RETENTION='yes'
+
+## Set these for all restic configurations
+export RESTIC_REPOSITORY='/path/to/repo'
+export AWS_ACCESS_KEY_ID='KEY-ID'
+export AWS_SECRET_ACCESS_KEY='SECRET-KEY'
+# readonly AWS_CONNECTIONS='2'
+# readonly AWS_UPLOAD='2440'
+export RESTIC_PASSWORD='PASSWORD'
+readonly RESTIC_SOURCE='/path/to/dir/to/backup'
+## If you want to backup multiple paths, comment out the above and set the below to 'yes'
+## And then add each path to a separate line in the resticIncludes file
+readonly RESTIC_SOURCE_FROM_FILE='no'
+
+## RESTIC_CALENDAR_RETENTION must be set to 'yes' for these
+## Remove readonly if you want different retention policies
+readonly RESTIC_RETENTION_DAYS='0'
+readonly RESTIC_RETENTION_WEEKS='4'
+readonly RESTIC_RETENTION_MONTHS='0'
+readonly RESTIC_RETENTION_YEARS='0'
+
+## RESTIC_CALENDAR_RETENTION must be set to 'no' for this
+## Remove readonly if you want different retention policies
+# readonly RESTIC_RETENTION_KEEP_LAST='2'
+
+## Recommendation is to leave the first tag be, and configure the second
+readonly RESTIC_TAG_01="${HOSTNAME}"
+readonly RESTIC_TAG_02='offsite'
+
+## Leave this var be
+readonly RESTIC_EXCLUDES="${SCRIPT_DIR}/resticExcludes"

templates/backupsVariablesTemplate.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#####################
+# FUNCTION SWITCHES #
+#####################
+
+## For any function that you want to run, change from "no" to "yes"
+
+## Update restic (restic must be installed and script must be ran as root/sudo for this to work)
+readonly UPDATE='no'
+
+## For debugging the script
+readonly DEBUG='no'
+
+## For use with NTFY.
+## If set 'yes' will need to set NTFY_URL further below
+readonly NTFY_NOTIFICATIONS='no'
+
+## For running the rsync script.
+## Will need to set the rsync vars further below
+readonly RSYNC_BACKUP='no'
+
+## For running the restic script.
+readonly LAN_RESTIC_BACKUP='no'
+readonly LAN_RESTIC_CLEAN='no'
+readonly AWS_RESTIC_BACKUP='no'
+readonly AWS_RESTIC_CLEAN='no'
+
+## Set to 'yes' if running a LAN_RESTIC_CLEAN prior to an AWS_RESTIC_BACKUP of the LAN repo
+## If set to 'yes' set the restic vars further below for the LAN repo
+## And set the AWS restic repo vars in the variables-aws.sh file
+## restic retention vars set in this file will be for LAN, variables-aws.sh will have the AWS retention
+## Also leave the RESTIC_TAGs and RESTIC_EXCLUDES vars commented out in this file
+readonly LAN_AND_AWS_RESTIC='no'
+
+## Set to 'no' if using RESTIC_RETENTION_KEEP_LAST further below
+readonly RESTIC_CALENDAR_RETENTION='yes'
+
+############################
+# LOGGING & ERROR HANDLING #
+############################
+
+## Leave these vars be
+readonly LOG_DIR="${SCRIPT_DIR}/logs"
+readonly DATETIME="$(date '+%Y-%m-%d_%H:%M:%S')"
+readonly BACKUP_LOG="${LOG_DIR}/"${DATETIME}"_backup.log"
+
+## Adjust to your preferred retention
+readonly LOG_RETENTION='30'
+
+## If using, must set NTFY_NOTIFICATIONS further above to 'yes'
+# readonly NTFY_URL='user:password@ntfy.tld/topic'
+
+################
+# RSYNC SCRIPT #
+################
+
+## Only uncomment and set these if RSYNC_BACKUP further above is set to 'yes'
+
+## Set these vars 
+# readonly RSYNC_SOURCE='/home'
+# readonly RSYNC_DEST='/path/to/dir/to/backup/to'
+# readonly RSYNC_RETENTION_KEEP_LAST='7'
+
+## Leave these vars be
+# readonly RSYNC_MANIFEST="${SCRIPT_DIR}/rsyncManifest"
+# readonly RSYNC_DEST_PATH="${RSYNC_DEST}/${DATETIME}"
+# readonly RSYNC_LATEST_LINK="${RSYNC_DEST}/latest"
+
+#################
+# RESTIC SCRIPT #
+#################
+
+## Only uncomment and set these if the relevant RESTIC vars further above are set to 'yes'
+
+## Frequency of progress reporting in output/log, defaulted to once per 5 minutes
+# export RESTIC_PROGRESS_FPS='0.0033333333333333335'
+
+## Only set these if backing up to a compatible repo
+# export AWS_ACCESS_KEY_ID='KEY-ID'
+# export AWS_SECRET_ACCESS_KEY='SECRET-KEY'
+# readonly AWS_CONNECTIONS='2'
+# readonly AWS_UPLOAD='2440'
+
+## Set these for all restic configurations
+# export RESTIC_REPOSITORY='/path/to/repo'
+# export RESTIC_PASSWORD='PASSWORD'
+# readonly RESTIC_SOURCE='/home'
+## If you want to backup multiple paths, leave the above commented and set the below to 'yes'
+## And then add each path to a separate line in the resticIncludes file
+RESTIC_SOURCE_FROM_FILE='no'
+
+## RESTIC_CALENDAR_RETENTION must be set to 'yes' for these
+## Remove readonly if LAN_AND_AWS_RESTIC is set to 'yes' and you want different retention policies
+# readonly RESTIC_RETENTION_DAYS='3'
+# readonly RESTIC_RETENTION_WEEKS='1'
+# readonly RESTIC_RETENTION_MONTHS='1'
+# readonly RESTIC_RETENTION_YEARS='2'
+
+## RESTIC_CALENDAR_RETENTION must be set to 'no' for this
+## Remove readonly if LAN_AND_AWS_RESTIC is set to 'yes' and you want different retention policies
+# readonly RESTIC_RETENTION_KEEP_LAST='2'
+
+## Recommendation is to leave the first tag be, and configure the second
+# readonly RESTIC_TAG_01="${HOSTNAME}"
+# readonly RESTIC_TAG_02='home'
+
+## Leave this var be
+# readonly RESTIC_EXCLUDES="${SCRIPT_DIR}/resticExcludes"

templates/resticCleanTemplate.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Run retention policy
+# Remove tags if you want to run against the entire repo
+
+# Remove any stale locks on repo
+restic unlock
+
+if [ $RESTIC_CALENDAR_RETENTION = 'yes' ]; then
+  restic forget --prune --verbose \
+    --tag "${RESTIC_TAG_01}","${RESTIC_TAG_02}" \
+    --keep-daily "${RESTIC_RETENTION_DAYS}" \
+    --keep-weekly "${RESTIC_RETENTION_WEEKS}" \
+    --keep-monthly "${RESTIC_RETENTION_MONTHS}" \
+    --keep-yearly "${RESTIC_RETENTION_YEARS}"
+fi
+
+if [ $RESTIC_CALENDAR_RETENTION = 'no' ]; then
+  restic forget --prune --verbose \
+    --tag "${RESTIC_TAG_01}","${RESTIC_TAG_02}" \
+    --keep-last "${RESTIC_RETENTION_KEEP_LAST}"
+fi
+
+# Verify the repo's integrity
+restic check --verbose
+
+# Ensure lock from above actions is removed
+restic unlock

templates/resticExcludesTemplate

@@ -1,2 +1,7 @@
-# Exclude the .resticPassword file
+/home/**/.cache
-/path/to/restic/password/.resticPassword
+/home/**/.config/discord/Cache
+/home/**/.steam
+/home/**/Games
+/home/**/.local/share/Trash
+**/backupsVariables.sh
+**/backupsVariablesAWS.sh

templates/resticTemplate.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+if [ $AWS_RESTIC_BACKUP = 'no' ] && [ $RESTIC_SOURCE_FROM_FILE = 'no' ]; then
+  restic backup --verbose \
+    --tag "${RESTIC_TAG_01}" --tag "${RESTIC_TAG_02}" \
+    --exclude-caches \
+    --exclude-file="${RESTIC_EXCLUDES}" \
+    "${RESTIC_SOURCE}"
+fi
+
+if [ $AWS_RESTIC_BACKUP = 'yes' ] && [ $RESTIC_SOURCE_FROM_FILE = 'no' ]; then
+  restic backup --verbose \
+    --tag "${RESTIC_TAG_01}" --tag "${RESTIC_TAG_02}" \
+    --exclude-caches \
+    --exclude-file="${RESTIC_EXCLUDES}" \
+    -o s3.connections="${AWS_CONNECTIONS}" \
+    --limit-upload "${AWS_UPLOAD}" \
+    "${RESTIC_SOURCE}"
+fi
+
+if [ $AWS_RESTIC_BACKUP = 'no' ] && [ $RESTIC_SOURCE_FROM_FILE = 'yes' ]; then
+  restic backup --verbose \
+    --files-from "${SCRIPT_DIR}"/resticIncludes \
+    --tag "${RESTIC_TAG_01}" --tag "${RESTIC_TAG_02}" \
+    --exclude-caches \
+    --exclude-file="${RESTIC_EXCLUDES}"
+fi
+
+if [ $AWS_RESTIC_BACKUP = 'yes' ] && [ $RESTIC_SOURCE_FROM_FILE = 'yes' ]; then
+  restic backup --verbose \
+    --files-from "${SCRIPT_DIR}"/resticIncludes \
+    --tag "${RESTIC_TAG_01}" --tag "${RESTIC_TAG_02}" \
+    --exclude-caches \
+    --exclude-file="${RESTIC_EXCLUDES}" \
+    -o s3.connections="${AWS_CONNECTIONS}" \
+    --limit-upload "${AWS_UPLOAD}"
+fi

templates/rsyncManifestTemplate

@@ -1,11 +1,11 @@
 # First, include all dirs
 + */
 # Second, exclude anything from within the next section that you don't want
-- /Sample-Directory/some/unwanted/dir/***
 # Then, include the dirs/paths/files you want to backup
-+ /file.name
++ /*/bin/***
-+ /*wildcard.possible
++ /*/.local/bin/***
-+ /Sample-Directory/***
++ /*/*shrc
-+ /some/deep/directory/oh/yeah/
++ /*/*history
++ **/docker-compose.yml
 # Finally, exclude everything not explicitely included above
 - *

templates/rsyncTemplate.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Create the backup directory if it doesn't exist
+mkdir -p "${RSYNC_DEST}"
+
+# Run the rsync backup
+rsync -avP --delete --prune-empty-dirs --include-from="${RSYNC_MANIFEST}" \
+  "${RSYNC_SOURCE}/" \
+  --link-dest "${RSYNC_LATEST_LINK}" \
+  "${RSYNC_DEST_PATH}"
+
+# Update the latest hardlink
+rm -rf "${RSYNC_LATEST_LINK}"
+ln -s "${RSYNC_DEST_PATH}" "${RSYNC_LATEST_LINK}"
+
+# A hacky fix for the NFS timestamp
+touch "${RSYNC_DEST_PATH}"/timestamp.fix
+
+# Run retention policy
+cd "${RSYNC_DEST}"
+rm -rf `ls -t | tail -n +$(("${RSYNC_RETENTION_KEEP_LAST}"+2))`