Compare commits

..

No commits in common. 'release/full-rewrite' and 'main' have entirely different histories.

11
.gitignore vendored

@ -1,11 +1,6 @@
*log* *.log
*test* .resticPassword
.*
backups.sh backups.sh
restic.sh
resticClean.sh
resticExcludes resticExcludes
rsync.sh
rsyncManifest rsyncManifest
backupsVariables.sh testing
backupsVariablesAWS.sh

@ -1,6 +1,6 @@
# Rsync and Restic Backup Scripts # Rsync and Restic Backup Scripts
> **Thank you for visiting!** If you are viewing this repo on GitHub or GitLab, please note that this is just a mirror. Please visit the [originating repo](https://tacksupport.net/git/Tack-Support/Rsync-and-Restic-Backup-Scripts) for any comments, issues, pull requests, etc. You can sign in with your GitHub or GitLab account via Oauth2. > **Thank you for visiting!** If you are viewing this repo on GitHub or GitLab, please note that this is just a mirror. Please visit the [originating repo](https://tacksupport.net/git/capntack/FOSview) for any comments, issues, pull requests, etc. You can sign in with your GitHub or GitLab account via Oauth2.
<br> <br>
@ -103,25 +103,25 @@ restic backup --verbose --compression max \
10. Verify your backup by first fetching the snapshot ID: 10. Verify your backup by first fetching the snapshot ID:
```bash ```bash
restic snapshots -r /path/to/repo restic -r /path/to/repo snapshots
``` ```
Then list the files within to verify eveything is there: Then list the files within to verify eveything is there:
```bash ```bash
restic ls --long -r /path/to/repo $SNAPSHOT_ID restic ls -r /path/to/repo --long $SNAPSHOT_ID
``` ```
Then compare the backup size to the size of the source. This will retrieve the uncompressed size of the repo, and it won't perfectly align. But it should give you an idea. Then compare the backup size to the size of the source. This will retrieve the uncompressed size of the repo, and it won't perfectly align. But it should give you an idea.
```bash ```bash
restic stats -r /path/to/repo restic ls -r /path/to/repo stats $SNAPSHOT_ID
``` ```
And finally, check the integrity of the repo: And finally, check the integrity of the repo:
```bash ```bash
restic check -r /path/to/repo restic -r /path/to/repo check
``` ```
<br> <br>

@ -1,27 +1,8 @@
#!/bin/bash #!/bin/bash
# Install prereqs cp ./templates/.resticPasswordTemplate ./.resticPassword
apt install rsync restic moreutils
restic self-update
# Copy templates to base directory and rename
cp ./templates/backupsTemplate.sh ./backups.sh cp ./templates/backupsTemplate.sh ./backups.sh
cp ./templates/resticCleanTemplate.sh ./resticClean.sh
cp ./templates/resticExcludesTemplate ./resticExcludes cp ./templates/resticExcludesTemplate ./resticExcludes
cp ./templates/resticTemplate.sh ./restic.sh
cp ./templates/rsyncManifestTemplate ./rsyncManifest cp ./templates/rsyncManifestTemplate ./rsyncManifest
cp ./templates/rsyncTemplate.sh ./rsync.sh chmod 600 ./.resticPassword
cp ./templates/backupsVariablesAWSTemplate.sh ./backupsVariablesAWS.sh
cp ./templates/backupsVariablesTemplate.sh ./backupsVariables.sh
# Set scripts to executable
chmod +x ./backups.sh chmod +x ./backups.sh
chmod +x ./resticClean.sh
chmod +x ./restic.sh
chmod +x ./rsync.sh
chmod +x ./backupsVariablesAWS.sh
chmod +x ./backupsVariables.sh
# Set variable files to r/w by owner only
chmod 600 ./backupsVariablesAWS.sh
chmod 600 ./backupsVariables.sh

@ -0,0 +1 @@
# Delete all text in this file and replace with only the password

@ -1,100 +1,131 @@
#!/bin/bash #!/bin/bash
###############
# SCRIPT PREP #
###############
# Source base directory of script and source variables
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
. "${SCRIPT_DIR}"/backupsVariables.sh
# Update restic
if [ $UPDATE = 'yes' ]; then
restic self-update
fi
############################ ############################
# LOGGING & ERROR HANDLING # # LOGGING & ERROR HANDLING #
############################ ############################
# Setup for logging # Ensure you set the SCRIPT_DIR variable correctly as the error handling will not catch it
mkdir -p "${SCRIPT_DIR}"/logs # Change the LOG_RETENTION if you wish for more or less.
readonly SCRIPT_DIR="/path/to/script/dir"
readonly LOG_DIR="${SCRIPT_DIR}/backupLogs"
readonly DATETIME="$(date '+%Y-%m-%d_%H:%M:%S')"
readonly BACKUP_LOG="${LOG_DIR}/backupLog_"${DATETIME}".log"
readonly LOG_RETENTION="14"
exec 3<&1 4<&2 exec 3<&1 4<&2
trap "exec 2<&4 1<&3" 0 1 2 3 trap "exec 2<&4 1<&3" 0 1 2 3
exec > >(tee >(ts "%Y-%m-%d_%H:%M:%S" > "${BACKUP_LOG}")) 2>&1 exec > >(tee >(ts "%Y-%m-%d_%H:%M:%S" > "${BACKUP_LOG}")) 2>&1
set -eEuo pipefail
# Set logging to debug set -eEuo pipefail
if [ $DEBUG = 'yes' ]; then # Uncomment the below to debug the script
set -x # set -x
fi
# trap 'err_report' ERR
# If script fails, send notification to NTFY
if [ $NTFY_NOTIFICATIONS = 'yes' ]; then # function err_report() {
trap 'err_report' ERR # sleep 5
# curl \
function err_report() { # -T "${BACKUP_LOG}" \
sleep 5 # -H "Filename: backupLog_"${DATETIME}".log" \
curl \ # -H prio:high \
-T "${BACKUP_LOG}" \ # -H "Title: Backup Failed on ${HOSTNAME}" \
-H "Filename: backupLog_"${DATETIME}".log" \ # ntfyUser:ntfyPassword@ntfyDomain/ntfyTopic
-H prio:high \ # }
-H "Title: Backup Failed on ${HOSTNAME}" \
"${NTFY_URL}" ################
} # RSYNC SCRIPT #
fi ################
################## # Configure variables from here...
# BACKUP SCRIPTS # readonly RSYNC_SOURCE_01="/path/to/dir/to/backup-01"
################## readonly RSYNC_DEST_01="/path/to/dir/to/backup/to-01"
readonly RSYNC_MANIFEST_01="${SCRIPT_DIR}/rsyncManifest"
# If selected, run an rsync backup readonly RSYNC_RETENTION_DAYS_01="9"
if [ $RSYNC_BACKUP = 'yes' ]; then # ...to here
. "${SCRIPT_DIR}"/rsync.sh readonly RSYNC_DEST_PATH_01="${RSYNC_DEST_01}/${DATETIME}"
fi readonly RSYNC_LATEST_LINK_01="${RSYNC_DEST_01}/latest"
# If selected, run a local restic backup # Creates the backup directory
if [ $LAN_RESTIC_BACKUP = 'yes' ]; then mkdir -p "${RSYNC_DEST_01}"
. "${SCRIPT_DIR}"/restic.sh
fi # -avP will tell rsync to run in archive mode, be verbose, keep partial files if interrupted, and show progress
rsync -avP --delete --prune-empty-dirs --include-from="${RSYNC_MANIFEST_01}" \
# If selected, run a forget, prune, and check on the local restic repo after a local restic backup "${RSYNC_SOURCE_01}/" \
if [ $LAN_RESTIC_CLEAN = 'yes' ]; then --link-dest "${RSYNC_LATEST_LINK_01}" \
. "${SCRIPT_DIR}"/resticClean.sh "${RSYNC_DEST_PATH_01}"
fi
# This will update the latest hardlink
# Overwrite LAN restic variables with the AWS restic variables rm -rf "${RSYNC_LATEST_LINK_01}"
if [ $LAN_AND_AWS_RESTIC = 'yes' ]; then ln -s "${RSYNC_DEST_PATH_01}" "${RSYNC_LATEST_LINK_01}"
. "${SCRIPT_DIR}"/backupsVariablesAWS.sh
fi # The hacky fix for the NFS destination timestamp bug
touch "${RSYNC_DEST_PATH_01}"/timestamp.fix
# If selected, run a remote restic backup
if [ $AWS_RESTIC_BACKUP = 'yes' ]; then # This will prune excess version folders.
. "${SCRIPT_DIR}"/restic.sh cd "${RSYNC_DEST_01}"
fi rm -rf `ls -t | tail -n +"${RSYNC_RETENTION_DAYS_01}"`
# If selected, run a forget, prune, and check on the remote restic repo after a remote restic backup #################
if [ $AWS_RESTIC_CLEAN = 'yes' ]; then # RESTIC SCRIPT #
. "${SCRIPT_DIR}"/resticClean.sh #################
fi
# Configure all but first and last accordingly.
readonly RESTIC_PASSWORD_01="${SCRIPT_DIR}/.resticPassword"
readonly RESTIC_SOURCE_01="/path/to/dir/to/backup-01"
readonly RESTIC_REPO_01="/path/to/restic/repo-01"
readonly RESTIC_RETENTION_DAYS_01="7"
readonly RESTIC_RETENTION_WEEKS_01="4"
readonly RESTIC_RETENTION_MONTHS_01="6"
readonly RESTIC_RETENTION_YEARS_01="1"
# If you prefer a keep last retention policy, comment out the above 4 and uncomment the below and configure
# readonly RESTIC_RETENTION_KEEP_LAST_01="2"
readonly RESTIC_TAG_01="tag01"
readonly RESTIC_TAG_02="tag02"
readonly RESTIC_EXCLUDES_01="${SCRIPT_DIR}/resticExcludes"
# --p points to the password file, -r points to the restic repo path
restic backup --verbose \
-p "${RESTIC_PASSWORD_01}" \
-r "${RESTIC_REPO_01}" \
--tag "${RESTIC_TAG_01}" --tag "${RESTIC_TAG_02}" \
--exclude-caches \
--exclude-file="${RESTIC_EXCLUDES_01}" \
"${RESTIC_SOURCE_01}"
# Now we forget snapshots and prune data for the same tags in the repo
restic forget --prune --verbose --tag "${RESTIC_TAG_01}","${RESTIC_TAG_02}" \
-p "${RESTIC_PASSWORD_01}" \
-r "${RESTIC_REPO_01}" \
--keep-daily "${RESTIC_RETENTION_DAYS_01}" \
--keep-weekly "${RESTIC_RETENTION_WEEKS_01}" \
--keep-monthly "${RESTIC_RETENTION_MONTHS_01}" \
--keep-yearly "${RESTIC_RETENTION_YEARS_01}"
# If using a keep last retention policy, comment out the above forget command and uncomment the below
# restic forget --prune --verbose --tag "${RESTIC_TAG_01}","${RESTIC_TAG_02}" \
# -p "${RESTIC_PASSWORD_01}" \
# -r "${RESTIC_REPO_01}" \
# --keep-last "${RESTIC_RETENTION_KEEP_LAST_01}"
# Finally, we verify the integrity of the repo
restic check \
-p "${RESTIC_PASSWORD_01}" \
-r "${RESTIC_REPO_01}"
############## ##############
# TIDYING UP # # TIDYING UP #
############## ##############
# Clean up log files older than log retention days # Clean up log files older than 14 days
# find "${LOG_DIR}" -mtime +"${LOG_RETENTION}" -type f -delete
find "${LOG_DIR}"/*.log -mtime +"${LOG_RETENTION}" -type f -delete find "${LOG_DIR}"/*.log -mtime +"${LOG_RETENTION}" -type f -delete
# End of script message in log # End of script message in log
echo > >(tee >(echo "$(ts "%Y-%m-%d_%H:%M:%S") Backup Script Complete on ${HOSTNAME}" >> "${BACKUP_LOG}")) echo > >(tee >(echo "$(ts "%Y-%m-%d_%H:%M:%S") Backup Script Complete" >> "${BACKUP_LOG}"))
# sleep 5
# If script succeeds, send notification to NTFY # curl \
if [ $NTFY_NOTIFICATIONS = 'yes' ]; then # -T "${BACKUP_LOG}" \
sleep 5 # -H "Filename: backupLog_"${DATETIME}".log" \
curl \ # -H prio:low \
-T "${BACKUP_LOG}" \ # -H "Title: Backup Succeeded on ${HOSTNAME}" \
-H "Filename: backupLog_"${DATETIME}".log" \ # ntfyUser:ntfyPassword@ntfyDomain/ntfyTopic
-H prio:low \
-H "Title: Backup Succeeded on ${HOSTNAME}" \
"${NTFY_URL}"
fi

@ -1,41 +0,0 @@
#!/bin/bash
########################
# AWS RESTIC REPO VARS #
########################
## Only set these if LAN_AND_AWS_RESTIC is set to 'yes' in variables.sh
## Also ensure the restic vars in that file or for the LAN restic
## Only uncomment and set this var if you want it different from variables.sh
# readonly RESTIC_CALENDAR_RETENTION='yes'
## Set these for all restic configurations
export RESTIC_REPOSITORY='/path/to/repo'
export AWS_ACCESS_KEY_ID='KEY-ID'
export AWS_SECRET_ACCESS_KEY='SECRET-KEY'
# readonly AWS_CONNECTIONS='2'
# readonly AWS_UPLOAD='2440'
export RESTIC_PASSWORD='PASSWORD'
readonly RESTIC_SOURCE='/path/to/dir/to/backup'
## If you want to backup multiple paths, comment out the above and set the below to 'yes'
## And then add each path to a separate line in the resticIncludes file
readonly RESTIC_SOURCE_FROM_FILE='no'
## RESTIC_CALENDAR_RETENTION must be set to 'yes' for these
## Remove readonly if you want different retention policies
readonly RESTIC_RETENTION_DAYS='0'
readonly RESTIC_RETENTION_WEEKS='4'
readonly RESTIC_RETENTION_MONTHS='0'
readonly RESTIC_RETENTION_YEARS='0'
## RESTIC_CALENDAR_RETENTION must be set to 'no' for this
## Remove readonly if you want different retention policies
# readonly RESTIC_RETENTION_KEEP_LAST='2'
## Recommendation is to leave the first tag be, and configure the second
readonly RESTIC_TAG_01="${HOSTNAME}"
readonly RESTIC_TAG_02='offsite'
## Leave this var be
readonly RESTIC_EXCLUDES="${SCRIPT_DIR}/resticExcludes"

@ -1,109 +0,0 @@
#!/bin/bash
#####################
# FUNCTION SWITCHES #
#####################
## For any function that you want to run, change from "no" to "yes"
## Update restic (restic must be installed and script must be ran as root/sudo for this to work)
readonly UPDATE='no'
## For debugging the script
readonly DEBUG='no'
## For use with NTFY.
## If set 'yes' will need to set NTFY_URL further below
readonly NTFY_NOTIFICATIONS='no'
## For running the rsync script.
## Will need to set the rsync vars further below
readonly RSYNC_BACKUP='no'
## For running the restic script.
readonly LAN_RESTIC_BACKUP='no'
readonly LAN_RESTIC_CLEAN='no'
readonly AWS_RESTIC_BACKUP='no'
readonly AWS_RESTIC_CLEAN='no'
## Set to 'yes' if running a LAN_RESTIC_CLEAN prior to an AWS_RESTIC_BACKUP of the LAN repo
## If set to 'yes' set the restic vars further below for the LAN repo
## And set the AWS restic repo vars in the variables-aws.sh file
## restic retention vars set in this file will be for LAN, variables-aws.sh will have the AWS retention
## Also leave the RESTIC_TAGs and RESTIC_EXCLUDES vars commented out in this file
readonly LAN_AND_AWS_RESTIC='no'
## Set to 'no' if using RESTIC_RETENTION_KEEP_LAST further below
readonly RESTIC_CALENDAR_RETENTION='yes'
############################
# LOGGING & ERROR HANDLING #
############################
## Leave these vars be
readonly LOG_DIR="${SCRIPT_DIR}/logs"
readonly DATETIME="$(date '+%Y-%m-%d_%H:%M:%S')"
readonly BACKUP_LOG="${LOG_DIR}/"${DATETIME}"_backup.log"
## Adjust to your preferred retention
readonly LOG_RETENTION='30'
## If using, must set NTFY_NOTIFICATIONS further above to 'yes'
# readonly NTFY_URL='user:password@ntfy.tld/topic'
################
# RSYNC SCRIPT #
################
## Only uncomment and set these if RSYNC_BACKUP further above is set to 'yes'
## Set these vars
# readonly RSYNC_SOURCE='/home'
# readonly RSYNC_DEST='/path/to/dir/to/backup/to'
# readonly RSYNC_RETENTION_KEEP_LAST='7'
## Leave these vars be
# readonly RSYNC_MANIFEST="${SCRIPT_DIR}/rsyncManifest"
# readonly RSYNC_DEST_PATH="${RSYNC_DEST}/${DATETIME}"
# readonly RSYNC_LATEST_LINK="${RSYNC_DEST}/latest"
#################
# RESTIC SCRIPT #
#################
## Only uncomment and set these if the relevant RESTIC vars further above are set to 'yes'
## Frequency of progress reporting in output/log, defaulted to once per 5 minutes
# export RESTIC_PROGRESS_FPS='0.0033333333333333335'
## Only set these if backing up to a compatible repo
# export AWS_ACCESS_KEY_ID='KEY-ID'
# export AWS_SECRET_ACCESS_KEY='SECRET-KEY'
# readonly AWS_CONNECTIONS='2'
# readonly AWS_UPLOAD='2440'
## Set these for all restic configurations
# export RESTIC_REPOSITORY='/path/to/repo'
# export RESTIC_PASSWORD='PASSWORD'
# readonly RESTIC_SOURCE='/home'
## If you want to backup multiple paths, leave the above commented and set the below to 'yes'
## And then add each path to a separate line in the resticIncludes file
RESTIC_SOURCE_FROM_FILE='no'
## RESTIC_CALENDAR_RETENTION must be set to 'yes' for these
## Remove readonly if LAN_AND_AWS_RESTIC is set to 'yes' and you want different retention policies
# readonly RESTIC_RETENTION_DAYS='3'
# readonly RESTIC_RETENTION_WEEKS='1'
# readonly RESTIC_RETENTION_MONTHS='1'
# readonly RESTIC_RETENTION_YEARS='2'
## RESTIC_CALENDAR_RETENTION must be set to 'no' for this
## Remove readonly if LAN_AND_AWS_RESTIC is set to 'yes' and you want different retention policies
# readonly RESTIC_RETENTION_KEEP_LAST='2'
## Recommendation is to leave the first tag be, and configure the second
# readonly RESTIC_TAG_01="${HOSTNAME}"
# readonly RESTIC_TAG_02='home'
## Leave this var be
# readonly RESTIC_EXCLUDES="${SCRIPT_DIR}/resticExcludes"

@ -1,28 +0,0 @@
#!/bin/bash
# Run retention policy
# Remove tags if you want to run against the entire repo
# Remove any stale locks on repo
restic unlock
if [ $RESTIC_CALENDAR_RETENTION = 'yes' ]; then
restic forget --prune --verbose \
--tag "${RESTIC_TAG_01}","${RESTIC_TAG_02}" \
--keep-daily "${RESTIC_RETENTION_DAYS}" \
--keep-weekly "${RESTIC_RETENTION_WEEKS}" \
--keep-monthly "${RESTIC_RETENTION_MONTHS}" \
--keep-yearly "${RESTIC_RETENTION_YEARS}"
fi
if [ $RESTIC_CALENDAR_RETENTION = 'no' ]; then
restic forget --prune --verbose \
--tag "${RESTIC_TAG_01}","${RESTIC_TAG_02}" \
--keep-last "${RESTIC_RETENTION_KEEP_LAST}"
fi
# Verify the repo's integrity
restic check --verbose
# Ensure lock from above actions is removed
restic unlock

@ -1,7 +1,2 @@
/home/**/.cache # Exclude the .resticPassword file
/home/**/.config/discord/Cache /path/to/restic/password/.resticPassword
/home/**/.steam
/home/**/Games
/home/**/.local/share/Trash
**/backupsVariables.sh
**/backupsVariablesAWS.sh

@ -1,37 +0,0 @@
#!/bin/bash
if [ $AWS_RESTIC_BACKUP = 'no' ] && [ $RESTIC_SOURCE_FROM_FILE = 'no' ]; then
restic backup --verbose \
--tag "${RESTIC_TAG_01}" --tag "${RESTIC_TAG_02}" \
--exclude-caches \
--exclude-file="${RESTIC_EXCLUDES}" \
"${RESTIC_SOURCE}"
fi
if [ $AWS_RESTIC_BACKUP = 'yes' ] && [ $RESTIC_SOURCE_FROM_FILE = 'no' ]; then
restic backup --verbose \
--tag "${RESTIC_TAG_01}" --tag "${RESTIC_TAG_02}" \
--exclude-caches \
--exclude-file="${RESTIC_EXCLUDES}" \
-o s3.connections="${AWS_CONNECTIONS}" \
--limit-upload "${AWS_UPLOAD}" \
"${RESTIC_SOURCE}"
fi
if [ $AWS_RESTIC_BACKUP = 'no' ] && [ $RESTIC_SOURCE_FROM_FILE = 'yes' ]; then
restic backup --verbose \
--files-from "${SCRIPT_DIR}"/resticIncludes \
--tag "${RESTIC_TAG_01}" --tag "${RESTIC_TAG_02}" \
--exclude-caches \
--exclude-file="${RESTIC_EXCLUDES}"
fi
if [ $AWS_RESTIC_BACKUP = 'yes' ] && [ $RESTIC_SOURCE_FROM_FILE = 'yes' ]; then
restic backup --verbose \
--files-from "${SCRIPT_DIR}"/resticIncludes \
--tag "${RESTIC_TAG_01}" --tag "${RESTIC_TAG_02}" \
--exclude-caches \
--exclude-file="${RESTIC_EXCLUDES}" \
-o s3.connections="${AWS_CONNECTIONS}" \
--limit-upload "${AWS_UPLOAD}"
fi

@ -1,11 +1,11 @@
# First, include all dirs # First, include all dirs
+ */ + */
# Second, exclude anything from within the next section that you don't want # Second, exclude anything from within the next section that you don't want
- /Sample-Directory/some/unwanted/dir/***
# Then, include the dirs/paths/files you want to backup # Then, include the dirs/paths/files you want to backup
+ /*/bin/*** + /file.name
+ /*/.local/bin/*** + /*wildcard.possible
+ /*/*shrc + /Sample-Directory/***
+ /*/*history + /some/deep/directory/oh/yeah/
+ **/docker-compose.yml
# Finally, exclude everything not explicitely included above # Finally, exclude everything not explicitely included above
- * - *

@ -1,21 +0,0 @@
#!/bin/bash
# Create the backup directory if it doesn't exist
mkdir -p "${RSYNC_DEST}"
# Run the rsync backup
rsync -avP --delete --prune-empty-dirs --include-from="${RSYNC_MANIFEST}" \
"${RSYNC_SOURCE}/" \
--link-dest "${RSYNC_LATEST_LINK}" \
"${RSYNC_DEST_PATH}"
# Update the latest hardlink
rm -rf "${RSYNC_LATEST_LINK}"
ln -s "${RSYNC_DEST_PATH}" "${RSYNC_LATEST_LINK}"
# A hacky fix for the NFS timestamp
touch "${RSYNC_DEST_PATH}"/timestamp.fix
# Run retention policy
cd "${RSYNC_DEST}"
rm -rf `ls -t | tail -n +$(("${RSYNC_RETENTION_KEEP_LAST}"+2))`
Loading…
Cancel
Save