From dc99f875a7a00b3750481e82e2c04891d30f32e2 Mon Sep 17 00:00:00 2001 From: Rohit Hazra Date: Wed, 11 Mar 2020 13:35:10 +0530 Subject: [PATCH 1/8] allow backup name to be changed --- README.md | 3 ++- backup.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ee573a1..e005d69 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ docker run -d --name mongodump \ lgatica/mongodump-s3 ``` -## IAM Policity +## IAM Policiy You need to add a user with the following policies. Be sure to change `your_bucket` by the correct. @@ -103,6 +103,7 @@ You need to add a user with the following policies. Be sure to change `your_buck - `S3_PATH` - Default value is `mongodb`. Example `s3://your_bucket/mongodb` - `MONGO_COMPLETE` - Default not set. If set doing backup full mongodb - `MAX_BACKUPS` - Default not set. If set doing it keeps the last n backups in /backup +- `BACKUP_NAME` - Default is `$(date -u +%Y-%m-%d_%H-%M-%S)_UTC.gz`. If set this is the name of the backup file. Useful when using s3 versioning. (Remember to place .gz extension on your filename) ## License diff --git a/backup.sh b/backup.sh index 5ddc97b..f883809 100755 --- a/backup.sh +++ b/backup.sh @@ -1,7 +1,8 @@ #!/usr/bin/env sh OPTIONS=`python /usr/local/bin/mongouri` -BACKUP_NAME="$(date -u +%Y-%m-%d_%H-%M-%S)_UTC.gz" +DEFAULT_BACKUP_NAME="$(date -u +%Y-%m-%d_%H-%M-%S)_UTC.gz" +BACKUP_NAME=${BACKUP_NAME:-$DEFAULT_BACKUP_NAME} # Run backup mongodump ${OPTIONS} -o /backup/dump From eb7c7a56e1bd57092cf002d0a083332eb1c2fdf6 Mon Sep 17 00:00:00 2001 From: Rohit Hazra Date: Wed, 11 Mar 2020 13:38:18 +0530 Subject: [PATCH 2/8] fix spellings --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e005d69..bf381b8 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ docker run -d --name mongodump \ lgatica/mongodump-s3 ``` -### Inmediatic backup +### Immediate backup ```bash docker run -d --name mongodump \ @@ -65,9 +65,9 @@ docker run -d --name mongodump \ lgatica/mongodump-s3 ``` -## IAM Policiy +## IAM Policy -You need to add a user with the following policies. Be sure to change `your_bucket` by the correct. +You need to add a user with the following policies. Be sure to change `your_bucket` by the correct name. ```xml { From b211965595d10c37594dccc271511c79c555bab3 Mon Sep 17 00:00:00 2001 From: Rohit Hazra Date: Wed, 11 Mar 2020 13:38:53 +0530 Subject: [PATCH 3/8] fix spellings --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bf381b8..952c332 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ You need to add a user with the following policies. Be sure to change `your_buck } ``` -## Extra environmnet +## Extra environment - `S3_PATH` - Default value is `mongodb`. Example `s3://your_bucket/mongodb` - `MONGO_COMPLETE` - Default not set. If set doing backup full mongodb From cf03afbbef3dfa12ac161ec01c379c8e4413c57d Mon Sep 17 00:00:00 2001 From: Rohit Hazra Date: Wed, 11 Mar 2020 14:33:25 +0530 Subject: [PATCH 4/8] add extra options --- README.md | 1 + backup.sh | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 952c332..c1cdcfa 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ You need to add a user with the following policies. Be sure to change `your_buck - `MONGO_COMPLETE` - Default not set. If set doing backup full mongodb - `MAX_BACKUPS` - Default not set. If set doing it keeps the last n backups in /backup - `BACKUP_NAME` - Default is `$(date -u +%Y-%m-%d_%H-%M-%S)_UTC.gz`. If set this is the name of the backup file. Useful when using s3 versioning. (Remember to place .gz extension on your filename) +- `EXTRA_OPTIONS` - Default not set. You would probably need `--authenticationDatabase=admin` ## License diff --git a/backup.sh b/backup.sh index f883809..0878302 100755 --- a/backup.sh +++ b/backup.sh @@ -1,6 +1,7 @@ #!/usr/bin/env sh -OPTIONS=`python /usr/local/bin/mongouri` +OPTIONS=`python mongouri.py` +OPTIONS="$OPTIONS $EXTRA_OPTIONS" DEFAULT_BACKUP_NAME="$(date -u +%Y-%m-%d_%H-%M-%S)_UTC.gz" BACKUP_NAME=${BACKUP_NAME:-$DEFAULT_BACKUP_NAME} @@ -22,4 +23,4 @@ if [ -n "${MAX_BACKUPS}" ]; then done else rm -rf /backup/* -fi +fi \ No newline at end of file From 4ee3b29e09ec0581cc5fc802abe6a9b683f8b3cf Mon Sep 17 00:00:00 2001 From: Rohit Hazra Date: Wed, 11 Mar 2020 14:43:14 +0530 Subject: [PATCH 5/8] add things in readme --- README.md | 7 ++++++- backup.sh | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c1cdcfa..e0dad3b 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,12 @@ You need to add a user with the following policies. Be sure to change `your_buck - `MONGO_COMPLETE` - Default not set. If set doing backup full mongodb - `MAX_BACKUPS` - Default not set. If set doing it keeps the last n backups in /backup - `BACKUP_NAME` - Default is `$(date -u +%Y-%m-%d_%H-%M-%S)_UTC.gz`. If set this is the name of the backup file. Useful when using s3 versioning. (Remember to place .gz extension on your filename) -- `EXTRA_OPTIONS` - Default not set. You would probably need `--authenticationDatabase=admin` +- `EXTRA_OPTIONS` - Default not set. + +## Troubleshoot + +1. If you get SASL Authentication failure, add `--authenticationDatabase=admin` to EXTRA_OPTIONS. +2. If you get "Failed: error writing data for collection ... Unrecognized field 'snapshot'", add `--forceTableScan` to EXTRA_OPTIONS. ## License diff --git a/backup.sh b/backup.sh index 0878302..d9c44e3 100755 --- a/backup.sh +++ b/backup.sh @@ -1,6 +1,6 @@ #!/usr/bin/env sh -OPTIONS=`python mongouri.py` +OPTIONS=`python /usr/local/bin/mongouri` OPTIONS="$OPTIONS $EXTRA_OPTIONS" DEFAULT_BACKUP_NAME="$(date -u +%Y-%m-%d_%H-%M-%S)_UTC.gz" BACKUP_NAME=${BACKUP_NAME:-$DEFAULT_BACKUP_NAME} From 677fc849087fd20265dc6533996eeced9aaf5ae3 Mon Sep 17 00:00:00 2001 From: Rohit Hazra Date: Wed, 11 Mar 2020 17:22:43 +0530 Subject: [PATCH 6/8] add slack notifications and success checks --- README.md | 13 +++++++++++++ backup.sh | 44 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e0dad3b..f37066c 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,19 @@ docker run -d --name mongodump \ lgatica/mongodump-s3 ``` +### Slack Hook +```bash +docker run -d --name mongodump \ + -e "MONGO_URI=mongodb://user:pass@host:port/dbname" + -e "AWS_ACCESS_KEY_ID=your_aws_access_key" + -e "AWS_SECRET_ACCESS_KEY=your_aws_secret_access_key" + -e "AWS_DEFAULT_REGION=us-west-1" + -e "S3_BUCKET=your_aws_bucket" + -e "SLACK_URI=your_slack_uri" + lgatica/mongodump-s3 +``` + + ## IAM Policy You need to add a user with the following policies. Be sure to change `your_bucket` by the correct name. diff --git a/backup.sh b/backup.sh index d9c44e3..152c39f 100755 --- a/backup.sh +++ b/backup.sh @@ -1,26 +1,54 @@ #!/usr/bin/env sh -OPTIONS=`python /usr/local/bin/mongouri` +OPTIONS=`python mongouri.py` OPTIONS="$OPTIONS $EXTRA_OPTIONS" DEFAULT_BACKUP_NAME="$(date -u +%Y-%m-%d_%H-%M-%S)_UTC.gz" BACKUP_NAME=${BACKUP_NAME:-$DEFAULT_BACKUP_NAME} +LOCAL_BACKUP_ROOT_FOLDER="/backup" +LOCAL_DUMP_LOCATION="$LOCAL_BACKUP_ROOT_FOLDER/dump" + +notify() { + message="$BACKUP_NAME has been backed up at s3://${S3_BUCKET}/${S3_PATH}/${BACKUP_NAME}" + if [ "${1}" != "0" ]; then + message="Unable to backup $BACKUP_NAME at s3://${S3_BUCKET}/${S3_PATH}/${BACKUP_NAME}. See Logs." + fi + curl -X POST --data-urlencode "payload={\"text\": \"$message\"}" $SLACK_URI +} # Run backup -mongodump ${OPTIONS} -o /backup/dump +mongodump ${OPTIONS} -o "${LOCAL_DUMP_LOCATION}" +status=$? +if [ "${status}" -eq "1" ]; then + echo "ERROR: Mongodump failed." + notify 1 + exit 1 +fi + # Compress backup -cd /backup/ && tar -cvzf "${BACKUP_NAME}" dump +tar -cvzf "${BACKUP_NAME}" "${LOCAL_DUMP_LOCATION}" + # Upload backup -aws s3 cp "/backup/${BACKUP_NAME}" "s3://${S3_BUCKET}/${S3_PATH}/${BACKUP_NAME}" +aws s3 cp "${LOCAL_BACKUP_ROOT_FOLDER}/${BACKUP_NAME}" "s3://${S3_BUCKET}/${S3_PATH}/${BACKUP_NAME}" +status=$? +echo $status +if [ "${status}" != "0" ]; then + echo "ERROR: AWS Upload failed." + notify 1 + exit 1 +fi + +notify 0 + # Delete temp files -rm -rf /backup/dump +rm -rf "${LOCAL_DUMP_LOCATION}" # Delete backup files if [ -n "${MAX_BACKUPS}" ]; then - while [ $(ls /backup -w 1 | wc -l) -gt ${MAX_BACKUPS} ]; + while [ $(ls ${LOCAL_BACKUP_ROOT_FOLDER} -w 1 | wc -l) -gt ${MAX_BACKUPS} ]; do BACKUP_TO_BE_DELETED=$(ls /backup -w 1 | sort | head -n 1) - rm -rf /backup/${BACKUP_TO_BE_DELETED} + rm -rf ${LOCAL_BACKUP_ROOT_FOLDER}/${BACKUP_TO_BE_DELETED} done else - rm -rf /backup/* + rm -rf ${LOCAL_BACKUP_ROOT_FOLDER}/* fi \ No newline at end of file From 6eebd11b86737ffceb5a5e7543d56f9bc9713fba Mon Sep 17 00:00:00 2001 From: Rohit Hazra Date: Wed, 11 Mar 2020 17:49:40 +0530 Subject: [PATCH 7/8] add slack notifications and success checks --- Dockerfile | 2 +- README.md | 1 + backup.sh | 16 +++++++++------- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 181b4a4..25a8a6c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.7@sha256:8c03bb07a531c53ad7d0f6e7041b64d81f99c6e493cb39abba56d956b MAINTAINER Leonardo Gatica -RUN apk add --no-cache mongodb-tools py2-pip && \ +RUN apk add --no-cache mongodb-tools curl py2-pip && \ pip install pymongo awscli && \ mkdir /backup diff --git a/README.md b/README.md index f37066c..91b08cc 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,7 @@ You need to add a user with the following policies. Be sure to change `your_buck - `MAX_BACKUPS` - Default not set. If set doing it keeps the last n backups in /backup - `BACKUP_NAME` - Default is `$(date -u +%Y-%m-%d_%H-%M-%S)_UTC.gz`. If set this is the name of the backup file. Useful when using s3 versioning. (Remember to place .gz extension on your filename) - `EXTRA_OPTIONS` - Default not set. +- `SLACK_URI` - Default not set. Sends a curl notification to the Slack Incoming Webhook. ## Troubleshoot diff --git a/backup.sh b/backup.sh index 152c39f..6886536 100755 --- a/backup.sh +++ b/backup.sh @@ -1,18 +1,20 @@ #!/usr/bin/env sh -OPTIONS=`python mongouri.py` +OPTIONS=`python /usr/local/bin/mongouri` OPTIONS="$OPTIONS $EXTRA_OPTIONS" DEFAULT_BACKUP_NAME="$(date -u +%Y-%m-%d_%H-%M-%S)_UTC.gz" BACKUP_NAME=${BACKUP_NAME:-$DEFAULT_BACKUP_NAME} -LOCAL_BACKUP_ROOT_FOLDER="/backup" +LOCAL_BACKUP_ROOT_FOLDER="backup" LOCAL_DUMP_LOCATION="$LOCAL_BACKUP_ROOT_FOLDER/dump" notify() { - message="$BACKUP_NAME has been backed up at s3://${S3_BUCKET}/${S3_PATH}/${BACKUP_NAME}" - if [ "${1}" != "0" ]; then - message="Unable to backup $BACKUP_NAME at s3://${S3_BUCKET}/${S3_PATH}/${BACKUP_NAME}. See Logs." + if [ "${SLACK_URI}" ]; then + message="$BACKUP_NAME has been backed up at s3://${S3_BUCKET}/${S3_PATH}/${BACKUP_NAME}" + if [ "${1}" != "0" ]; then + message="Unable to backup $BACKUP_NAME at s3://${S3_BUCKET}/${S3_PATH}/${BACKUP_NAME}. See Logs." + fi + curl -X POST --data-urlencode "payload={\"text\": \"$message\"}" $SLACK_URI fi - curl -X POST --data-urlencode "payload={\"text\": \"$message\"}" $SLACK_URI } # Run backup @@ -25,7 +27,7 @@ if [ "${status}" -eq "1" ]; then fi # Compress backup -tar -cvzf "${BACKUP_NAME}" "${LOCAL_DUMP_LOCATION}" +tar -cvzf "${LOCAL_BACKUP_ROOT_FOLDER}/${BACKUP_NAME}" "${LOCAL_DUMP_LOCATION}" # Upload backup aws s3 cp "${LOCAL_BACKUP_ROOT_FOLDER}/${BACKUP_NAME}" "s3://${S3_BUCKET}/${S3_PATH}/${BACKUP_NAME}" From 15d1b01c5195cda360d5c9f326ae11d0eba58553 Mon Sep 17 00:00:00 2001 From: Rohit Hazra Date: Wed, 11 Mar 2020 17:52:08 +0530 Subject: [PATCH 8/8] fix root backup --- backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backup.sh b/backup.sh index 6886536..588a01a 100755 --- a/backup.sh +++ b/backup.sh @@ -4,7 +4,7 @@ OPTIONS=`python /usr/local/bin/mongouri` OPTIONS="$OPTIONS $EXTRA_OPTIONS" DEFAULT_BACKUP_NAME="$(date -u +%Y-%m-%d_%H-%M-%S)_UTC.gz" BACKUP_NAME=${BACKUP_NAME:-$DEFAULT_BACKUP_NAME} -LOCAL_BACKUP_ROOT_FOLDER="backup" +LOCAL_BACKUP_ROOT_FOLDER="/backup" LOCAL_DUMP_LOCATION="$LOCAL_BACKUP_ROOT_FOLDER/dump" notify() {