diff options
| -rw-r--r-- | CHANGELOG.md | 1 | ||||
| -rw-r--r-- | Dockerfile | 2 | ||||
| -rw-r--r-- | docker/Dockerfile.aarch64 | 2 | ||||
| -rw-r--r-- | docker/Dockerfile.rpi | 2 | ||||
| -rw-r--r-- | docker/README.md | 38 | ||||
| -rw-r--r-- | docker/runtime/backup-init.sh | 140 | ||||
| -rw-r--r-- | docker/runtime/backup-job.sh | 33 | ||||
| -rw-r--r-- | docker/runtime/backup-rotator.sh | 27 | ||||
| -rwxr-xr-x | docker/start.sh | 1 |
9 files changed, 243 insertions, 3 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index ad1662d6..8477051b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ All notable changes to Gogs are documented in this file. - Able to fill in pull request title with a template. [#5901](https://github.com/gogs/gogs/pull/5901) - Able to override static files under `public/` directory, please refer to [documentation](https://gogs.io/docs/features/custom_template) for usage. [#5920](https://github.com/gogs/gogs/pull/5920) - New API endpoint `GET /admin/teams/:teamid/members` to list members of a team. [#5877](https://github.com/gogs/gogs/issues/5877) +- Support backup with retention policy for Docker deployments. [#6140](https://github.com/gogs/gogs/pull/6140) ### Changed @@ -38,7 +38,7 @@ COPY --from=binarybuilder /gogs.io/gogs/gogs . RUN ./docker/finalize.sh # Configure Docker Container -VOLUME ["/data"] +VOLUME ["/data", "/backup"] EXPOSE 22 3000 ENTRYPOINT ["/app/gogs/docker/start.sh"] CMD ["/bin/s6-svscan", "/app/gogs/docker/s6/"] diff --git a/docker/Dockerfile.aarch64 b/docker/Dockerfile.aarch64 index 730b7f87..167eb102 100644 --- a/docker/Dockerfile.aarch64 +++ b/docker/Dockerfile.aarch64 @@ -38,7 +38,7 @@ COPY --from=binarybuilder /gogs.io/gogs/gogs . RUN ./docker/finalize.sh # Configure Docker Container -VOLUME ["/data"] +VOLUME ["/data", "/backup"] EXPOSE 22 3000 ENTRYPOINT ["/app/gogs/docker/start.sh"] CMD ["/bin/s6-svscan", "/app/gogs/docker/s6/"] diff --git a/docker/Dockerfile.rpi b/docker/Dockerfile.rpi index b1c61b0e..e2dabcc3 100644 --- a/docker/Dockerfile.rpi +++ b/docker/Dockerfile.rpi @@ -38,7 +38,7 @@ COPY --from=binarybuilder /gogs.io/gogs/gogs . RUN ./docker/finalize.sh # Configure Docker Container -VOLUME ["/data"] +VOLUME ["/data", "/backup"] EXPOSE 22 3000 ENTRYPOINT ["/app/gogs/docker/start.sh"] CMD ["/bin/s6-svscan", "/app/gogs/docker/s6/"] diff --git a/docker/README.md b/docker/README.md index 30542316..51489706 100644 --- a/docker/README.md +++ b/docker/README.md @@ -102,6 +102,44 @@ This container have some options available via environment variables, these opti `false` - <u>Action:</u> Request crond to be run inside the container. Its default configuration will periodically run all scripts from `/etc/periodic/${period}` but custom crontabs can be added to `/var/spool/cron/crontabs/`. +- **BACKUP_INTERVAL**: + - <u>Possible value:</u> + `3h`, `7d`, `3M` + - <u>Default:</u> + `null` + - <u>Action:</u> + In combination with `RUN_CROND` set to `true`, enables backup system.\ + See: [Backup System](#backup-system) +- **BACKUP_RETENTION**: + - <u>Possible value:</u> + `360m`, `7d`, `...m/d` + - <u>Default:</u> + `7d` + - <u>Action:</u> + Used by backup system. Backups older than specified in expression are deleted periodically.\ + See: [Backup System](#backup-system) +- **BACKUP_ARG_CONFIG**: + - <u>Possible value:</u> + `/app/gogs/example/custom/config` + - <u>Default:</u> + `null` + - <u>Action:</u> + Used by backup system. If defined, supplies `--config` argument to `gogs backup`.\ + See: [Backup System](#backup-system) +- **BACKUP_ARG_EXCLUDE_REPOS**: + - <u>Possible value:</u> + `test-repo1`, `test-repo2` + - <u>Default:</u> + `null` + - <u>Action:</u> + Used by backup system. If defined, supplies `--exclude-repos` argument to `gogs backup`.\ + See: [Backup System](#backup-system) + +## Backup System +Automated backups with retention policy: + +- `BACKUP_INTERVAL` controls how often the backup job runs and supports interval in hours (h), days (d), and months (M), eg. `3h`, `7d`, `3M`. The lowest possible value is one hour (`1h`). +- `BACKUP_RETENTION` supports expressions in minutes (m) and days (d), eg. `360m`, `2d`. The lowest possible value is 60 minutes (`60m`). ## Upgrade diff --git a/docker/runtime/backup-init.sh b/docker/runtime/backup-init.sh new file mode 100644 index 00000000..07298c5d --- /dev/null +++ b/docker/runtime/backup-init.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +set -e + +BACKUP_PATH="/backup" + +# Make sure that required directories exist +mkdir -p "${BACKUP_PATH}" +mkdir -p "/etc/crontabs" +chown git:git /backup +chmod 2770 /backup + +# [string] BACKUP_INTERVAL Period expression +# [string] BACKUP_RETENTION Period expression +if [ -z "${BACKUP_INTERVAL}" ]; then + echo "Backup disabled: BACKUP_INTERVAL has not been found" 1>&2 + exit 1 +fi + +if [ -z "${BACKUP_RETENTION}" ]; then + echo "Backup retention period is not defined, default to 7 days" 1>&2 + BACKUP_RETENTION='7d' +fi + +# Parse BACKUP_INTERVAL environment variable and generate appropriate cron expression. Backup cron task will be run as scheduled. +# Expected format: nu (n - number, u - unit) (eg. 3d means 3 days) +# Supported units: h - hours, d - days, M - months +parse_generate_cron_expression() { + CRON_EXPR_MINUTES="*" + CRON_EXPR_HOURS="*" + CRON_EXPR_DAYS="*" + CRON_EXPR_MONTHS="*" + + TIME_INTERVAL=$(echo "${BACKUP_INTERVAL}" | sed -e 's/[hdM]$//') + TIME_UNIT=$(echo "${BACKUP_INTERVAL}" | sed -e 's/^[0-9]\+//') + + if [ "${TIME_UNIT}" = "h" ]; then + if [ ! "${TIME_INTERVAL}" -le 23 ]; then + echo "Parse error: Time unit 'h' (hour) cannot be greater than 23" 1>&2 + exit 1 + fi + + CRON_EXPR_MINUTES=0 + CRON_EXPR_HOURS="*/${TIME_INTERVAL}" + elif [ "${TIME_UNIT}" = "d" ]; then + if [ ! "${TIME_INTERVAL}" -le 30 ]; then + echo "Parse error: Time unit 'd' (day) cannot be greater than 30" 1>&2 + exit 1 + fi + + CRON_EXPR_MINUTES=0 + CRON_EXPR_HOURS=0 + CRON_EXPR_DAYS="*/${TIME_INTERVAL}" + elif [ "${TIME_UNIT}" = "M" ]; then + if [ ! "${TIME_INTERVAL}" -le 12 ]; then + echo "Parse error: Time unit 'M' (month) cannot be greater than 12" 1>&2 + exit 1 + fi + + CRON_EXPR_MINUTES=0 + CRON_EXPR_HOURS=0 + CRON_EXPR_DAYS="1" + CRON_EXPR_MONTHS="*/${TIME_INTERVAL}" + else + echo "Parse error: BACKUP_INTERVAL expression is invalid" 1>&2 + exit 1 + fi + + echo "${CRON_EXPR_MINUTES} ${CRON_EXPR_HOURS} ${CRON_EXPR_DAYS} ${CRON_EXPR_MONTHS} *" +} + +# Parse BACKUP_RETENTION environment variable and generate appropriate find command expression. +# Expected format: nu (n - number, u - unit) (eg. 3d means 3 days) +# Supported units: m - minutes, d - days +parse_generate_retention_expression() { + FIND_TIME_EXPR='mtime' + + TIME_INTERVAL=$(echo "${BACKUP_RETENTION}" | sed -e 's/[mhdM]$//') + TIME_UNIT=$(echo "${BACKUP_RETENTION}" | sed -e 's/^[0-9]\+//') + + if [ "${TIME_UNIT}" = "m" ]; then + if [ "${TIME_INTERVAL}" -le 59 ]; then + echo "Warning: Minimal retention is 60m. Value set to 60m" 1>&2 + TIME_INTERVAL=60 + fi + + FIND_TIME_EXPR="mmin" + elif [ "${TIME_UNIT}" = "h" ]; then + echo "Error: Unsupported expression - Try: eg. 120m for 2 hours." 1>&2 + exit 1 + elif [ "${TIME_UNIT}" = "d" ]; then + FIND_TIME_EXPR="mtime" + elif [ "${TIME_UNIT}" = "M" ]; then + echo "Error: Unsupported expression - Try: eg. 60d for 2 months." 1>&2 + exit 1 + else + echo "Parse error: BACKUP_RETENTION expression is invalid" 1>&2 + exit 1 + fi + + echo "${FIND_TIME_EXPR} +${TIME_INTERVAL:-7}" +} + +add_backup_cronjob() { + CRONTAB_USER="${1:-git}" + CRONTAB_FILE="/etc/crontabs/${CRONTAB_USER}" + CRONJOB_EXPRESSION="${2:-}" + CRONJOB_EXECUTOR="${3:-}" + CRONJOB_EXECUTOR_ARGUMENTS="${4:-}" + CRONJOB_TASK="${CRONJOB_EXPRESSION} /bin/sh ${CRONJOB_EXECUTOR} ${CRONJOB_EXECUTOR_ARGUMENTS}" + + if [ -f "${CRONTAB_FILE}" ]; then + CRONJOB_EXECUTOR_COUNT=$(grep -c "${CRONJOB_EXECUTOR}" "${CRONTAB_FILE}" || exit 0) + if [ "${CRONJOB_EXECUTOR_COUNT}" != "0" ]; then + echo "Cron job already exists for ${CRONJOB_EXECUTOR}. Updating existing." 1>&2 + CRONJOB_TASK=$(echo "{CRONJOB_TASK}" | sed 's/\//\\\//g' ) + CRONJOB_EXECUTOR=$(echo "{CRONJOB_EXECUTOR}" | sed 's/\//\\\//g' ) + sed -i "/${CRONJOB_EXECUTOR}/c\\${CRONJOB_TASK}" "${CRONTAB_FILE}" + return 0 + fi + fi + + # Finally append new line with cron task expression + echo "${CRONJOB_TASK}" >>"${CRONTAB_FILE}" +} + +CRONTAB_USER=$(awk -v val="${PUID}" -F ":" '$3==val{print $1}' /etc/passwd) + +# Up to this point, it was desirable that interpreter handles the command errors and halts execution upon any error. +# From now, we handle the errors our self. +set +e +RETENTION_EXPRESSION="$(parse_generate_retention_expression)" + +if [ -z "${RETENTION_EXPRESSION}" ]; then + echo "Couldn't generate backup retention expression. Aborting backup setup" 1>&2 + exit 1 +fi + +# Backup rotator cron will run every 5 minutes +add_backup_cronjob "${CRONTAB_USER}" "*/5 * * * *" "/app/gogs/docker/runtime/backup-rotator.sh" "'${BACKUP_PATH}' '${RETENTION_EXPRESSION}'" +add_backup_cronjob "${CRONTAB_USER}" "$(parse_generate_cron_expression)" "/app/gogs/docker/runtime/backup-job.sh" "'${BACKUP_PATH}'" diff --git a/docker/runtime/backup-job.sh b/docker/runtime/backup-job.sh new file mode 100644 index 00000000..2b8efd70 --- /dev/null +++ b/docker/runtime/backup-job.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env sh + +execute_backup_job() { + BACKUP_ARG_PATH="${1:-}" + BACKUP_ARG_CONFIG="${BACKUP_ARG_CONFIG:-}" + BACKUP_ARG_EXCLUDE_REPOS="${BACKUP_ARG_EXCLUDE_REPOS:-}" + cd "/app/gogs" || exit 1 + + BACKUP_ARGS="--target=${BACKUP_ARG_PATH}" + + if [ -n "${BACKUP_ARG_CONFIG}" ]; then + BACKUP_ARGS="${BACKUP_ARGS} --config=${BACKUP_ARG_CONFIG}" + fi + + if [ -n "${BACKUP_ARG_EXCLUDE_REPOS}" ]; then + BACKUP_ARGS="${BACKUP_ARGS} --exclude-repos=${BACKUP_ARG_EXCLUDE_REPOS}" + fi + + ./gogs backup "${BACKUP_ARGS}" || echo "Error: Backup job returned non-successful code." && exit 1 +} + +main() { + BACKUP_PATH="${1:-}" + + if [ -z "${BACKUP_PATH}" ]; then + echo "Required argument missing BACKUP_PATH" 1>&2 + exit 1 + fi + + execute_backup_job "${BACKUP_PATH}" +} + +main "$@" diff --git a/docker/runtime/backup-rotator.sh b/docker/runtime/backup-rotator.sh new file mode 100644 index 00000000..51665f57 --- /dev/null +++ b/docker/runtime/backup-rotator.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env sh + +# This is very simple, yet effective backup rotation script. +# Using find command, all files that are older than BACKUP_RETENTION_DAYS are accumulated and deleted using rm. +main() { + BACKUP_PATH="${1:-}" + FIND_EXPRESSION="${2:-mtime +7}" + + if [ -z "${BACKUP_PATH}" ]; then + echo "Error: Required argument missing BACKUP_PATH" 1>&2 + exit 1 + fi + + if [ "$(realpath "${BACKUP_PATH}")" = "/" ]; then + echo "Error: Dangerous BACKUP_PATH: /" 1>&2 + exit 1 + fi + + if [ ! -d "${BACKUP_PATH}" ]; then + echo "Error: BACKUP_PATH does't exist or is not a directory" 1>&2 + exit 1 + fi + + find "${BACKUP_PATH}/" -type f -name "gogs-backup-*.zip" -${FIND_EXPRESSION} -print -exec rm "{}" + +} + +main "$@" diff --git a/docker/start.sh b/docker/start.sh index 017a1099..77418e18 100755 --- a/docker/start.sh +++ b/docker/start.sh @@ -64,6 +64,7 @@ CROND=$(echo "$RUN_CROND" | tr '[:upper:]' '[:lower:]') if [ "$CROND" = "true" -o "$CROND" = "1" ]; then echo "init:crond | Cron Daemon (crond) will be run as requested by s6" 1>&2 rm -f /app/gogs/docker/s6/crond/down + /bin/sh /app/gogs/docker/runtime/backup-init.sh "${PUID}" else # Tell s6 not to run the crond service touch /app/gogs/docker/s6/crond/down |