一、备份方式

区分环境建立好git分支
编写备份脚本、通知脚本
编写流水线yml文件
1. 代码克隆
2. 设置tag
3. docker build
4. 制作Helm chart
5. 部署到集群

Dockerfile

FROM alpine:3.8
ENV LANG C.UTF-8

WORKDIR /

COPY docker-entrypoint.sh /
COPY send-message.sh /

RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \
    apk -v --update add --no-cache \
    bash \
    bzip2 \
    curl \
    mariadb-client \
    openssh-client \
    pv && \
    rm -f /var/cache/apk/* && \
    mkdir -p /var/lib/mysql/backup && \
    mkdir /var/lib/mysql/log && \
    wget https://gosspublic.alicdn.com/ossutil/1.7.14/ossutil64 && \
    chmod 755 ossutil64 && mv ossutil64 /usr/bin/ && \
    chmod +x docker-entrypoint.sh send-message.sh
#ENTRYPOINT ["docker-entrypoint.sh"]

备份脚本

常用参数说明

-x | --lock-all-tables：为所有表加上一个持续到dump结束的全局读锁。该选项在dump阶段仅加一次锁，一锁锁永久且锁所有。该选项自动禁用--lock-tables和--single-transaction选项。
-R | --routines
--flush-logs：导出存储过程和函数。但不会导出它们的属性值
-B | --databases：
-A | --all-databases：
-r | --result-file：
-d | --no-data：不导出表数据，可以用在仅导出表结构的情况

$ mysqldump -u${DB_USER} -p${DB_PASS} -h${DB_HOST} -P${DB_PORT} \
--lock-all-tables \
--flush-logs \
--databases "${db}" \
--log-error=${LOG_PATH}/luna_dump_error_log \
--flush-logs \
--result-file=${DUMP_PATH}/"${db}".sql

完整脚本

#!/bin/bash
set -x
#
# env vars
#
# database vars
DB_HOST=${DB_HOST}
DB_USER=${DB_USER}
DB_PASS=${DB_PASS}
DB_PORT=${DB_PORT}
# aliyun oss vars
OSS_BUCKET=${OSS_BUCKET}
OSS_ENDPOINT=${OSS_ENDPOINT}
OSS_ACCESSKEY_ID=${OSS_ACCESSKEY_ID}
OSS_ACCESSKEY_SECRET=${OSS_ACCESSKEY_SECRET}
# other vars
DUMP_PATH=/var/lib/mysql/backup
LOG_PATH=/var/lib/mysql/log
MISSING=false
BAK_NAME=luna_$(date +%Y_%m_%d_%H_%M)
ALL_DATABASES=$(mysql -u"${DB_USER}" -p"${DB_PASS}" -h"${DB_HOST}" -P"${DB_PORT}" -e "SHOW DATABASES;" | tr -d "| " | grep -Ev '(Database|information_schema|performance_schema|mysql|backup|sys)')

#
# validate all env vars have been provided and error on missing env vars
#
if [[ $ALL_DATABASES == "" ]]; then
    if [[ $DB_NAME == "" ]]; then
        echo "Missing DB_NAME and ALL_DATABASES has not been set"
        MISSING=true
    fi
fi

if [[ $DB_HOST == "" ]]; then
    echo "Missing DB_HOST"
    MISSING=true
fi

if [[ $DB_USER == "" ]]; then
    echo "Missing DB_USER"
    MISSING=true
fi

if [[ $DB_PASS == "" ]]; then
    echo "Missing DB_PASS"
    MISSING=true
fi

if [[ ${OSS_ACCESSKEY_ID} == "" ]]; then
    echo "Missing OSS_ACCESSKEY_ID"
    MISSING=true
fi

if [[ ${OSS_ACCESSKEY_SECRET} == "" ]]; then
    echo "Missing OSS_ACCESSKEY_SECRET"
    MISSING=true
fi

if [[ ${OSS_BUCKET} == "" ]]; then
    echo "Missing OSS_BUCKET"
    MISSING=true
fi

if [[ ${OSS_ENDPOINT} == "" ]]; then
    echo "Missing OSS_ENDPOINT"
    MISSING=true
fi
#
#Dump database
#
DB_DUMP() {
    #
    # dump all databases
    #
    echo "Dumping all databases"
    for db in $ALL_DATABASES; do
        echo "begin to dump ${db}......"

        # shellcheck disable=SC2086
        mysqldump -u${DB_USER} -p${DB_PASS} -h${DB_HOST} -P${DB_PORT} \
            --lock-all-tables \
            --flush-logs \
            --databases "${db}" \
            --routines \
            --log-error=${LOG_PATH}/luna_dump_error.log \
            --result-file=${DUMP_PATH}/"${db}".sql
    done
}
#
#Upload to aliyun oss
#
UPLOAD_TO_OSS() {
    cd ${DUMP_PATH}
    #tar -cvf - ./*.sql | pv -s $(($(du -sk ./*.sql | awk '{print $1}') * 1024)) | gzip >${backup_name}.tar.gz
    tar -cvf - ./*.sql | pv | gzip >"${BAK_NAME}.tar.gz"
    rm -f ${DUMP_PATH}/backup/*.sql
    ossutil64 cp \
        "${BAK_NAME}".tar.gz \
        oss://"${OSS_BUCKET}"/mysql/backup/"${BAK_NAME}".tar.gz \
        -f \
        -e \
        "${OSS_ENDPOINT}" \
        -i "${OSS_ACCESSKEY_ID}" \
        -k "${OSS_ACCESSKEY_SECRET}" \
        >>${LOG_PATH}/oss.log
    rm -f "${BAK_NAME}".tar.gz
    echo "[PATH = oss://"${OSS_BUCKET}"/mysql/backup/]" >>${LOG_PATH}/oss.log
    echo "[Succeed. Backup Name = "${BAK_NAME}".tar.gz]" >>${LOG_PATH}/oss.log
}

#
#dump all databases
#
if [[ $MISSING == "true" ]]; then
    echo "Need to provide missing env vars, exiting now."
    exit 1
else
    echo "Checked."
    DB_DUMP
    if [ $? != 0 ]; then
        if [ -s /var/lib/mysql/log/luna_dump_error.log ]; then
            echo "dump failed."
            /failed.sh
        else
            UPLOAD_TO_OSS
            /succeed.sh
        fi
    else
        UPLOAD_TO_OSS
        /succeed.sh
    fi
fi

二、CronJob

关注点

schedule：每日6:30分开始备份，耗时约两分钟。

$ kubectl get cronjobs.batch -n luna-system
NAME         SCHEDULE     SUSPEND   ACTIVE   LAST SCHEDULE   AGE
mysql-dump   30 6 * * *   False     0        4h43m           35h

ttlSecondsAfterFinished：Job完成后保留2天

$ kubectl get jobs.batch -n luna-system | grep mysql
mysql-dump-1668378600   1/1           105s       41h
mysql-dump-1668465000   1/1           101s       27h
mysql-dump-1668551400   1/1           103s       3h4m

restartPolicy: Job失败重试
backoffLimit：限制重试次数为3
nodeName: 指定调度有公网的k8s节点

CronJob YAML示例

apiVersion: batch/v1beta1
kind: CronJob
metadata:
  namespace: luna-system
  name: mysql-dump
spec:
  schedule: "30 1 * * *"
  concurrencyPolicy: Replace
  jobTemplate:
    spec:
      ttlSecondsAfterFinished: 172800
      backoffLimit: 3
      template:
        spec:
          restartPolicy: OnFailure
          nodeName: demo-k8s-02
          containers:
            - name: mysql-dump
              image: registry.cn-hangzhou.aliyuncs.com/luna/mysqldump:v1.0
              env:
                - name: DB_USER
                  value: root
                - name: DB_HOST
                  value: 192.168.1.1
                - name: DB_PASS
                  value: XXX
                - name: DB_PORT
                  value: "3306"
                - name: OSS_BUCKET
                  value: luna-backup
                - name: OSS_ENDPOINT
                  value: oss-cn-hangzhou-internal.aliyuncs.com
                - name: OSS_ACCESSKEY_ID
                  value: XXX
                - name: OSS_ACCESSKEY_SECRET
                  value: XXX
              command: ["/bin/bash", "-c", "/docker-entrypoint.sh"]
              volumeMounts:
                - name: host-time
                  mountPath: /etc/localtime
                  readOnly: true
          volumes:
            - name: host-time
              hostPath:
                path: /etc/localtime

三、Job日志查看

$ kubectl get cronjobs.batch -n luna-system
NAME         SCHEDULE     SUSPEND   ACTIVE   LAST SCHEDULE   AGE
mysql-dump   30 6 * * *   False     1        9h              16h

$ kubectl get pod -A | grep mysql-dump
luna-system          mysql-dump-1668360600-v875r                          0/1     Completed   0          12m

$ kubectl logs mysql-dump-1668360600-v875r -n luna-system

四、备份结果通知

需部署到可访问公网的worker节点备份成功：执行succeed.sh 发送${LOG_PATH}/oss.log 内容到@用户

备份失败：执行failed.sh 发送${LOG_PATH}/luna_dump_error.log 内容到@用户

七月

七月

Kubernetes CronJob备份集群中MySQL数据

一、备份方式

Dockerfile

备份脚本

二、CronJob

关注点

CronJob YAML示例

三、Job日志查看

四、备份结果通知

分享