OrochW's Blog

首页 归档 标签 关于

kubernetes学习之etcd备份同步恢复

2022-07-20 9 min read # 云原生


etcd是一种高度一致的分布式键值存储，它提供了一种可靠的方式来存储需要由分布式系统或机器集群访问的数据。它在网络分区期间优雅地处理领导者选举，并且可以容忍机器故障，即使在领导者节点中也是如此。更多参考etcd文档

etcd的备份

1.开始备份之前，先来查看下etcd数据

[root@master-etcd1-234-31 etcd_backup_dir]# cat /etc/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd
ExecStart=/usr/bin/etcd \
  --name=etcd-192.168.234.31 \
  --cert-file=/etc/kubernetes/ssl/etcd.pem \
  --key-file=/etc/kubernetes/ssl/etcd-key.pem \
  --peer-cert-file=/etc/kubernetes/ssl/etcd.pem \
  --peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \
  --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --initial-advertise-peer-urls=https://192.168.234.31:2380 \
  --listen-peer-urls=https://192.168.234.31:2380 \
  --listen-client-urls=https://192.168.234.31:2379,http://127.0.0.1:2379 \
  --advertise-client-urls=https://192.168.234.31:2379 \
  --initial-cluster-token=etcd-cluster-0 \
  --initial-cluster=etcd-192.168.234.31=https://192.168.234.31:2380,etcd-192.168.234.32=https://192.168.234.32:2380,etcd-192.168.234.33=https://192.168.234.33:2380 \
  --initial-cluster-state=new \
  --data-dir=/var/lib/etcd \ ##当前的etcd的目录
  --wal-dir= \
  --snapshot-count=50000 \
  --auto-compaction-retention=1 \
  --auto-compaction-mode=periodic \
  --max-request-bytes=10485760 \
  --quota-backend-bytes=8589934592
Restart=always
RestartSec=15
LimitNOFILE=65536
OOMScoreAdjust=-999

[Install]
WantedBy=multi-user.target

2.执行etcd集群数据备份

在etcd集群的其中一个节点执行备份操作，然后将备份文件拷贝到其他节点上。
先在etcd集群的每个节点上创建备份目录

mkdir -p /data/etcd_backup_dir

在etcd集群其中个一个节点（这里在k8s-master01）上执行备份：

[root@master-etcd1-234-31 ~]# mkdir -p /data/etcd_backup_dir
[root@master-etcd1-234-31 ~]# etcdctl  \
> snapshot  save /data/etcd_backup_dir/etcd-snapshot-`date +%Y%m%d`.db \
> --endpoints=https://192.168.234.31:2379 \
> --cacert=/etc/kubernetes/ssl/ca.pem \
> --cert=/etc/kubernetes/ssl/etcd.pem \
> --key=/etc/kubernetes/ssl/etcd-key.pem
{"level":"info","ts":"2022-07-19T01:07:55.810-0400","caller":"snapshot/v3_snapshot.go:65","msg":"created temporary db file","path":"/data/etcd_backup_di                       r/etcd-snapshot-20220719.db.part"}
{"level":"info","ts":"2022-07-19T01:07:55.815-0400","logger":"client","caller":"v3/maintenance.go:211","msg":"opened snapshot stream; downloading"}
{"level":"info","ts":"2022-07-19T01:07:55.815-0400","caller":"snapshot/v3_snapshot.go:73","msg":"fetching snapshot","endpoint":"https://192.168.234.31:2                       379"}
{"level":"info","ts":"2022-07-19T01:07:55.837-0400","logger":"client","caller":"v3/maintenance.go:219","msg":"completed snapshot read; closing"}
{"level":"info","ts":"2022-07-19T01:07:55.840-0400","caller":"snapshot/v3_snapshot.go:88","msg":"fetched snapshot","endpoint":"https://192.168.234.31:23                       79","size":"2.5 MB","took":"now"}
{"level":"info","ts":"2022-07-19T01:07:55.840-0400","caller":"snapshot/v3_snapshot.go:97","msg":"saved","path":"/data/etcd_backup_dir/etcd-snapshot-2022                       0719.db"}
Snapshot saved at /data/etcd_backup_dir/etcd-snapshot-20220719.db

可以将上面k8s-master01节点的etcd备份命令放在脚本里，结合crontab进行定时备份：

[root@master-etcd1-234-31 ~]cat /data/etcd_backup_dir/etcd_backup.sh
#!/usr/bin/bash
 
DATE=`date +%Y-%m-%d-%H-%M-%S`
ETCDCTL_API=3 /opt/kube/bin/etcdctl  \
snapshot  save /data/etcd_backup_dir/etcd-snapshot-$DATE.db \
--endpoints=https://192.168.234.31:2379 \
--cacert=/etc/kubernetes/ssl/ca.pem \
--cert=/etc/kubernetes/ssl/etcd.pem \
--key=/etc/kubernetes/ssl/etcd-key.pem
 
# 备份保留30天
find /data/etcd_backup_dir/ -name "*.db" -mtime +30 -exec rm -f {} \;
 
# 同步到其他两个etcd节点
scp /data/etcd_backup_dir/etcd-snapshot-$DATE.db 192.168.234.32:/data/etcd_backup_dir/
scp /data/etcd_backup_dir/etcd-snapshot-$DATE.db 192.168.234.33:/data/etcd_backup_dir

设置crontab定时备份任务，每天凌晨4点执行备份：

[root@master-etcd1-234-31 ~]# chmod 755 /data/etcd_backup_dir/etcd_backup.sh
[root@master-etcd1-234-31 ~]# crontab -e
crontab: installing new crontab
[root@master-etcd1-234-31 ~]# crontab -l
*/10 * * * * ntpdate time.windows.com
0 4 * * * /bin/bash -x /data/etcd_backup_dir/etcd_backup.sh > /dev/null 2>&1

恢复

[root@master-etcd1-234-31 etcd_backup_dir]# pwd
/data/etcd_backup_dir
[root@master-etcd1-234-31 etcd_backup_dir]# mkdir /data/opt/etcddir -p
[root@master-etcd1-234-31 etcd_backup_dir]# etcdctl  snapshot restore etcd-snapshot-20220719.db --data-dir=/data/opt/etcddir/ #将数据恢复到一个新的目录中
Deprecated: Use `etcdutl snapshot restore` instead.

2022-07-21T16:53:50+08:00       info    snapshot/v3_snapshot.go:248     restoring snapshot      {"path": "etcd-snapshot-20220719.db", "wal-dir": "/data/opt/etcddir/member/wal", "data-dir": "/data/opt/etcddir/", "snap-dir": "/data/opt/etcddir/member/snap", "stack": "go.etcd.io/etcd/etcdutl/v3/snapshot.(*v3Manager).Restore\n\t/go/src/go.etcd.io/etcd/release/etcd/etcdutl/snapshot/v3_snapshot.go:254\ngo.etcd.io/etcd/etcdutl/v3/etcdutl.SnapshotRestoreCommandFunc\n\t/go/src/go.etcd.io/etcd/release/etcd/etcdutl/etcdutl/snapshot_command.go:147\ngo.etcd.io/etcd/etcdctl/v3/ctlv3/command.snapshotRestoreCommandFunc\n\t/go/src/go.etcd.io/etcd/release/etcd/etcdctl/ctlv3/command/snapshot_command.go:129\ngithub.com/spf13/cobra.(*Command).execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:856\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:960\ngithub.com/spf13/cobra.(*Command).Execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:897\ngo.etcd.io/etcd/etcdctl/v3/ctlv3.Start\n\t/go/src/go.etcd.io/etcd/release/etcd/etcdctl/ctlv3/ctl.go:107\ngo.etcd.io/etcd/etcdctl/v3/ctlv3.MustStart\n\t/go/src/go.etcd.io/etcd/release/etcd/etcdctl/ctlv3/ctl.go:111\nmain.main\n\t/go/src/go.etcd.io/etcd/release/etcd/etcdctl/main.go:59\nruntime.main\n\t/go/gos/go1.16.15/src/runtime/proc.go:225"}
2022-07-21T16:53:50+08:00       info    membership/store.go:141 Trimming membership information from the backend...
2022-07-21T16:53:50+08:00       info    membership/cluster.go:421       added member    {"cluster-id": "cdf818194e3a8c32", "local-member-id": "0", "added-peer-id": "8e9e05c52164694d", "added-peer-peer-urls": ["http://localhost:2380"]}
2022-07-21T16:53:50+08:00       info    snapshot/v3_snapshot.go:269     restored snapshot       {"path": "etcd-snapshot-20220719.db", "wal-dir": "/data/opt/etcddir/member/wal", "data-dir": "/data/opt/etcddir/", "snap-dir": "/data/opt/etcddir/member/snap"}
[root@master-etcd1-234-31 etcd_backup_dir]#
Powered by Gridea RSS