记忆角落

  • {{ item.name }}
  • k8s部署redis一主两从三哨兵
  • 部署清单
  • namespace
  • configmap
  • Secret
  • rabc
  • service
  • statefulSet
  • 执行完毕后
  • 查看节点集群状态
  • 参考
  • 首页
  • 关于
  • 归档
  • 邻居
  • 捐赠
  • 速查表

k8s部署redis一主两从三哨兵

  • 郭良俊只狗
  • 2024-06-19
  • 0

k8s部署redis一主两从三哨兵

部署清单

  1. 一共6个容器，1个主节点、2个从节点、3个哨兵节点。
  2. 编写namespace脚本，创建专门的namespace
  3. 编写configmap脚本，分别将主节点、从节点、哨兵的配置写在里面
  4. 编写secret脚本，将redis的密码保存在里面
  5. 编写RABC脚本，sts使用
  6. 编写service脚本，1个提供redis之间的网络，1个提供哨兵之间的网络，1个对外提供服务
  7. 编写statefulSet脚本，1个提供redis主从服务和哨兵

pvc是使用nfs的磁盘。如有ceph可自行选择

存在问题：

1.Kubernetes 部署 Redis主从+Sentinel模式本地无法连接到pod ip，因为技术代码需要连Sentinel后获取到所有ip，但是ip信息都是k8s pod ip，这暂时还未能解决本地连接

namespace

apiVersion: v1
kind: Namespace
metadata:
  name: redis-cluster
  labels:
    app: redis

configmap

apiVersion: v1
kind: ConfigMap
metadata:
  name: redis-configmap
  namespace: redis-cluster
  labels:
    app: redis
data:
  #这里定义了多个数据信息
  master.conf: |
    # Master配置
    requirepass redisPassword
    masterauth redisPassword
    bind 0.0.0.0
    daemonize no
    protected-mode yes
    port 6379
    tcp-backlog 2048
    timeout 0
    tcp-keepalive 300
    pidfile /var/run/redis_6379.pid
    loglevel notice
    #logfile /data/redis.log
    databases 16
    always-show-logo no
    stop-writes-on-bgsave-error yes
    set-proc-title yes
    proc-title-template "{title} {listen-addr} {server-mode}"
    rdbcompression yes
    rdbchecksum yes
    dbfilename dump.rdb
    rdb-del-sync-files no
    dir /data
    replica-serve-stale-data yes
    replica-read-only no
    repl-diskless-sync no
    repl-diskless-sync-delay 5
    repl-diskless-load disabled
    repl-disable-tcp-nodelay no
    replica-priority 100
    acllog-max-len 128
    lazyfree-lazy-eviction no
    lazyfree-lazy-expire no
    lazyfree-lazy-server-del no
    replica-lazy-flush no
    lazyfree-lazy-user-del no
    lazyfree-lazy-user-flush no
    oom-score-adj no
    oom-score-adj-values 0 200 800
    disable-thp yes
    appendonly no
    appendfsync everysec
    no-appendfsync-on-rewrite no
    auto-aof-rewrite-percentage 100
    auto-aof-rewrite-min-size 64mb
    aof-load-truncated yes
    aof-use-rdb-preamble yes
    lua-time-limit 5000
    slowlog-log-slower-than 10000
    slowlog-max-len 128
    latency-monitor-threshold 0
    notify-keyspace-events ""
    hash-max-ziplist-entries 512
    hash-max-ziplist-value 64
    list-max-ziplist-size -2
    list-compress-depth 0
    set-max-intset-entries 512
    zset-max-ziplist-entries 128
    zset-max-ziplist-value 64
    hll-sparse-max-bytes 3000
    stream-node-max-bytes 4096
    stream-node-max-entries 100
    activerehashing yes
    client-output-buffer-limit normal 0 0 0
    client-output-buffer-limit replica 256mb 64mb 60
    client-output-buffer-limit pubsub 32mb 8mb 60
    hz 10
    dynamic-hz yes
    aof-rewrite-incremental-fsync yes
    rdb-save-incremental-fsync yes
    jemalloc-bg-thread yes
  slave.conf: |
    # Slave配置 注意service名称
    replicaof redis-cluster-0.redis-headless 6379
    masterauth redisPassword
    requirepass redisPassword
    replica-read-only yes
    bind 0.0.0.0
    daemonize no
    protected-mode yes
    port 6379
    tcp-backlog 511
    timeout 0
    tcp-keepalive 300
    pidfile /var/run/redis_6379.pid
    loglevel notice
    #logfile /data/redis.log
    databases 16
    always-show-logo no
    set-proc-title yes
    proc-title-template "{title} {listen-addr} {server-mode}"
    stop-writes-on-bgsave-error yes
    rdbcompression yes
    rdbchecksum yes
    dbfilename dump.rdb
    rdb-del-sync-files no
    dir /data
    replica-serve-stale-data yes
    repl-diskless-sync no
    repl-diskless-sync-delay 5
    repl-diskless-load disabled
    repl-disable-tcp-nodelay no
    replica-priority 100
    acllog-max-len 128
    lazyfree-lazy-eviction no
    lazyfree-lazy-expire no
    lazyfree-lazy-server-del no
    replica-lazy-flush no
    lazyfree-lazy-user-del no
    lazyfree-lazy-user-flush no
    oom-score-adj no
    oom-score-adj-values 0 200 800
    disable-thp yes
    appendonly no
    appendfsync everysec
    no-appendfsync-on-rewrite no
    auto-aof-rewrite-percentage 100
    auto-aof-rewrite-min-size 64mb
    aof-load-truncated yes
    aof-use-rdb-preamble yes
    lua-time-limit 5000
    slowlog-log-slower-than 10000
    slowlog-max-len 128
    latency-monitor-threshold 0
    notify-keyspace-events ""
    hash-max-ziplist-entries 512
    hash-max-ziplist-value 64
    list-max-ziplist-size -2
    list-compress-depth 0
    set-max-intset-entries 512
    zset-max-ziplist-entries 128
    zset-max-ziplist-value 64
    hll-sparse-max-bytes 3000
    stream-node-max-bytes 4096
    stream-node-max-entries 100
    activerehashing yes
    client-output-buffer-limit normal 0 0 0
    client-output-buffer-limit replica 256mb 64mb 60
    client-output-buffer-limit pubsub 32mb 8mb 60
    hz 10
    dynamic-hz yes
    aof-rewrite-incremental-fsync yes
    rdb-save-incremental-fsync yes
    jemalloc-bg-thread yes
  sentinel.conf: |
    # 哨兵配置
    # 哨兵sentinel监控的redis主节点的 ip port
    # master-name 可以自己命名的主节点名字 只能由字母A-z、数字0-9 、这三个字符".-_"组成。
    # quorum 配置多少个sentinel哨兵统一认为master主节点失联 那么这时客观上认为主节点失联了
    # sentinel monitor <master-name> <ip> <redis-port> <quorum>
    port 26379
    #bind 0.0.0.0
    daemonize no
    pidfile "/var/run/redis-sentinel.pid"
    #logfile /data/redis-sentinel.log
    dir "/data"
    # 注意service名称
    sentinel monitor mymaster redis-cluster-0.redis-headless 6379 2
    sentinel auth-pass mymaster redisPassword
    sentinel down-after-milliseconds mymaster 30000
    sentinel parallel-syncs mymaster 1
    sentinel failover-timeout mymaster 180000
    acllog-max-len 128
    sentinel deny-scripts-reconfig yes
    sentinel resolve-hostnames yes
    sentinel announce-hostnames no
    protected-mode no
    user default on nopass sanitize-payload ~* &* +@all

Secret

apiVersion: v1
kind: Secret
metadata:
  name: redis-secret
  namespace: redis-cluster
  labels:
    app: redis
# Opaque 类型的数据是一个 map 类型，要求value是base64编码。

type: Opaque
data:
  redisPassword: MTIzNDU2 #123456转成base64 echo -n "123456" | base64

rabc

apiVersion: v1
kind: ServiceAccount
metadata:
  name: redis
  namespace: redis-cluster
  labels:
    app: redis

---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: redis
  namespace: redis-cluster
  labels:
    app: redis
rules:
  - apiGroups:
      - ""
    resources:
      - endpoints
    verbs:
      - get

---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: redis
  namespace: redis-cluster
  labels:
    app: redis
subjects:
  - kind: ServiceAccount
    name: redis
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: redis

service

apiVersion: v1
kind: Service
metadata:
  name: redis-headless
  namespace: redis-cluster
  labels:
    app: redis
spec:
  selector:
    app: redis-cluster
  clusterIP: None
  ports:
    - name: server
      port: 6379
      protocol: TCP
      targetPort: redis
    - name: sentinel
      port: 26379
      protocol: TCP
      targetPort: sentinel

---
apiVersion: v1
kind: Service
metadata:
  name: redis-svc
  namespace: redis-cluster
  labels:
    app: redis
spec:
  type: NodePort
  ports:
    - name: server
      port: 6379
      protocol: TCP
      targetPort: redis
      nodePort: 42301
    - name: sentinel
      port: 26379
      protocol: TCP
      targetPort: sentinel
      nodePort: 42302
  selector:
    app: redis-cluster

statefulSet

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis-cluster
  namespace: redis-cluster
  labels:
    app: redis
spec:
  selector:
    matchLabels:
      app: redis-cluster
  serviceName: redis-headless
  replicas: 3
  template:
    metadata:
      labels:
        app: redis-cluster
    spec:
#      securityContext:
#        fsGroup: 1000
#        runAsNonRoot: true
#        runAsUser: 1000
      serviceAccountName: redis
      initContainers:
        - name: init-redis
          image: reg-hub.gzeport.com/library/redis:6.2.14-alpine3.20
          imagePullPolicy: IfNotPresent
          securityContext:
            privileged: true
          command:
            - /bin/sh
            - "-c"
            - |
              set -ex
              #从pod的hostname中通过正则获取序号，如果没有截取到就退出程序
              # redis-cluster-2
              #ordinal=`hostname | awk -F"-" '{print $3}'`
              ordinal=`echo $POD_NAME | awk -F"-" '{print $3}'`
              mkdir -p /data/conf/
              cp /mnt/config-map/sentinel.conf /data/conf/redis-sentinel.conf
              if [ ${ordinal} -eq 0 ]; then
                # 如果Pod的序号为0，说明它是Master节点
                cp /mnt/config-map/master.conf /data/conf/redis.conf
              else
                # 否则，拷贝ConfigMap里的Slave的配置文件
                cp /mnt/config-map/slave.conf /data/conf/redis.conf
              fi
              sed -i "s/redisPassword/${REDIS_PASSWORD}/g"  /data/conf/redis.conf
              sed -i "s/redisPassword/${REDIS_PASSWORD}/g"  /data/conf/redis-sentinel.conf
              sysctl -w vm.dirty_bytes=33554432
              sysctl vm.overcommit_memory=1
              echo 2048 > /proc/sys/net/core/somaxconn
          env:
            - name: REDIS_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: redis-secret
                  key: redisPassword
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
          volumeMounts:
            - name: config-map
              mountPath: /mnt/config-map
            - name: data
              mountPath: /data
      containers:
        - name: redis
          image: reg-hub.gzeport.com/library/redis:6.2.14-alpine3.20
          command: ["redis-server"]
          args:
            - "/data/conf/redis.conf"
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: TZ
              value: Asia/Shanghai
            - name: LANG
              value: C.UTF-8
          ports:
            - name: redis
              containerPort: 6379
          resources:
            requests:
              cpu: "0.1"
              memory: 256Mi
            limits:
              cpu: "1"
              memory: 1Gi
          volumeMounts:
            - name: data
              mountPath: /data
        - name: sentinel
          image: reg-hub.gzeport.com/library/redis:6.2.14-alpine3.20
          imagePullPolicy: IfNotPresent
          command:
            - redis-sentinel
          args:
            - /data/conf/redis-sentinel.conf
          env:
            - name: TZ
              value: Asia/Shanghai
            - name: LANG
              value: C.UTF-8
          resources:
            requests:
              cpu: "0.1"
              memory: 128Mi
            limits:
              cpu: "0.5"
              memory: 512Mi
          ports:
            - name: sentinel
              containerPort: 26379
          volumeMounts:
            - mountPath: /data
              name: data
      volumes:
        - name: config-map
          configMap:
            name: redis-configmap
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        accessModes:
          - "ReadWriteMany"
        resources:
          requests:
            storage: "5Gi"
        storageClassName: "nfs-145"

执行完毕后

[root@k140 ~]# kubectl get all -n redis-cluster
NAME                  READY   STATUS    RESTARTS   AGE
pod/redis-cluster-0   2/2     Running   0          4h36m
pod/redis-cluster-1   2/2     Running   0          4h36m
pod/redis-cluster-2   2/2     Running   0          4h36m

NAME                     TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)                          AGE
service/redis-headless   ClusterIP   None           <none>        6379/TCP,26379/TCP               6h19m
service/redis-svc        NodePort    10.43.201.87   <none>        6379:42301/TCP,26379:42302/TCP   4h52m

NAME                             READY   AGE
statefulset.apps/redis-cluster   3/3     4h36m

查看节点集群状态

kubectl exec -it -n redis-cluster pod/redis-cluster-0  -- sh -c redis-cli 

127.0.0.1:6379> auth 123456
127.0.0.1:6379> info replication
# Replication
role:master
connected_slaves:2
slave0:ip=10.42.3.220,port=6379,state=online,offset=3397392,lag=1
slave1:ip=10.42.2.154,port=6379,state=online,offset=3397392,lag=1
master_failover_state:no-failover
master_replid:fc870e46e41d60de8cf7732620a3279bc9356778
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:3397817
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:2349242
repl_backlog_histlen:1048576
127.0.0.1:6379>

参考

https://blog.csdn.net/m0_70331483/article/details/134455896

© 2012 - 2024 记忆角落 网站统计    由又拍云CDN赞助 提供云储存服务
Theme by Wing 粤ICP备14056850号