Redis集群

生产环境配置

2C8G三台，三主三从，一台二实例 注意事项：一台2实例部署，主从节点要交叉，避免主从节点同时宕机影响业务

部署步骤

下载软件包 wget http://download.redis.io/releases/redis-5.0.5.tar.gz

安装依赖 yum -y install gcc gcc-c++

解压tar.gz包 tar xf redis-5.0.5.tar.gz -C /usr/local/

编译 cd /usr/local/redis-5.0.5/ make && make install cp ./src/redis-server /usr/bin/ && cp ./src/redis-cli /usr/bin/

Redis优化 echo 511 > /proc/sys/net/core/somaxconn echo "vm.overcommit_memory = 1" >> /etc/sysctl.conf && echo "net.core.somaxconn = 1024" >> /etc/sysctl.conf echo never > /sys/kernel/mm/transparent_hugepage/enabled #开机启动 echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled" >> /etc/rc.d/rc.local

创建目录及配置文件 mkdir /usr/local/redis-cluster/{7000,7001}/data -pv cp redis.conf /usr/local/redis-cluster/7000/ [root@redis-cs-node01 redis-cluster]# grep "^[a-Z]" 7000/redis.conf bind 0.0.0.0 protected-mode yes port 7000 tcp-backlog 511 timeout 0 tcp-keepalive 30 daemonize yes supervised no pidfile /var/run/redis_7000.pid loglevel notice logfile "/usr/local/redis-cluster/7000/redis.log" databases 16 always-show-logo yes save "" stop-writes-on-bgsave-error yes rdbcompression yes rdbchecksum yes dbfilename dump.rdb dir /usr/local/redis-cluster/7000/data replica-serve-stale-data yes replica-read-only yes repl-diskless-sync no repl-diskless-sync-delay 5 repl-disable-tcp-nodelay no replica-priority 100 maxclients 10000 maxmemory 3g maxmemory-policy allkeys-lru lazyfree-lazy-eviction no lazyfree-lazy-expire no lazyfree-lazy-server-del no replica-lazy-flush no appendonly no appendfilename "appendonly.aof" appendfsync everysec no-appendfsync-on-rewrite no auto-aof-rewrite-percentage 100 auto-aof-rewrite-min-size 64mb aof-load-truncated yes aof-use-rdb-preamble yes lua-time-limit 5000 cluster-enabled yes cluster-config-file nodes-7000.conf cluster-node-timeout 5000 cluster-require-full-coverage no slowlog-log-slower-than 10000 slowlog-max-len 128 latency-monitor-threshold 0 notify-keyspace-events "" hash-max-ziplist-entries 512 hash-max-ziplist-value 64 list-max-ziplist-size -2 list-compress-depth 0 set-max-intset-entries 512 zset-max-ziplist-entries 128 zset-max-ziplist-value 64 hll-sparse-max-bytes 3000 stream-node-max-bytes 4096 stream-node-max-entries 100 activerehashing yes client-output-buffer-limit normal 0 0 0 client-output-buffer-limit replica 256mb 64mb 60 client-output-buffer-limit pubsub 32mb 8mb 60 hz 10 dynamic-hz yes aof-rewrite-incremental-fsync yes rdb-save-incremental-fsync yes #修改配置信息 cp 7000/redis.conf 7001/ sed -i 's/7000/7001/g' 7001/redis.conf #同理另外两台节点同理 mkdir /usr/local/redis-cluster/{7002,7003}/data -pv sed -i 's/7000/7001/g' 7002/redis.conf sed -i 's/7000/7001/g' 7003/redis.conf mkdir /usr/local/redis-cluster/{7004,7005}/data -pv sed -i 's/7000/7004/g' 7004/redis.conf sed -i 's/7000/7005/g' 7005/redis.conf

创建启动脚本及开启启动 #第一个节点 [root@redis-cs-node01 redis-cluster]# cat start_redis_cluster.sh #!/bin/bash cd /usr/local/redis-cluster cd 7000 && /usr/bin/redis-server /usr/local/redis-cluster/7000/redis.conf cd ../7001 &&/usr/bin/redis-server /usr/local/redis-cluster/7001/redis.conf #第二个节点 [root@redis-cs-node02 redis-cluster]# cat start_redis_cluster.sh #!/bin/bash cd /usr/local/redis-cluster cd 7002 && /usr/bin/redis-server /usr/local/redis-cluster/7002/redis.conf cd ../7003 &&/usr/bin/redis-server /usr/local/redis-cluster/7003/redis.conf #第三个节点 [root@redis-cs-node03 redis-cluster]# cat start_redis_cluster.sh #!/bin/bash cd /usr/local/redis-cluster cd 7004 && /usr/bin/redis-server /usr/local/redis-cluster/7004/redis.conf cd ../7005 &&/usr/bin/redis-server /usr/local/redis-cluster/7005/redis.conf #启动节点 [root@redis-cs-node01 redis-cluster]# sh start_redis_cluster.sh [root@redis-cs-node02 redis-cluster]# sh start_redis_cluster.sh [root@redis-cs-node03 redis-cluster]# sh start_redis_cluster.sh #配置开机启动 [root@redis-cs-node01 redis-cluster]# cat /etc/rc.d/rc.local #!/bin/bash # THIS FILE IS ADDED FOR COMPATIBILITY PURPOSES # # It is highly advisable to create own systemd services or udev rules # to run scripts during boot instead of using this file. # # In contrast to previous versions due to parallel execution during boot # this script will NOT be run after all other services. # # Please note that you must run 'chmod +x /etc/rc.d/rc.local' to ensure # that this script will be executed during boot. touch /var/lock/subsys/local echo never > /sys/kernel/mm/transparent_hugepage/enabled /usr/bin/sh /usr/local/redis-cluster/start_redis_cluster.sh /usr/bin/sh /usr/local/redis-cluster/start_exporter.sh

创建集群 redis-cli --cluster create 172.16.83.248:7000 172.16.83.248:7001 172.16.83.249:7002 172.16.83.249:7003 172.16.83.250:7004 172.16.83.250:7005 --cluster-replicas 1

查看集群状态 [root@redis-cs-node01 ~]# redis-cli -c -p 7000 cluster nodes 18a0169a22e7b440213728b1fb45ae5d59bf8eef 172.16.83.249:7002@17002 master - 0 1601184964498 41 connected 6251-10922 83280280d19223180eabffb1a88ff078efdc748d 172.16.83.249:7003@17003 slave 27c6bff6b4b20d5620a81879324cf61a8e2439a1 0 1601184965000 39 connected 3933efb9b77e474ab036d7741bc4517746483496 172.16.83.248:7001@17001 slave 16ab783181c05ed6c0e3db933be6a1f56efd3317 0 1601184964097 42 connected 16ab783181c05ed6c0e3db933be6a1f56efd3317 172.16.83.250:7004@17004 master - 0 1601184965501 42 connected 88-332 5461-5794 10923-11255 11711-16383 30469e7f2ce74d6e0ff224463468909aa268c6cf 172.16.83.250:7005@17005 slave 18a0169a22e7b440213728b1fb45ae5d59bf8eef 0 1601184965501 41 connected 27c6bff6b4b20d5620a81879324cf61a8e2439a1 172.16.83.248:7000@17000 myself,master - 0 1601184965000 39 connected 0-87 333-5460 5795-6250 11256-11710 [root@redis-cs-node01 redis-cluster]# redis-cli --cluster check 127.0.0.1:7001 172.16.83.249:7002 (18a0169a...) -> 10 keys | 4672 slots | 1 slaves. 172.16.83.250:7004 (16ab7831...) -> 9 keys | 5585 slots | 1 slaves. 172.16.83.248:7000 (27c6bff6...) -> 5 keys | 6127 slots | 1 slaves. [OK] 24 keys in 3 masters. 0.00 keys per slot on average. >>> Performing Cluster Check (using node 127.0.0.1:7001) S: 3933efb9b77e474ab036d7741bc4517746483496 127.0.0.1:7001 slots: (0 slots) slave replicates 16ab783181c05ed6c0e3db933be6a1f56efd3317 M: 18a0169a22e7b440213728b1fb45ae5d59bf8eef 172.16.83.249:7002 slots:[6251-10922] (4672 slots) master 1 additional replica(s) S: 30469e7f2ce74d6e0ff224463468909aa268c6cf 172.16.83.250:7005 slots: (0 slots) slave replicates 18a0169a22e7b440213728b1fb45ae5d59bf8eef S: 83280280d19223180eabffb1a88ff078efdc748d 172.16.83.249:7003 slots: (0 slots) slave replicates 27c6bff6b4b20d5620a81879324cf61a8e2439a1 M: 16ab783181c05ed6c0e3db933be6a1f56efd3317 172.16.83.250:7004 slots:[88-332],[5461-5794],[10923-11255],[11711-16383] (5585 slots) master 1 additional replica(s) M: 27c6bff6b4b20d5620a81879324cf61a8e2439a1 172.16.83.248:7000 slots:[0-87],[333-5460],[5795-6250],[11256-11710] (6127 slots) master 1 additional replica(s) [OK] All nodes agree about slots configuration. >>> Check for open slots... >>> Check slots coverage... [OK] All 16384 slots covered.

扩容节点内存步骤

说明：线上Redis节点内存接近限制3g，进行无感知进行升级内存。

查看主从节点分配；

确认主从不在同一节点；

逐台操作从节点，进行扩容服务器配置；

动态升级内存限制； config set maxmemory 6g

再进行从升级主操作 在slave的Redis执行 CLUSTER FAILOVER slave会接管主, master会转化为salve

说明：故障转移的主服务器的客户端都已停止。同时，主服务器将其复制偏移发送到从服务器，从服务器等待到达其一侧的偏移。达到复制偏移量后，故障转移将开始，并且将向旧的主服务器通知配置切换。当客户端在旧的主服务器上解除阻止时，它们将被重定向到新的主服务器；

此时从变为主再进行升级主变为从节点服务器，升级配置；

再动态升级内存限； config set maxmemory 6g

此时升级配置扩容内存已完成，注意检查主从分配信息，确保主从不在同一点，保证服务的高可用。

prometheus监控Redis

客户端脚本 [root@redis-cs-node01 redis-cluster]# cat start_exporter.sh #!/bin/bash cd /usr/local/redis-cluster ports=`netstat -anltp|grep LISTEN|grep -v 1700|egrep "(700|6379)"|awk '{print $4}'|cut -d ":" -f 2` echo "find ports: $ports" for p in $ports do ip=`ip addr |grep 172|grep inet|awk '{print $2}'|awk -F "/" '{print $1}'` echo "start exporter 127.0.0.1:${p} with ${ip}:5${p}" nohup ./redis_exporter -web.listen-address ":5${p}" -redis.addr 127.0.0.1:${p} & done

prometheus服务端配置 [root@k8s-master test]# cat redis-test-yc.yaml --- apiVersion: v1 kind: Service metadata: labels: node: redis-exporter name: redis-test-yc namespace: monitoring spec: ports: - name: metrics port: 8080 sessionAffinity: None type: ClusterIP status: loadBalancer: {} --- apiVersion: v1 kind: Endpoints metadata: name: redis-test-yc namespace: monitoring subsets: - addresses: - ip: 172.16.83.248 ports: - name: metrics port: 57000 protocol: TCP - addresses: - ip: 172.16.83.248 ports: - name: metrics port: 57001 protocol: TCP - addresses: - ip: 172.16.83.249 ports: - name: metrics port: 57002 protocol: TCP - addresses: - ip: 172.16.83.249 ports: - name: metrics port: 57003 protocol: TCP - addresses: - ip: 172.16.83.250 ports: - name: metrics port: 57004 protocol: TCP - addresses: - ip: 172.16.83.250 ports: - name: metrics port: 57005 protocol: TCP

查看prometheus后台监控项

【文章原创作者：阿里云代理 http://www.558idc.com/aliyun.html 复制请保留原URL】