1、服务器基础配置
服务器配置
系统
相关服务
4VCPU8G
Centos72
Zookeeper、hadoop、flink
4VCPU8G
Centos72
Zookeeper、hadoop、flink
4VCPU8G
Centos72
Zookeeper、hadoop、flink
1.1、Node01节点
[root@localhost ~]# yum update -y
[root@localhost ~]# yum install vim net-toolsntpdate
[root@localhost ~]# hostnamectl set-hostname node01
[root@localhost ~]# systemctl stop firewalld && systemctl disable firewalld
[root@localhost ~]# iptables -F
[root@localhost ~]# vim /etc/selinux/config
SELINUX=disabled
[root@localhost ~]# vim /etc/hosts
192.168.2.121 node01
192.168.2.122 node02
192.168.2.123 node03
[root@node01 ~]# crontab -e
*/5 * * * * /usr/sbin/ntpdate ntp1.aliyun.com
[root@localhost ~]# useradd clu_flink
[root@localhost ~]# su clu_flink
[clu_flink@node01 root]$ cd ~
[clu_flink@node01 ~]$ ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
Generating public/private rsa key pair.
Created directory '/home/clu_flink/.ssh'.
Your identification has been saved in /home/clu_flink/.ssh/id_rsa.
Your public key has been saved in /home/clu_flink/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:FGWZO2HmdLLatFOsTBdR5ZxmlXE/PULJERyo8Cbjglw clu_flink@node01
The key's randomart image is:
+---[RSA 2048]----+
| ..oo=B*o*|
| . oO.++ +*|
| +=.B o *=|
| E + +B + + o|
| . o . S* * |
| o . .. * |
| . . |
| |
| |
+----[SHA256]-----+
[clu_flink@node01 ~]$ cd .ssh/
[clu_flink@node01 .ssh]$ cp id_rsa.pub authorized_keys
[clu_flink@node01 .ssh]$ ssh-copy-id clu_flink@node01
[clu_flink@node01 .ssh]$ ssh-copy-id clu_flink@node02
[clu_flink@node01 .ssh]$ ssh-copy-id clu_flink@node03
[root@node01 flink]# tar -xf jdk1.8.0_92.tar.gz -C /usr/local/
[root@node01 flink]# vim /etc/profile
export JAVA_HOME=/usr/local/jdk1.8.0_92/
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
[root@node01 flink]# source /etc/profile
[root@node01 flink]# java -version
java version "1.8.0_92"
Java(TM) SE Runtime Environment (build 1.8.0_92-b14)
Java HotSpot(TM) 64-Bit Server VM (build 25.92-b14, mixed mode)
1.2、Node02节点
[root@localhost ~]# yum update -y
[root@localhost ~]# yum install vim net-toolsntpdate
[root@localhost ~]# hostnamectl set-hostname node02
[root@localhost ~]# systemctl stop firewalld && systemctl disable firewalld
[root@localhost ~]# iptables -F
[root@localhost ~]# vim /etc/selinux/config
SELINUX=disabled
[root@localhost ~]# vim /etc/hosts
192.168.2.121 node01
192.168.2.122 node02
192.168.2.123 node03
[root@node02 ~]# crontab -e
*/5 * * * * /usr/sbin/ntpdate ntp1.aliyun.com
[root@localhost ~]# useradd clu_flink
[root@localhost ~]# su clu_flink
[clu_flink@node02root]$ cd ~
[clu_flink@node02~]$ ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
Generating public/private rsa key pair.
Created directory '/home/clu_flink/.ssh'.
Your identification has been saved in /home/clu_flink/.ssh/id_rsa.
Your public key has been saved in /home/clu_flink/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:FGWZO2HmdLLatFOsTBdR5ZxmlXE/PULJERyo8Cbjglw clu_flink@node02
The key's randomart image is:
+---[RSA 2048]----+
| ..oo=B*o*|
| . oO.++ +*|
| +=.B o *=|
| E + +B + + o|
| . o . S* * |
| o . .. * |
| . . |
| |
| |
+----[SHA256]-----+
[clu_flink@node02~]$ cd .ssh/
[clu_flink@node02.ssh]$ cp id_rsa.pub authorized_keys
[clu_flink@node02.ssh]$ ssh-copy-id clu_flink@node01
[clu_flink@node02 .ssh]$ ssh-copy-id clu_flink@node02
[clu_flink@node02 .ssh]$ ssh-copy-id clu_flink@node03
[root@node02 ~]# tar -xf jdk1.8.0_92.tar.gz -C /usr/local/
[root@node02 ~]# vim /etc/profile
export JAVA_HOME=/usr/local/jdk1.8.0_92/
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
[root@node02 ~]# source /etc/profile
[root@node02 ~]# java -version
java version "1.8.0_92"
Java(TM) SE Runtime Environment (build 1.8.0_92-b14)
Java HotSpot(TM) 64-Bit Server VM (build 25.92-b14, mixed mode)
1.3、Node03节点
[root@localhost ~]# yum update -y
[root@localhost ~]# yum install vim net-toolsntpdate
[root@localhost ~]# hostnamectl set-hostname node03
[root@localhost ~]# systemctl stop firewalld && systemctl disable firewalld
[root@localhost ~]# iptables -F
[root@localhost ~]# vim /etc/selinux/config
SELINUX=disabled
[root@localhost ~]# vim /etc/hosts
192.168.2.121 node01
192.168.2.122 node02
192.168.2.123 node03
[root@node03 ~]# crontab -e
*/5 * * * * /usr/sbin/ntpdate ntp1.aliyun.com
[root@localhost ~]# useradd clu_flink
[root@localhost ~]# su clu_flink
[clu_flink@node03 root]$ cd ~
[clu_flink@node03 ~]$ ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
Generating public/private rsa key pair.
Created directory '/home/clu_flink/.ssh'.
Your identification has been saved in /home/clu_flink/.ssh/id_rsa.
Your public key has been saved in /home/clu_flink/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:FGWZO2HmdLLatFOsTBdR5ZxmlXE/PULJERyo8Cbjglw clu_flink@node03
The key's randomart image is:
+---[RSA 2048]----+
| ..oo=B*o*|
| . oO.++ +*|
| +=.B o *=|
| E + +B + + o|
| . o . S* * |
| o . .. * |
| . . |
| |
| |
+----[SHA256]-----+
[clu_flink@node03 ~]$ cd .ssh/
[clu_flink@node03 .ssh]$ cp id_rsa.pub authorized_keys
[clu_flink@node03 .ssh]$ ssh-copy-id clu_flink@node01
[clu_flink@node03 .ssh]$ ssh-copy-id clu_flink@node02
[clu_flink@node03 .ssh]$ ssh-copy-id clu_flink@node03
[root@node03 ~]# tar -xf jdk1.8.0_92.tar.gz -C /usr/local/
[root@node03 ~]# vim /etc/profile
export JAVA_HOME=/usr/local/jdk1.8.0_92/
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
[root@node03 ~]# source /etc/profile
[root@node03 ~]# java -version
java version "1.8.0_92"
Java(TM) SE Runtime Environment (build 1.8.0_92-b14)
Java HotSpot(TM) 64-Bit Server VM (build 25.92-b14, mixed mode)
2、Zookeeper集群
2.1、zookeeper概述
ZooKeeper是一种为分布式应用所设计的高可用、高性能且一致的开源协调服务，它提供一项基本服务：分布式锁服务。
分布式应用可以基于它实现更高级的服务，实现诸如同步服务、配置维护和集群管理或者命名的服务。Zookeeper服务自身组成一个集群，2n+1个（奇数）服务允许n个失效，集群内一半以上机器可用，Zookeeper就可用。
假设3台机器组成的集群，可以有允许一台失效，如果有2台失效，这个集群就不可用，1<1.5，一般的搭建zookeeper集群时，以奇数台机器来搭建。目的：是为了提高容错能允许多损失一台。
Leader主要有三个功能：
1.恢复数据；
2.维持与follower的心跳，接收follower请求并判断follower的请求消息类型；
3.follower的消息类型主要有PING消息、REQUEST消息、ACK消息、REVALIDATE消息，根据不同的消息类型，进行不同的处理。
Follower主要有四个功能：
1.向Leader发送请求（PING消息、REQUEST消息、ACK消息、REVALIDATE消息）；
2.接收Leader消息并进行处理；
3.接收Client的请求，如果为写请求，发送给Leader进行投票；
4.返回Client结果。
读操作分析：
ZooKeeper集群中所有的server节点都拥有相同的数据，所以读的时候可以在任意一台server节点上，客户端连接到集群中某一节点，读请求，然后直接返回。当然因为ZooKeeper协议的原因（一半以上的server节点都成功写入了数据，这次写请求便算是成功），读数据的时候可能会读到数据不是最新的server节点，所以比较推荐使用watch机制，在数据改变时，及时感应到。
写操作分析：
当一个客户端进行写数据请求时，会指定ZooKeeper集群中的一个server节点，如果该节点为Follower，则该节点会把写请求转发给Leader，Leader通过内部的协议进行原子广播，直到一半以上的server节点都成功写入了数据，这次写请求便算是成功，然后Leader便会通知相应Follower节点写请求成功，该节点向client返回写入成功响应。
zookeeper 的三个端口作用：
2181 : 对 client 端提供服务
2888 : 集群内机器通信使用
3888 : 选举 leader 使用
2.2、Node01节点配置
[root@node01 flink]# mkdir /data
[root@node01 flink]# tar -xf apache-zookeeper-3.8.0-bin.tar.gz -C /data/
[root@node01 flink]# mv /data/apache-zookeeper-3.8.0-bin/ /data/zookeeper
[root@node01 ~]# cp /data/zookeeper/conf/zoo_sample.cfg /data/zookeeper/conf/zoo.cfg
[root@node01 zookeeper]# vim /data/zookeeper/conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/data/zookeeper/data
dataLogDir=/data/zookeeper/logs
clientPort=2181
server.1=192.168.2.121:3188:3288
server.2=192.168.2.122:3188:3288
server.3=192.168.2.123:3188:3288
[root@node01 zookeeper]# vim /etc/profile
#zookeeper
export ZK_HOME=/data/zookeeper
export PATH=$PATH:$ZK_HOME/bin
[root@node01 zookeeper]# source /etc/profile
添加java环境变量
[root@node01 zookeeper]# vim /data/zookeeper/bin/zkEnv.sh
JAVA_HOME="/usr/local/jdk1.8.0_92/"
[root@node01 zookeeper]# vim /etc/init.d/zookeeper
#!/bin/bash
ZK_HOME='/data/zookeeper'
ZK_BIN='bin/zkServer.sh'
case $1 in
start)
echo "---------- zookeeper 启 动 ------------"
sudo -H -u clu_flink bash -c "cd ${ZK_HOME} && ${ZK_BIN} start"
;;
stop)
echo "---------- zookeeper 停 止 ------------"
sudo -H -u clu_flink bash -c "cd ${ZK_HOME} && ${ZK_BIN} stop"
;;
restart)
echo "---------- zookeeper 重 启 ------------"
sudo -H -u clu_flink bash -c "cd ${ZK_HOME} && ${ZK_BIN} restart"
;;
status)
echo "---------- zookeeper 状 态 ------------"
sudo -H -u clu_flink bash -c "cd ${ZK_HOME} && ${ZK_BIN} status"
;;
*)
echo "Usage: $0 {start|stop|restart|status}"
esac
[root@node01 zookeeper]# chmod +x /etc/init.d/zookeeper
[root@node01 ~]# echo 1 > /data/zookeeper/data/myid
这个ID存放绝对不能放错，放在数据目录下
[root@node01 zookeeper]# chown clu_flink.clu_flink /data/zookeeper/ -R
[root@node01 ~]# /etc/init.d/zookeeper start
2.3、Node02节点配置
[root@node02 ~]# mkdir /data
[root@node02 ~]# tar -xf apache-zookeeper-3.8.0-bin.tar.gz -C /data/
[root@node02 ~]# mv /data/apache-zookeeper-3.8.0-bin/ /data/zookeeper
[root@node02 ~]# cp /data/zookeeper/conf/zoo_sample.cfg /data/zookeeper/conf/zoo.cfg
[root@node02 ~]# vim /data/zookeeper/conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/data/zookeeper/data
dataLogDir=/data/zookeeper/logs
clientPort=2181
server.1=192.168.2.121:3188:3288
server.2=192.168.2.122:3188:3288
server.3=192.168.2.123:3188:3288
[root@node02 zookeeper]# vim /etc/profile
#zookeeper
export ZK_HOME=/data/zookeeper
export PATH=$PATH:$ZK_HOME/bin
[root@node01 zookeeper]# source /etc/profile
添加java环境变量
[root@node02 zookeeper]# vim /data/zookeeper/bin/zkEnv.sh
JAVA_HOME="/usr/local/jdk1.8.0_92/"
[root@node02 ~]# vim /etc/init.d/zookeeper
#!/bin/bash
ZK_HOME='/data/zookeeper'
ZK_BIN='bin/zkServer.sh'
case $1 in
start)
echo "---------- zookeeper 启 动 ------------"
sudo -H -u clu_flink bash -c "cd ${ZK_HOME} && ${ZK_BIN} start"
;;
stop)
echo "---------- zookeeper 停 止 ------------"
sudo -H -u clu_flink bash -c "cd ${ZK_HOME} && ${ZK_BIN} stop"
;;
restart)
echo "---------- zookeeper 重 启 ------------"
sudo -H -u clu_flink bash -c "cd ${ZK_HOME} && ${ZK_BIN} restart"
;;
status)
echo "---------- zookeeper 状 态 ------------"
sudo -H -u clu_flink bash -c "cd ${ZK_HOME} && ${ZK_BIN} status"
;;
*)
echo "Usage: $0 {start|stop|restart|status}"
esac
[root@node02 ~]# chmod +x /etc/init.d/zookeeper
[root@node02 ~]# echo 2 > /data/zookeeper/data/myid
[root@node02 ~]# chown clu_flink.clu_flink /data/zookeeper/ -R
[root@node02 ~]# /etc/init.d/zookeeper start
2.4、Node03节点配置
[root@node03 ~]# mkdir /data
[root@node03 ~]# tar -xf apache-zookeeper-3.8.0-bin.tar.gz -C /data/
[root@node03 ~]# mv /data/apache-zookeeper-3.8.0-bin/ /data/zookeeper
[root@node03 ~]# cp /data/zookeeper/conf/zoo_sample.cfg /data/zookeeper/conf/zoo.cfg
[root@node03 ~]# vim /data/zookeeper/conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/data/zookeeper/data
dataLogDir=/data/zookeeper/logs
clientPort=2181
server.1=192.168.2.121:3188:3288
server.2=192.168.2.122:3188:3288
server.3=192.168.2.123:3188:3288
[root@node03 zookeeper]# vim /etc/profile
#zookeeper
export ZK_HOME=/data/zookeeper
export PATH=$PATH:$ZK_HOME/bin
[root@node01 zookeeper]# source /etc/profile
添加java环境变量
[root@node03 zookeeper]# vim /data/zookeeper/bin/zkEnv.sh
JAVA_HOME="/usr/local/jdk1.8.0_92/"
[root@node03 ~]# vim /etc/init.d/zookeeper
#!/bin/bash
ZK_HOME='/data/zookeeper'
ZK_BIN='bin/zkServer.sh'
case $1 in
start)
echo "---------- zookeeper 启 动 ------------"
sudo -H -u clu_flink bash -c "cd ${ZK_HOME} && ${ZK_BIN} start"
;;
stop)
echo "---------- zookeeper 停 止 ------------"
sudo -H -u clu_flink bash -c "cd ${ZK_HOME} && ${ZK_BIN} stop"
;;
restart)
echo "---------- zookeeper 重 启 ------------"
sudo -H -u clu_flink bash -c "cd ${ZK_HOME} && ${ZK_BIN} restart"
;;
status)
echo "---------- zookeeper 状 态 ------------"
sudo -H -u clu_flink bash -c "cd ${ZK_HOME} && ${ZK_BIN} status"
;;
*)
echo "Usage: $0 {start|stop|restart|status}"
esac
[root@node03 ~]# chmod +x /etc/init.d/zookeeper
[root@node03 ~]# echo 3 > /data/zookeeper/data/myid
[root@node03 ~]# chown clu_flink.clu_flink /data/zookeeper/ -R
[root@node03 ~]# /etc/init.d/zookeeper start
以上zookeeper集群搭建完成，可以通过以下命令进行检查 zkCli.sh -server 192.168.2.123:2181 记得更换相关节点IP即可
