HAPROXY实战案例：https反向代理的实现、TCP四层反向代理MariaDB及自定义错误页面

​内容说明：HAPROXY下https实现；HAPROXY四层代理MariaDB;自定义错误页面​

1. 架构及说明

1 2台web服务器 ：
主机名：WebServer-IP17
CentOS 7.9
IP:192.168.250.17

主机名：WebServer-IP27
CentOS 7.9
IP:192.168.250.27



2 2台 MariaDB 数据库服务器 ：
主机名：MariaDB-IP37
CentOS 7.9
IP:192.168.250.37

主机名：MariaDB-IP47
CentOS 7.9
IP:192.168.250.47


3 1台 haproxy-ip07 服务器 ：
主机名： haproxy-ip07
CentOS 7.9
IP:192.168.250.7/24
HAProxy version 2.4.15
socat version 1.7.4.3


4 2台client主机 ：
主机名：Client-IP172-8
CentOS 8.4
IP:172.16.0.8/24 NAT成192.168.250.254 访问192.168.250.X网段

主机名：Client-IP192-68
CentOS 8.4
IP:192.168.250.68/24

WEB服务器环境准备

[root@webserver-ip17 <sub>]#yum -y install httpd;hostname > /var/www/html/indexTmp.html;hostname -I >> /var/www/html/indexTmp.html;cat /var/www/html/indexTmp.html | xargs > /var/www/html/index.html;rm -rf /var/www/html/indexTmp.html;systemctl enable --now httpd

[root@webserver-ip27 </sub>]#yum -y install httpd;hostname > /var/www/html/indexTmp.html;hostname -I >> /var/www/html/indexTmp.html;cat /var/www/html/indexTmp.html | xargs > /var/www/html/index.html;rm -rf /var/www/html/indexTmp.html;systemctl enable --now httpd

MariaDB数据库服务器准备

[root@mariadb-ip37 _{]# yum -y install mariadb-server
[root@mariadb-ip37}]# mysql -e 'grant all on *.* to test@"%.%.%.%" identified by "shone8888"'
[root@mariadb-ip37 <sub>]# systemctl enable --now mariadb.service

[root@mariadb-ip47 _{]# yum -y install mariadb-server
[root@mariadb-ip47}]# mysql -e 'grant all on *.* to test@"%.%.%.%" identified by "shone8888"'
[root@mariadb-ip47 <sub>]# systemctl enable --now mariadb.service

2. 实现TCP四层负载均衡代理MariaDB数据库

​在四层负载设备中，把client发送的报文目标地址(原来是负载均衡设备的IP地址)，根据均衡设备设置的选择web服务器的规则选择对应的web服务器IP地址，这样client就可以直接跟此服务器建立TCP连接并发送数据，而四层负载自身不参与建立连接，而和LVS不同，haproxy效率低些，因为haproxy需要分别和前端客户端及后端服务器建立连接。​

[root@haproxy-ip07 <sub>]# cat /etc/haproxy/haproxy.cfg
global
maxconn 100000
chroot /apps/haproxy
#stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin
stats socket /var/lib/haproxy/haproxy.sock1 mode 600 level admin process 1
stats socket /var/lib/haproxy/haproxy.sock2 mode 600 level admin process 2
#uid 99
#gid 99
user haproxy
group haproxy
daemon
nbproc 2
cpu-map 1 0
cpu-map 2 1
#cpu-map 3 2
#cpu-map 4 3
pidfile /var/lib/haproxy/haproxy.pid
log 127.0.0.1 local2 info

defaults
option http-keep-alive
option forwardfor
maxconn 100000
mode http
timeout connect 300000ms
timeout client 300000ms
timeout server 300000ms
######################## listen Single file ##############################
listen stats
mode http
bind 0.0.0.0:9999
stats enable
log global
stats uri /haproxy-status
stats auth haadmin:shone8888


######################## http + front + backend #############################
frontend WEB_PORT_80
bind 192.168.250.7:80
mode http
use_backend web_port_http_nodes
log global

backend web_port_http_nodes
mode http
#balance static-rr
option forwardfor
server web1 192.168.250.17:80 check inter 3000 fall 2 rise 5 weight 1
server web2 192.168.250.27:80 check inter 3000 fall 2 rise 5 weight 1

######################## listen SQL Singlefile ##############################
listen SQL_PORT_3306
bind 192.168.250.7:3306
mode tcp
log global
#balance static-rr
option forwardfor
server sql37 192.168.250.37:3306 check inter 3000 fall 2 rise 5 weight 2
server sql47 192.168.250.47:3306 check inter 3000 fall 2 rise 5 weight 1
[root@haproxy-ip07 </sub>]# systemctl restart haproxy.service
[root@haproxy-ip07 _{]# ss -tln
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 20480 192.168.250.7:3306 *:*
LISTEN 0 20480 *:9999 *:*
LISTEN 0 128 *:111 *:*
LISTEN 0 20480 192.168.250.7:80 *:*
LISTEN 0 128 *:22 *:*
LISTEN 0 100 127.0.0.1:25 *:*
LISTEN 0 128 [::]:111 [::]:*
LISTEN 0 128 [::]:22 [::]:*
LISTEN 0 100 [::1]:25 [::]:*
[root@haproxy-ip07}]#

# 终端上都服务器的访问
[root@CentOS84-IP172-08 ]#while :;do mysql -utest -pshone8888 -h192.168.250.7 -e 'select @@hostname';sleep 1;done
+--------------+
| @@hostname |
+--------------+
| mariadb-ip37 |
+--------------+
+--------------+
| @@hostname |
+--------------+
| mariadb-ip37 |
+--------------+
+--------------+
| @@hostname |
+--------------+
| mariadb-ip47 |
+--------------+
+--------------+
| @@hostname |
+--------------+
| mariadb-ip37 |
+--------------+
+--------------+
| @@hostname |
+--------------+
| mariadb-ip37 |
+--------------+
+--------------+
| @@hostname |
+--------------+
| mariadb-ip37 |
+--------------+
+--------------+
| @@hostname |
+--------------+
| mariadb-ip37 |
+--------------+
+--------------+
| @@hostname |
+--------------+
| mariadb-ip47 |
+--------------+
+--------------+
| @@hostname |
+--------------+
| mariadb-ip47 |
+--------------+
+--------------+
| @@hostname |
+--------------+
| mariadb-ip37 |
+--------------+
+--------------+
| @@hostname |
+--------------+
| mariadb-ip37 |
+--------------+
+--------------+
| @@hostname |
+--------------+
| mariadb-ip47 |
+--------------+
+--------------+
| @@hostname |
+--------------+
| mariadb-ip37 |
+--------------+
+--------------+
| @@hostname |
+--------------+
| mariadb-ip37 |
+--------------+
+--------------+
| @@hostname |
+--------------+
| mariadb-ip37 |
+--------------+
+--------------+
| @@hostname |
+--------------+
| mariadb-ip47 |
+--------------+
+--------------+
| @@hostname |
+--------------+
| mariadb-ip37 |
+--------------+
+--------------+
| @@hostname |
+--------------+
| mariadb-ip37 |
+--------------+
^C
[root@CentOS84-IP172-08 ]#

3. 自定义HAProxy 错误界面

[root@haproxy-ip07 _{]# mkdir -p /apps/haproxy/html/
[root@haproxy-ip07}]#

[root@haproxy-ip07 <sub>]# cat /apps/haproxy/html/503.http
HTTP/1.1 503 Service Unavailable
Content-Type:text/html;charset=utf-8

<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>报错页面</title>
</head>
<body>
<center><h1>网站维护中......请稍候再试</h1></center>
<center><h2>联系电话：400-123-4567</h2></center>
<center><h3>503 Service Unavailable</h3></center>
</body>
[root@haproxy-ip07 </sub>]#

[root@haproxy-ip07 <sub>]# cat /etc/haproxy/haproxy.cfg
global
maxconn 100000
chroot /apps/haproxy
#stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin
stats socket /var/lib/haproxy/haproxy.sock1 mode 600 level admin process 1
stats socket /var/lib/haproxy/haproxy.sock2 mode 600 level admin process 2
#uid 99
#gid 99
user haproxy
group haproxy
daemon
nbproc 2
cpu-map 1 0
cpu-map 2 1
#cpu-map 3 2
#cpu-map 4 3
pidfile /var/lib/haproxy/haproxy.pid
log 127.0.0.1 local2 info

defaults
option http-keep-alive
option forwardfor
maxconn 100000
mode http
timeout connect 300000ms
timeout client 300000ms
timeout server 300000ms
errorfile 503 /apps/haproxy/html/503.http

######################## listen Single file ##############################
listen stats
mode http
bind 0.0.0.0:9999
stats enable
log global
stats uri /haproxy-status
stats auth haadmin:shone8888


######################## http + front + backend #############################
frontend WEB_PORT_80
bind 192.168.250.7:80
mode http
use_backend web_port_http_nodes
log global

backend web_port_http_nodes
mode http
#balance static-rr
option forwardfor
server web1 192.168.250.17:80 check inter 3000 fall 2 rise 5 weight 1
server web2 192.168.250.27:80 check inter 3000 fall 2 rise 5 weight 1

######################## listen SQL Singlefile ##############################L SQL_PORT_3306
listen SQL_PORT_3306
bind 192.168.250.7:3306
mode tcp
log global
#balance static-rr
option forwardfor
server sql37 192.168.250.37:3306 check inter 3000 fall 2 rise 5 weight 2
server sql47 192.168.250.47:3306 check inter 3000 fall 2 rise 5 weight 1
[root@haproxy-ip07 </sub>]#

[root@webserver-ip17 _{]# systemctl stop httpd
[root@webserver-ip27}]# systemctl stop httpd

4. HAProxy https 实现

​haproxy可以实现https的证书安全,从用户到haproxy为https,从haproxy到后端服务器用http通信，但基于性能考虑,生产中证书都是在后端服务器比如nginx上实现，nginx上实现https前文都有过介绍。​

4.1 制作证书

[root@haproxy-ip07 _{]# mkdir /etc/haproxy/certs
[root@haproxy-ip07}]# cd /etc/haproxy/certs/
[root@haproxy-ip07 certs]# openssl genrsa -out haproxy.key 2048
Generating RSA private key, 2048 bit long modulus
...................................+++
.................................................+++
e is 65537 (0x10001)
[root@haproxy-ip07 certs]# openssl req -new -x509 -key haproxy.key -out haproxy.crt -subj "/CN=www.shone.cn"
[root@haproxy-ip07 certs]# cat haproxy.key haproxy.crt > haproxy.pem
[root@haproxy-ip07 certs]# openssl x509 -in haproxy.pem -noout -text
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
b6:fe:b8:2b:c2:bb:ad:85
Signature Algorithm: sha256WithRSAEncryption
Issuer: CN=www.shone.cn
Validity
Not Before: Apr 5 16:12:54 2022 GMT
Not After : May 5 16:12:54 2022 GMT
Subject: CN=www.shone.cn
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
Public-Key: (2048 bit)
Modulus:
00:d4:f3:49:41:03:9a:69:70:68:8b:04:61:9b:2e:
da:c3:00:de:63:16:bb:81:b7:19:cd:ab:25:2b:39:
5c:ae:89:83:4c:91:df:94:1b:b8:57:e7:c9:49:ef:
e1:90:33:4b:20:ec:b0:10:82:ca:7b:1d:a8:00:ae:
25:8d:7b:b7:02:3e:42:6c:6c:4e:3b:a8:62:98:04:
2a:f5:08:0c:ff:3b:c3:69:fc:00:ee:b5:54:a2:1f:
70:96:22:05:6f:c7:fa:60:71:05:6e:c4:2f:28:26:
fe:9e:78:03:32:50:6f:38:87:90:7d:f6:31:2d:76:
c1:c4:1e:27:7d:3e:1d:07:1f:56:4c:e9:4c:7f:c2:
a3:b2:20:0d:17:70:2e:3a:12:f6:02:56:30:5c:fe:
52:57:ff:c8:82:f9:ec:f1:ed:25:08:d1:01:43:3c:
7b:b9:ce:94:a2:d3:29:dc:5f:d5:9a:d8:b9:9f:05:
56:c9:08:0b:68:1d:50:e4:b6:c0:7e:09:f2:ef:22:
1a:b5:79:da:8d:0a:5b:9c:66:85:bf:e3:6e:0c:f2:
8b:0c:04:04:dd:22:36:1d:62:39:30:68:35:01:21:
99:6a:88:ac:55:26:33:8d:55:4b:d0:57:ae:71:2b:
5a:30:d3:a5:2b:c4:2a:68:23:b1:96:70:43:62:dd:
1c:57
Exponent: 65537 (0x10001)
X509v3 extensions:
X509v3 Subject Key Identifier:
72:C5:FE:13:75:52:F1:31:BA:5B:47:38:34:FB:21:E5:1A:86:33:6D
X509v3 Authority Key Identifier:
keyid:72:C5:FE:13:75:52:F1:31:BA:5B:47:38:34:FB:21:E5:1A:86:33:6D

X509v3 Basic Constraints:
CA:TRUE
Signature Algorithm: sha256WithRSAEncryption
03:a7:35:32:a7:bd:d4:be:66:0a:b0:2a:6f:4a:c4:04:02:92:
b6:0e:8a:d3:cf:59:69:5a:1a:7f:47:16:fd:b3:9e:a1:bb:72:
13:85:de:52:79:a1:aa:f9:85:e4:28:1b:1f:4b:df:82:34:89:
d2:96:3a:11:bd:43:4d:7b:6e:32:97:ca:17:c5:c1:06:2c:a6:
1b:48:cb:84:d2:82:ec:21:86:45:67:b1:09:4d:99:e8:fd:2f:
9f:63:c2:41:8a:40:02:93:ba:d1:ae:83:f4:b8:d1:41:e7:33:
2e:1e:91:9f:66:be:fd:02:ca:98:69:67:2d:b8:c9:99:bc:f4:
af:96:fd:2e:92:37:e9:0b:6e:f7:01:2d:a4:0b:a6:20:9d:6f:
8c:ab:11:ae:b6:dd:d4:61:7b:2d:62:cc:df:4e:5b:32:08:a1:
39:54:92:50:25:92:be:82:10:4b:d8:f9:a3:12:59:a3:1a:9d:
8a:18:72:21:0e:69:db:63:b2:de:a5:55:9e:f3:b0:da:54:9d:
bc:32:20:19:a1:75:2c:ea:58:0d:dd:5a:d7:58:c9:68:c8:93:
3d:9d:3a:5e:dc:e9:95:16:ac:8b:80:69:5b:64:3c:9a:d7:58:
f8:dc:8b:cf:44:b5:ea:27:85:83:68:39:2f:e1:4c:35:dc:16:
96:a1:a0:43
[root@haproxy-ip07 certs]#
[root@haproxy-ip07 certs]#

4.2 https配置

[root@haproxy-ip07 certs]# vim /etc/haproxy/haproxy.cfg
global
maxconn 100000
chroot /apps/haproxy
#stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin
stats socket /var/lib/haproxy/haproxy.sock1 mode 600 level admin process 1
stats socket /var/lib/haproxy/haproxy.sock2 mode 600 level admin process 2
#uid 99
#gid 99
user haproxy
group haproxy
daemon
nbproc 2
cpu-map 1 0
cpu-map 2 1
#cpu-map 3 2
#cpu-map 4 3
pidfile /var/lib/haproxy/haproxy.pid
log 127.0.0.1 local2 info

defaults
option http-keep-alive
option forwardfor
maxconn 100000
mode http
timeout connect 300000ms
timeout client 300000ms
timeout server 300000ms
errorfile 503 /apps/haproxy/html/503.http

######################## listen Single file ##############################
listen stats
mode http
bind 0.0.0.0:9999
stats enable
log global
stats uri /haproxy-status
stats auth haadmin:shone8888


######################## http + front + backend #############################
frontend WEB_PORT_80
bind 192.168.250.7:80
##################### https start #####################################
bind 192.168.250.7:443 ssl crt /etc/haproxy/certs/haproxy.pem
redirect scheme https if !{ ssl_fc }
http-request set-header X-forwarded-Port %[dst_port]
http-request add-header X-forwarded-Proto https if { ssl_fc }
##################### https end #####################################
mode http
use_backend web_port_http_nodes
log global

backend web_port_http_nodes
mode http
#balance static-rr
option forwardfor
server web1 192.168.250.17:80 check inter 3000 fall 2 rise 5 weight 1
server web2 192.168.250.27:80 check inter 3000 fall 2 rise 5 weight 1

######################## listen SQL Singlefile ##############################L SQL_PORT_3306
listen SQL_PORT_3306
bind 192.168.250.7:3306
mode tcp
log global
#balance static-rr
option forwardfor
server sql37 192.168.250.37:3306 check inter 3000 fall 2 rise 5 weight 2
server sql47 192.168.250.47:3306 check inter 3000 fall 2 rise 5 weight 1
[root@haproxy-ip07 certs]#

4.3 验证https

[root@CentOS84-IP68 ]#vim /etc/hosts
192.168.250.7 www.shone.cn

[root@CentOS84-IP68 ]#ping www.shone.cn
PING www.shone.cn (192.168.250.7) 56(84) bytes of data.
64 bytes from www.shone.cn (192.168.250.7): icmp_seq=1 ttl=64 time=0.278 ms
64 bytes from www.shone.cn (192.168.250.7): icmp_seq=2 ttl=64 time=0.346 ms
64 bytes from www.shone.cn (192.168.250.7): icmp_seq=3 ttl=64 time=0.262 ms
[root@CentOS84-IP68 ]#

[root@CentOS84-IP68 ]#curl -IkL http://www.shone.cn
HTTP/1.1 302 Found
content-length: 0
location: https://www.shone.cn/
cache-control: no-cache

HTTP/1.1 200 OK
date: Tue, 05 Apr 2022 08:54:28 GMT
server: Apache/2.4.6 (CentOS)
last-modified: Thu, 31 Mar 2022 09:16:22 GMT
etag: "1e-5db801d97cc28"
accept-ranges: bytes
content-length: 30
content-type: text/html; charset=UTF-8

[root@CentOS84-IP68 ]#curl -Ik https://www.shone.cn
HTTP/1.1 200 OK
date: Tue, 05 Apr 2022 08:54:20 GMT
server: Apache/2.4.6 (CentOS)
last-modified: Mon, 04 Apr 2022 10:25:33 GMT
etag: "1e-5dbd18c656fc8"
accept-ranges: bytes
content-length: 30
content-type: text/html; charset=UTF-8