题记：此文档是生产环境中实战之后的总结， 会根据项目实战情况陆续更新。但是不一定完全正确。欢迎大家指正！！！
要求：rhel6.3挂载HP磁阵上的vdisk
如果是做双机共用相同存储请看最后的说明
1.安装部署multipath
yum search multipath
yum install device-mapper-multipath.x86_64
找到multipath.conf文件，然后复制到/etc/下
[root@SCWRNOPDB1 ~]# find / -name "multipath.conf"
/usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf
[root@SCWRNOPDB1 ~]# cp /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf /etc/
编辑 multipath.conf
vim /etc/multipath.conf
在最后加入下面一段
device {
vendor “HP”
product "HSV2[01]0|HSV300|HSV4[05]0"
getuid_callout "/sbin/scsi_id -g -u -s /block/%n"
prio_callout "/sbin/mpath_prio_alua /dev/%n"
hardware_handler "0"
path_selector "round-robin 0"
path_grouping_policy group_by_prio
failback immediate
rr_weight uniform
rr_min_io 100
no_path_retry 18
path_checker tur
}
chkconfig multipathd on
service multipathd start
查看存储映射过来的磁盘，执行如下2个命令
(说明：磁阵上面vdisk划分省略，大家可以参考我的另外一个博文)
地址：http://myhoop.blog.51cto.com/5556534/1075110
[root@SCWRNOPDB1 ~]# multipath
Jul 16 00:01:46 | multipath.conf line 100, invalid keyword: device
Jul 16 00:01:46 | multipath.conf line 101, invalid keyword: vendor
Jul 16 00:01:46 | multipath.conf line 102, invalid keyword: product
Jul 16 00:01:46 | multipath.conf line 103, invalid keyword: getuid_callout
Jul 16 00:01:46 | multipath.conf line 104, invalid keyword: prio_callout
Jul 16 00:01:46 | multipath.conf line 105, invalid keyword: hardware_handler
Jul 16 00:01:46 | multipath.conf line 106, invalid keyword: path_selector
Jul 16 00:01:46 | multipath.conf line 107, invalid keyword: path_grouping_policy
Jul 16 00:01:46 | multipath.conf line 108, invalid keyword: failback
Jul 16 00:01:46 | multipath.conf line 109, invalid keyword: rr_weight
Jul 16 00:01:46 | multipath.conf line 110, invalid keyword: rr_min_io
Jul 16 00:01:46 | multipath.conf line 111, invalid keyword: no_path_retry
Jul 16 00:01:46 | multipath.conf line 112, invalid keyword: path_checker
Jul 16 00:01:46 | multipath.conf line 113, invalid keyword: }
Jul 16 00:01:47 | mpatha: ignoring map
[root@SCWRNOPDB1 ~]# multipath -ll
Jul 16 00:02:46 | multipath.conf line 100, invalid keyword: device
Jul 16 00:02:46 | multipath.conf line 101, invalid keyword: vendor
Jul 16 00:02:46 | multipath.conf line 102, invalid keyword: product
Jul 16 00:02:46 | multipath.conf line 103, invalid keyword: getuid_callout
Jul 16 00:02:46 | multipath.conf line 104, invalid keyword: prio_callout
Jul 16 00:02:46 | multipath.conf line 105, invalid keyword: hardware_handler
Jul 16 00:02:46 | multipath.conf line 106, invalid keyword: path_selector
Jul 16 00:02:46 | multipath.conf line 107, invalid keyword: path_grouping_policy
Jul 16 00:02:46 | multipath.conf line 108, invalid keyword: failback
Jul 16 00:02:46 | multipath.conf line 109, invalid keyword: rr_weight
Jul 16 00:02:46 | multipath.conf line 110, invalid keyword: rr_min_io
Jul 16 00:02:46 | multipath.conf line 111, invalid keyword: no_path_retry
Jul 16 00:02:46 | multipath.conf line 112, invalid keyword: path_checker
Jul 16 00:02:46 | multipath.conf line 113, invalid keyword: }
mpathd (36001438005df07aa0000600001050000) dm-3 HP,HSV400
size=500G features='1 queue_if_no_path' hwhandler='0' wp=rw
|-+- policy='round-robin 0' prio=50 status=active
| |- 0:0:0:5 sde 8:64 active ready running
| |- 0:0:2:5 sds 65:32 active ready running
| |- 2:0:0:5 sdag 66:0 active ready running
| `- 2:0:2:5 sdau 66:224 active ready running
`-+- policy='round-robin 0' prio=10 status=enabled
|- 0:0:1:5 sdl 8:176 active ready running
|- 2:0:1:5 sdan 66:112 active ready running
|- 0:0:3:5 sdz 65:144 active ready running
`- 2:0:3:5 sdbb 67:80 active ready running
mpathc (36001438005df07aa0000600000fd0000) dm-2 HP,HSV400
size=500G features='1 queue_if_no_path' hwhandler='0' wp=rw
|-+- policy='round-robin 0' prio=50 status=active
| |- 0:0:0:3 sdc 8:32 active ready running
| |- 0:0:2:3 sdq 65:0 active ready running
| |- 2:0:0:3 sdae 65:224 active ready running
| `- 2:0:2:3 sdas 66:192 active ready running
`-+- policy='round-robin 0' prio=10 status=enabled
|- 0:0:1:3 sdj 8:144 active ready running
|- 0:0:3:3 sdx 65:112 active ready running
|- 2:0:1:3 sdal 66:80 active ready running
`- 2:0:3:3 sdaz 67:48 active ready running
mpathb (36001438005df07aa0000600001110000) dm-4 HP,HSV400
size=1.0G features='1 queue_if_no_path' hwhandler='0' wp=rw
|-+- policy='round-robin 0' prio=50 status=active
| |- 0:0:1:1 sdh 8:112 active ready running
| |- 0:0:3:1 sdv 65:80 active ready running
| |- 2:0:1:1 sdaj 66:48 active ready running
| `- 2:0:3:1 sdax 67:16 active ready running
`-+- policy='round-robin 0' prio=10 status=enabled
|- 0:0:0:1 sda 8:0 active ready running
|- 0:0:2:1 sdo 8:224 active ready running
|- 2:0:0:1 sdac 65:192 active ready running
`- 2:0:2:1 sdaq 66:160 active ready running
上面显示的为存储路径聚合，mpathc mpathd 为新添加磁盘，如果查看不到，果断reboot。
[root@SCWRNOPDB2 mapper]# pwd
/dev/mapper
[root@SCWRNOPDB2 mapper]# ll
总用量 0
crw-rw---- 1 root root 10, 58 7月 17 11:42 control
lrwxrwxrwx 1 root root 7 7月 17 11:43 mpathb -> ../dm-2
lrwxrwxrwx 1 root root 7 7月 17 11:43 mpathc -> ../dm-4
lrwxrwxrwx 1 root root 8 7月 17 11:43 mpathd -> ../dm-12
lrwxrwxrwx 1 root root 7 7月 17 11:43 mpathe -> ../dm-7
lrwxrwxrwx 1 root root 8 7月 17 11:43 mpathf -> ../dm-14
lrwxrwxrwx 1 root root 8 7月 17 11:43 mpathg -> ../dm-13
lrwxrwxrwx 1 root root 7 7月 17 11:43 mpathh -> ../dm-3
mpathc -> ../dm-4
mpathd -> ../dm-12
这就是映射过来的设备
2.划分LVM磁盘：（用dm-4做例子，其他做法一样）
[root@SCWRNOPDB2 mapper]# fdisk /dev/dm-4
WARNING: DOS-compatible mode is deprecated. It's strongly recommended to
switch off the mode (command 'c') and change display units to
sectors (command 'u').
Command (m for help): n
Command action
e extended
p primary partition (1-4)
p
Partition number (1-4): 1
First cylinder (1-65270, default 1):
Using default value 1
Last cylinder, +cylinders or +size{K,M,G} (1-65270, default 65270):
Using default value 65270
Command (m for help): t
Selected partition 1
Hex code (type L to list codes): 8e
Changed system type of partition 1 to 8e (Linux LVM)
Command (m for help): w
The partition table has been altered!
Calling ioctl() to re-read partition table.
WARNING: Re-reading the partition table failed with error 22: 无效的参数.
The kernel still uses the old table. The new table will be used at
the next reboot or after you run partprobe(8) or kpartx(8)
Syncing disks.
查看是否正确：
[root@SCWRNOPDB2 mapper]# fdisk /dev/dm-4
WARNING: DOS-compatible mode is deprecated. It's strongly recommended to
switch off the mode (command 'c') and change display units to
sectors (command 'u').
Command (m for help): p
Disk /dev/dm-4: 536.9 GB, 536870912000 bytes
255 heads, 63 sectors/track, 65270 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 65536 bytes / 65536 bytes
Disk identifier: 0x0d969172
Device Boot Start End Blocks Id System
/dev/dm-4p1 1 65270 524281243+ 8e Linux LVM
Partition 1 does not start on physical sector boundary.
Command (m for help): ^C
上面可以看到已经分出来了Id是8e linux LVM
[root@SCWRNOPDB2 mapper]# partprobe
Warning: WARNING: the kernel failed to re-read the partition table on /dev/sda (设备或资源忙). As a result, it may not reflect all of your changes until after reboot.
device-mapper: remove ioctl on mpathdp5 failed: 设备或资源忙
Warning: parted was unable to re-read the partition table on /dev/mapper/mpathd (设备或资源忙). This means Linux won't know anything about the modifications you made.
device-mapper: remove ioctl on mpathcp5 failed: 设备或资源忙
Warning: parted was unable to re-read the partition table on /dev/mapper/mpathc (设备或资源忙). This means Linux won't know anything about the modifications you made.
partprobe重新生成分区表，会发现多出一个mpathcp1 -> ../dm-8
这个就是我们要做pv的对象，这是一个链接文件，做pv我们使用dm-8
[root@SCWRNOPDB2 mapper]# ll
总用量 0
crw-rw---- 1 root root 10, 58 7月 17 11:42 control
lrwxrwxrwx 1 root root 7 7月 17 11:43 mpathb -> ../dm-2
lrwxrwxrwx 1 root root 7 7月 17 11:43 mpathc -> ../dm-4
lrwxrwxrwx 1 root root 7 7月 17 15:06 mpathcp1 -> ../dm-8
lrwxrwxrwx 1 root root 8 7月 17 11:43 mpathd -> ../dm-12
lrwxrwxrwx 1 root root 8 7月 17 15:06 mpathdp1 -> ../dm-15
创建lvm
1）创建pv
pvcreate /dev/dm-8 /dev/dm-15 #pvdisplay查看
Writing physical volume data to disk "/dev/dm-8"
Physical volume "/dev/dm-8" successfully created
Writing physical volume data to disk "/dev/dm-15"
Physical volume "/dev/dm-15" successfully created
2) 创建vg
vgcreate ORAVG01 /dev/dm-8 /dev/dm-15 #vgdisplay查看
3) 创建lv
lvcreate -L 999G -n ORADATALV01 ORAVG01
4) 创建文件系统
mkfs.ext4 /dev/ORAVG01/ORADATALV01
5) 挂载
创建挂载目录 mkdir /oradata01
使用UUID挂载
blkid 查看UUID
vim /etc/fstab
UUID= /oradata01 ext4 defaults 0 0
保存退出
mount -a 如果没有报错 执行df -h查看文件系统是否挂载到了对应的目录
/dev/mapper/ORAVG01-ORADATALV01
984G 200M 934G 1% /oradata01
注意事项：
如果是做双机共享相同存储vdisk，那么只需要在其中一个主机上做LVM，做好LV之后在另外一个主机上就可以查到了。
切忌不要2个主机都做LVM那样文件系统就出问题了。（好吧~我第一次做就是犯了这个错误）
我是在DB1上做的LVM，lvdisplay查看DB1 和DB2
文件系统共享挂载举例：
在其中一台主机上作出lv之后，格式化成文件系统。
mkfs.ext4 /dev/ORAVG03/lv03
另外一台上面执行：lvsan
lvchange -a y /dev/ORAVG03/lv03 激活
查看UUID:
[root@SCWRNOPDB1 /]# blkid |tail -n 5
/dev/mapper/mpathgp1: UUID="H6yl3J-p1ED-cZzR-Gkef-M1Cg-CwMe-dt4240" TYPE="LVM2_member"
/dev/mapper/mpathep1: UUID="OGxRgE-BUUe-TfKv-VL9R-pvZt-dWKJ-cmZKuJ" TYPE="LVM2_member"
/dev/mapper/mpathfp1: UUID="PTDNBt-uprU-06xH-WWFM-B9S2-na1J-647E8a" TYPE="LVM2_member"
/dev/loop0: LABEL="RHEL_6.3 x86_64 Disc 1" TYPE="iso9660"
/dev/mapper/ORAVG03-lv03: UUID="257114ed-a2f5-4d23-a2d1-4080d079318b" TYPE="ext4"
[root@SCWRNOPDB2 /]# blkid |tail -n 5
/dev/mapper/vg00-lvusr: UUID="9258f499-1e51-4de9-81a0-6e11bbf4b152" TYPE="ext4"
/dev/sda1: SEC_TYPE="msdos" UUID="A3FB-0754" TYPE="vfat"
/dev/sda2: UUID="3979054d-eb80-4063-bfa8-8c77df48078c" TYPE="ext4"
/dev/sda3: UUID="dvgQPh-qWMF-GduO-BnUz-r1OV-OCJ7-dz1ULS" TYPE="LVM2_member"
/dev/mapper/ORAVG03-lv03: UUID="257114ed-a2f5-4d23-a2d1-4080d079318b" TYPE="ext4"
lv03的UUID一致
挂载文件系统：
[root@SCWRNOPDB1 /]# mount /dev/mapper/ORAVG03-lv03 /tt/
[root@SCWRNOPDB1 /]# df -h
文件系统 容量 已用 可用 已用%% 挂载点
/dev/mapper/ORAVG03-lv03
12M 1.2M 10M 11% /tt
[root@SCWRNOPDB2 /]# mount /dev/mapper/ORAVG03-lv03 /tt01/
[root@SCWRNOPDB2 /]# df -h
文件系统 容量 已用 可用 已用%% 挂载点
/dev/mapper/ORAVG03-lv03
12M 1.2M 10M 11% /tt01
--myhoop 于成都
首次发布：2013年7月16日00:26
第一次修改：2013年7月18日11:36