Optimize filesystem performance on top of mdadm raid5 + LUKS

wallbunny · 11-22-2010, 02:22 PM

I never got a reply but here are some links you may find useful if you are trying to optimize your disk layout:
http://wiki.centos.org/HowTos/Disk_Optimization <- Some info about correct stride and stripe-width for ext filesystems.
http://alephnull.com/benchmarks/sata2009/index.html <- A bit of everything
http://sunsite.uakom.sk/sunworldonli...-09-raid5.html <- Only info about raid
http://everything2.com/node/1479435 <- Only for XFS

Here are the 'rules' I've found. Some raid-10 is still a big mystery, but from what I can tell it should not hurt performance much using a spindle count equal to disk count.

Spindle is number of disks carrying data:
5 drive raid-0: n=5
5 drive raid-1: n=1
5 drive raid-5: n=4
5 drive raid-6: n=3
5 drive raid-10:n=5 BUT 6 drive raid-10:n=3

ext4
stride=(chunk/4096)
stripe-width = stride * Spindle

xfs
sunit=(chunk/512)
swidth = sunit * Spindle

luks
align-payload = (chunk/512) * Spindle

This is how I've chosen to set up my new system with 5*2Tb 4096b-sector disks:

Quote:

#part1 64Mb #boot on RAID-1
#part2 2Gb(5Gb) #swap on RAID-10
#part3 512Mb #root on RAID-1
#part5 10Gb(50Gb) #contains a single partition on top of RAID-0. I will symlink /tmp , /var/tmp and such to folders here.
#part6 10Gb(25Gb) #contains LVM on RAID-10. The lvm will contain my system partitions
#part7 REST #contains a single partition on top of LUKS on top of RAID-5. This will hold my larger files such as movies.

mdadm --create /dev/md1 --level=1 --auto=md --chunk=128 --raid-devices=5 --metadata=0.90 /dev/sda1 /dev/sdb1 /dev/sdc1 /dev/sdd1 missing
mdadm --create /dev/md2 --level=10 --auto=md --chunk=64 --raid-devices=5 --layout=n2 /dev/sda2 /dev/sdb2 /dev/sdc2 /dev/sdd2 missing
mdadm --create /dev/md3 --level=10 --auto=md --chunk=128 --raid-devices=5 --layout=n2 --metadata=0.90 /dev/sda3 /dev/sdb3 /dev/sdc3 /dev/sdd3 missing
mdadm --create /dev/md5 --level=0 --auto=md --chunk=128 --raid-devices=5 /dev/sda5 /dev/sdb5 /dev/sdc5 /dev/sdd5 missing
mdadm --create /dev/md6 --level=10 --auto=md --chunk=128 --raid-devices=5 --layout=f2 /dev/sda6 /dev/sdb6 /dev/sdc6 /dev/sdd6 missing
mdadm --create /dev/md7 --level=5 --auto=md --chunk=128 --raid-devices=5 --layout=left-symmetric /dev/sda7 /dev/sdb7 /dev/sdc7 /dev/sdd7 missing

mkfs.ext4 -b 4096 -O ^has_journal -L boot /dev/md1
mkswap /dev/md2
mkfs.ext4 -b 4096 -O has_journal -L root /dev/md3
mkfs.ext4 -b 4096 -E stride=32,stripe-width=160 -L Stripe /dev/md5
pvcreate /dev/md6
vgcreate -s 16M System /dev/md6
lvcreate -n home -l 32 System #512m
lvcreate -n var -l 256 System #4G
lvcreate -n usr -l 512 System #8G
mkfs.ext4 -b 4096 -O has_journal -E stride=32,stripe-width=160 /dev/mapper/System-***
cryptsetup create cryptMedia /dev/md7 --cipher=aes-xts-plain64 --hash=sha256 --key-size=256 --align-payload=1024 --key-file=passwordfile
mkfs.xfs -b 4096 -s 4096 -d sunit=256 -d swidth=1024 -l size=64m -L Media /dev/mapper/cryptMedia

#fstab
/dev/md2 none swap sw 0 0
/dev/md3 / ext4 noatime,exec,rw 0 1
/dev/md1 /boot ext4 noatime,noauto 0 2
/dev/md5 /mnt/Speed ext4 noatime,nodev,nosuid,noexec,rw 0 3
/dev/mapper/System-usr /usr ext4 noatime,nodev,exec,rw 0 4
/dev/mapper/System-home /home ext4 noatime,nodev,nosuid,noexec,rw 0 5
/dev/mapper/System-var /var ext4 noatime,nodev,nosuid,noexec,rw 0 6
/dev/mapper/cryptMedia /mnt/Media xfs noatime,nodev,nosuid,noexec,rw,logbsize=64k,logbufs=12,allocsize=512m,swidth=1024,sunit=256 0 7