Tuesday, January 20, 2009

20090120 - A problem with Linux - device naming

One of the issues that really annoys me is the fact that some RH-based distributions (RHEL, CentOS, Oracle EL, Oracle VM), name SCSI devices in the order that they find them, in the controller order.

So, if you're doing an installation on /dev/cciss/c0d0p0, but you're also connected to a SAN, it's possible that the SAN LUNs will be found before the internal disk.

The normal recommendation? To disconnect the SAN HBA fibre connections, but there's a better way:

In the initrd.img file of the installation medium, the driver files are encapsulated. The easiest way to stop the SAN LUNs being found is to remove the SAN drivers.

eg:

ESX 3.x (probably the simplest to understand) :

unalias cp 2> /dev/null
unalias rm 2> /dev/null
unalias mv 2> /dev/null
mount -o ro,loop /export/isos/esx-3.5.0_Update_1-82663.iso /mnt
cd /mnt
mkdir /tmp/esxnab
cp images/boot.iso /tmp
cd /
umount /mnt
mount -o ro,loop /tmp/boot.iso /mnt
cp /mnt/isolinux/vmlinuz /tmp/esxnab
cp /mnt/isolinux/initrd.img /tmp/esxnab
umount /mnt
cp /tmp/esxnab/initrd.img /tmp/esxnab/initrd.img.orig
gunzip < /tmp/esxnab/initrd.img > /tmp/initrd
mount -o rw,loop /tmp/initrd /mnt
mkdir /tmp/newmods
cd /tmp/newmods
gunzip < /mnt/modules/modules.cgz | cpio -id
rm rm 2.4.21-47.0.1.ELBOOT/i386/qla2*
find . | cpio -ocv | gzip -9 > /mnt/modules/modules.cgz
cd /
rm -rf /tmp/newmods
umount /mnt
gzip -9 < /tmp/initrd > /tmp/esxnab/initrd.img


RedHat RHEL 3 :

mkdir /tmp/mnt1
mkdir /tmp/mnt2
mkdir /tmp/mnt3
cat /export/AS3/images/bootdisk.img > /tmp/linuxdisk1
cat /export/AS3/images/drvnet.img > /tmp/linuxdisk2
mount -o loop,rw /tmp/linuxdisk1 /tmp/mnt1

zcat < /tmp/mnt1/initrd.img > /tmp/initrd.img
mount -o loop,rw /tmp/initrd.img /tmp/mnt2
cp -Rp /tmp/mnt2 /tmp/initrd.new
cd /tmp/initrd.new/modules
zcat < modules.cgz | cpio -ivd

mount -o loop,ro /tmp/linuxdisk2 /tmp/mnt3
mkdir /tmp/newmods
cd /tmp/newmods
zcat < /tmp/mnt3/modules.cgz | cpio -ivd

cp 2*BOOT/i386/*.o /tmp/initrd.new/modules/2*BOOT/i386/
cd /tmp/initrd.new/modules/2*BOOT/i386
############# REMOVE UNNECESSARY MODULES HERE #############
rm -f usb-storage.o
rm -f nfs.o sunrpc.o lockd.o
rm -f olympic.o tulip.o dl2k.o natsemi.o
rm -f acenic.o dmfe.o epic100.o ewrk3.o
rm -f ns83820.o sis900.o tlan.o via-rhine.o
rm -f fealnx.o b44.o amd8111e.o pcnet_cs.o
rm -f fbcon-vga-planes.o vga16fb.o
rm -f typhoon.o xircom_cb.o pcmcia_core.o
rm -f sr_mod.o e100.o 8139cp.o 8139too.o ds.o r8169.o

cat /tmp/mnt3/pcitable >> /tmp/mnt2/modules/pcitable
sort -u -o /tmp/mnt2/modules/pcitable /tmp/mnt2/modules/pcitable
cat /tmp/mnt3/modules.dep >> /tmp/mnt2/modules/modules.dep
sort -u -o /tmp/mnt2/modules/modules.dep /tmp/mnt2/modules/modules.dep
cat /tmp/mnt3/modinfo >> /tmp/mnt2/modules/module-info
############# EDIT MODULE INFORMATION HERE #############
vi /tmp/mnt2/modules/module-info

cd ../..
find 2*BOOT/ -print | cpio -ov -H crc | gzip -c9 > /tmp/mnt2/modules/modules.cgz
cd /tmp
umount /tmp/mnt[123]
gzip -9 /tmp/initrd.img

dd if=/dev/zero of=/tmp/AS3.img bs=18k count=80
mkfs.vfat /tmp/AS3.img
syslinux /tmp/AS3.img
mount -o rw,loop /tmp/AS3.img /tmp/mnt1
cp /tmp/initrd.img.gz /tmp/mnt1/initrd.img
cp /export/configs/files/AS3/* /tmp/mnt1
cd /tmp
umount /tmp/mnt1

rm -rf /tmp/newmods
rm -f /tmp/initrd.img.gz /tmp/initrd.img
rmdir /tmp/mnt[123]
rm -rf /tmp/initrd.new
rm -f /tmp/linuxdisk?

cp AS3.img /export/NAB/kickstart_images/


RedHat RHEL 4 :

cp /export/AS4/images/pxeboot/initrd.img /export/AS4/images/pxeboot/initrd.img.orig
cp /export/AS4/RedHat/base/netstg2.img /export/AS4/RedHat/base/netstg2.img.orig
cp /export/AS4/images/boot.iso /export/AS4/images/boot.iso.orig

# initrd
gunzip < /export/AS4/images/pxeboot/initrd.img > /tmp/initrd
mount -o loop /tmp/initrd /mnt
sed 's/"e1000"/"ignore"/;s/"qla.*"/"ignore"/' < /mnt/modules/pcitable > /tmp/x
cat /tmp/x > /mnt/modules/pcitable
umount /mnt
gzip -9 < /tmp/initrd > /export/AS4/images/pxeboot/initrd.img
rm -f /tmp/initrd /tmp/x

# netstg2
mount -o ro,loop /export/AS4/RedHat/base/netstg2.img /mnt
mkdir /tmp/test
(cd /mnt ; find . -print | cpio -dump /tmp/test)
umount /mnt
sed 's/"e1000"/"ignore"/;s/"qla.*"/"ignore"/' < /tmp/test/modules/pcitable \
> /tmp/x
cat /tmp/x > /tmp/test/modules/pcitable
mkcramfs /tmp/test /tmp/netstg2.img
\cp /tmp/netstg2.img /export/AS4/RedHat/base/
rm -rf /tmp/netstg2.img /tmp/test /tmp/x

# make bootable cdimage:
mount -o ro,loop /export/AS4/images/boot.iso /mnt
mkdir /tmp/test
(cd /mnt ; find . -print | cpio -dump /tmp/test)
umount /mnt
\cp /export/AS4/images/pxeboot/initrd.img /tmp/test/isolinux/
printf "label kickstart\n kernel vmlinuz\n append initrd=initrd.img \
ramdisk_size=8192 text nofb ks=http://genesis.global.thenational.com/\
configs/ks4.cfg\n" >> /tmp/test/isolinux/isolinux.cfg
sed 's/^default .*/default kickstart/' < /tmp/test/isolinux/isolinux.cfg \
> /tmp/x
cat /tmp/x > /tmp/test/isolinux/isolinux.cfg
printf "\x0C\n\n - To install Linux, wait 60 seconds, or press \x00F0f\
\x00F07.\n\n\x00F02[F1-Main] [F2-Options] [F3-General] [F4-Kernel] \
[F5-Rescue]\x00F07\n" > /tmp/test/isolinux/boot.msg
(cd /tmp/test ; mkisofs -o /tmp/boot.iso -b isolinux/isolinux.bin \
-c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table . )
\cp /tmp/boot.iso /export/AS4/images/
rm -rf /tmp/test /tmp/boot.iso /tmp/x


ESX 2.1 :

unalias rm
unalias cp
unalias mv
mkdir /tmp/mnt{1,2,3}
mount -o loop,ro /export/ESX210/images/bootnet.img /tmp/mnt1
mount -o loop,ro /export/ESX210/images/drvnet.img /tmp/mnt2
zcat < /tmp/mnt1/initrd.img > /tmp/initrd.img
mount -o loop,rw /tmp/initrd.img /tmp/mnt3
cp -Rp /tmp/mnt3 /tmp/initrd.new
cd /tmp/initrd.new/modules
zcat < modules.cgz | cpio -ivd

mkdir /tmp/newmods
cd /tmp/newmods
zcat < /tmp/mnt2/modules.cgz | cpio -ivd

#### drvnet is wrong here. Contains 2.4.9-13BOOT directory
### cp 2*BOOT/*.o /tmp/initrd.new/modules/2*BOOT
cp *.o /tmp/initrd.new/modules/2*BOOT
cd /tmp/initrd.new/modules/2*BOOT
############# REMOVE UNNECESSARY MODULES HERE #############
rm -f 3c501.o 3c503.o 3c505.o 3c507.o 3c515.o
rm -f usb*.o hid.o
rm -f abyss.o ibmtr.o lanstreamer.o olympic.o plip.o tmspci.o
rm -f nfs.o sunrpc.o lockd.o
rm -f tulip*.o depca.o de4x5.o
rm -f de600.o de620.o dgrs.o dl2k.o natsemi.o
rm -f 82596.o ac3200.o acenic.o arlan.o at1700.o dmfe.o e2100.o epic100.o
rm -f es3210.o eth16i.o ewrk3.o hamachi.o hp.o hp-plus.o hp100.o lance.o
rm -f lne390.o ne3210.o ni5010.o ni52.o ni65.o ns83820.o sis900.o sk98lin.o
rm -f sundance.o sunhme.o tlan.o via-rhine.o wavelan.o wd.o winbond-840.o
rm -f yellowfin.o
rm -f parport*.o
rm -f cs89x0.o tms380tr.o

cat /tmp/mnt2/pcitable > /tmp/pcitable
cat /tmp/mnt3/modules/pcitable >> /tmp/pcitable
egrep -v '"3c90x"|"8139too"|"aic7xxx"|"bcm5700"|"e100"|"e1000"|"megaraid"|"ne2k-pci"|"pcnet32"' /tmp/pcitable | awk -F'\t' '{if (NF==4) {print $1 "\t" $2 "\t" "\"ignore\"" "\t" $4} else if (NF==6) {print $1 "\t" $2 "\t" $3 "\t" $4 "\t" "\"ignore\"" "\t" $6}}' > /tmp/pcitable.ignore
sort -u -o /tmp/pcitable.ignore /tmp/pcitable.ignore
egrep '"3c90x"|"8139too"|"aic7xxx"|"bcm5700"|"e100"|"e1000"|"megaraid"|"ne2k-pci"|"pcnet32"' /tmp/pcitable > /tmp/pcitable.keep
cat /tmp/pcitable.keep /tmp/pcitable.ignore > /tmp/mnt3/modules/pcitable
cat /tmp/mnt2/modules.dep >> /tmp/mnt3/modules/modules.dep
sort -u -o /tmp/mnt3/modules/modules.dep /tmp/mnt3/modules/modules.dep
cat /tmp/mnt2/modinfo >> /tmp/mnt3/modules/module-info
############# EDIT MODULE INFORMATION HERE #############
vi /tmp/mnt3/modules/module-info

cd /tmp/initrd.new/modules
find 2*BOOT/ -print | cpio -ov -H crc | gzip -c9 > /tmp/mnt3/modules/modules.cgz
cp /export/configs/ksesx210.cfg /tmp/mnt3/ks.cfg
sed 's/50000/10000/' < /export/configs/ksesx210.cfg > /tmp/mnt3/kssmall.cfg
cd /tmp
umount /tmp/mnt{1,2,3}
gzip -9 /tmp/initrd.img

dd if=/dev/zero of=/tmp/ESX210.img bs=18k count=80
mkfs.vfat /tmp/ESX210.img
syslinux /tmp/ESX210.img
mount -o rw,loop /tmp/ESX210.img /tmp/mnt1
cp /tmp/initrd.img.gz /tmp/mnt1/initrd.img
cp /export/configs/files/ESX210/* /tmp/mnt1
cd /tmp
umount /tmp/mnt1

rm -rf /tmp/newmods
rm -f /tmp/initrd.img.gz /tmp/initrd.img
rm -f /tmp/pcitable*
rmdir /tmp/mnt[123]
rm -rf /tmp/initrd.new

mv /tmp/ESX210.img /export/NAB/kickstart_images/
cd /tftpboot
make