小米路由系统启动过程

1 Bootloader

Broadcom 出的片子，一直用自己的 Bootloader，叫 CFE，全称 Common Firmware Environment

原来写过一个 CFE Quick Start，在这：http://www.jackslab.org/?portfolio=cfe-quick-start

小米路由系统 0.5.41 的 CFE 启动过程：

CFE version v1.0.4
BSP: 6.37.14.34 (r415984) based on BBP 1.0.37 for BCM947XX (32bit,SP,)
Build Date: Wed Apr 30 18:03:21 CST 2014 (szy@shenzhiyong-ct)
Copyright (C) 2000-2008 Broadcom Corporation.
 
Init Arena
Init Devs.
Boot up from NOR flash...
Boot partition size = 262144(0x40000)
Can't find nandflash! ccrev = 42, chipst= 0
DDR Clock: 800 MHz
Info: DDR frequency set from clkfreq=1000,*800*
et0: Broadcom BCM47XX 10/100/1000 Mbps Ethernet Controller 6.37.14.34 (r415984)
CPU type 0x0: 1000MHz
Tot mem: 262144 KBytes
 
CFE mem:    0x00F00000 - 0x0179FE38 (9043512)
Data:       0x00F58390 - 0x00F588C8 (1336)
BSS:        0x00F588D8 - 0x00F9DE38 (284000)
Heap:       0x00F9DE38 - 0x0179DE38 (8388608)
Stack:      0x0179DE38 - 0x0179FE38 (8192)
Text:       0x00F00000 - 0x00F4CCC4 (314564)
Boot:       0x017A0000 - 0x017E0000
Reloc:      I:00000000 - D:00000000
 
Device eth0:  hwaddr 8C-BE-BE-20-B7-48, ipaddr 192.168.1.1, mask 255.255.255.0
        gateway not set, nameserver not set
bootargs: boot -raw -z -addr=0x8000 -max=0xef8000 flash0.os:
Loader:raw Filesys:raw Dev:flash0.os File: Options:(null)
Loading: ..... 5372256 bytes read
Entry at 0x00008000
Closing network.
Starting program at 0x00008000

可以看到其启动后，直接加载 flash0.os 分区（/dev/mtdblock2 名为 'os'）的 raw 格式的内核文件到地址 0x8000 处，然后直接跳转到这个地址开始执行

2 内核启动参数

# cat /proc/cmdline
root=/dev/ram rw console=ttyS0,115200 init=/init libata.force=3.0Gbps earlyprintk debug

从这可以看到，内核启动时首先挂载的是一个 initramfs 内存文件系统，执行的第一个程序是 /init 这个脚本

3 /init 启动脚本

#!/bin/sh
# Copyright (C) 2006-2012 OpenWrt.org
#
#

klogger(){
        local msg="$@"
        test -z "$msg" && return 1

        echo "$msg" >> /dev/kmsg 2>/dev/null
}

usb_check_for_rescure(){
        local gpiobtn

        gpiobtn=`cat /proc/xiaoqiang/reset`
        rst=`gpio $gpiobtn | awk '{print $3}'`
        if [ "$rst" = "<0>" ]; then
                nvram set flag_reboot_system=2
                exec /sbin/init
        fi
}

getmtdblockdevbyname(){
        local name
        name="$1"
        test -z "$name" && return 1
        devnum=$(cat /proc/mtd | awk "/\"$name\"/"| awk -F':' '{print $1}' | awk -F'mtd' '{print $2}')
        test -z "$devnum" && return 1
        if [ -e "/dev/mtdblock${devnum}" ]
                then
                echo "/dev/mtdblock${devnum}"
                return 0
        else
                return 1
        fi
}

reboot() {
        nvram set flag_reboot_system=1
        exec /sbin/init
}

updone() {
        nvram set flag_package_update=0
        nvram set telnet_en=1
        nvram commit

        klogger "Update successfully!!! Reboot the System..."
        reboot
}

upnet() {
        #not vlan support in bootloader and ramfs, using trunk mode
        #set to "0 1 2 5u" for ramfs network only, do not commit
        rmmod et 2>/dev/null
        nvram set vlan1ports="0 2 5u"
        nvram set vlan2ports="4 5"

        mac=`nvram get et0macaddr`
        if [ "$mac" = "00:90:4C:0F:F2:a7" ]; then
                uuid=`cat /proc/sys/kernel/random/uuid`
                b1=`echo $uuid | cut -b 1-2`
                b2=`echo $uuid | cut -b 3-4`
                b3=`echo $uuid | cut -b 5-6`
                mac=`echo 00:90:4C:$b1:$b2:$b3`
                nvram set "pci/1/1/macaddr=$mac"

                uuid=`cat /proc/sys/kernel/random/uuid`
                b1=`echo $uuid | cut -b 1-2`
                b2=`echo $uuid | cut -b 3-4`
                b3=`echo $uuid | cut -b 5-6`
                mac=`echo 00:90:4C:$b1:$b2:$b3`
                nvram set "pci/2/1/macaddr=$mac"

                uuid=`cat /proc/sys/kernel/random/uuid`
                b1=`echo $uuid | cut -b 1-2`
                b2=`echo $uuid | cut -b 3-4`
                b3=`echo $uuid | cut -b 5-6`
                mac=`echo 00:90:4C:$b1:$b2:$b3`
                nvram set "et0macaddr=$mac"
        fi
        insmod /lib/modules/et.ko

        lip=`nvram get flag_local_ip`
        if [ "$lip" = "auto" ]; then
                /sbin/udhcpc -t 10 -n
                if [ $? -eq 1 ]; then
                        lip="192.168.31.1"

                        ifconfig eth0 $lip up
                fi
        else
                [ -n "$lip" ] || lip="192.168.31.1"
                ifconfig eth0 $lip up
        fi
        echo 1 > /proc/sys/net/ipv6/conf/eth0/forwarding

        ifconfig lo up
        nvram set vlan1ports="0 2 5*"
        nvram set vlan2ports="4 5"
}

uperror() {
        klogger "SQUASHFS Filesystem Error!!!"
        klogger "Switch to RAMFS..."

        upnet
        exec /sbin/init
}

squashfs() {
        klogger "Switch to SQUASHFS..."

        reboot

        rescuerootdev=$(getmtdblockdevbyname squashfs)
        if [ -n "$rescuerootdev" ]
                then
                mount -t squashfs $rescuerootdev /mnt
                if [ $? -ne 0 ]
                then
                        klogger "ERROR: mount -t squashfs $rescuerootdev /mnt failed."
                else
                        # boot_status: 1 sata mode
                        #              2 flash mode
                        echo 2 > /sys/power/boot_status
                        upmount
                        if [ -x /mnt/etc/rescueinit ]
                                then
                                klogger "switch to rescue system($rescuerootdev) by /etc/rescueinit ..."
                                exec switch_root /mnt /etc/rescueinit
                        fi
                        if [ -x /mnt/lib/preinit.sh ]
                                then
                                klogger "switch to rescue system($rescuerootdev) by /lib/preinit.sh ..."
                                exec switch_root /mnt /lib/preinit.sh
                        fi
                        umount -f $rescuerootdev
                fi
        fi
        uperror
}

upmount() {
        mount -n -o move /tmp /mnt/tmp
        mount -n -o move /sys /mnt/sys
        mount -n -o move /proc /mnt/proc
        mount -n -o move /dev /mnt/dev
}

[ -d /dev ] || mkdir -m 0755 /dev
[ -d /mnt ] || mkdir -m 0700 /mnt
[ -d /sys ] || mkdir /sys
[ -d /proc ] || mkdir /proc
[ -d /tmp ] || mkdir /tmp

mkdir -p /var/lock
mount -t sysfs -o nodev,noexec,nosuid none /sys 
mount -t proc -o nodev,noexec,nosuid none /proc 

klogger "Loading, please wait..."

# Note that this only becomes /dev on the real filesystem if udev's scripts
# are used; which they will be, but it's worth pointing out
if ! mount -t devtmpfs -o mode=0755 none /dev; then
        mount -t tmpfs -o mode=0755 none /dev
        mknod -m 0600 /dev/console c 5 1
        mknod /dev/null c 1 3
fi
mount -t tmpfs -o "nosuid,size=20%,mode=0755" tmpfs /tmp

klogger "Loading essential drivers..."
flag_usb_rootdisk_enable="$(nvram get flag_usb_rootdisk_enable)"
for i in `cat /lib/modules/mod.lst`;
do
        [ "$i" = "usb-storage.ko" -a "$flag_usb_rootdisk_enable" != 'true' ] && continue
        if [ "$i" != "et.ko" ] ; then
                insmod /lib/modules/$i || nvram set flag_package_update=5
        fi
done

# Check for USB rescure
klogger "Check for USB rescure..."
usb_check_for_rescure

model=`nvram get model`
if [ -z "$model" ]; then
        model=`cat /proc/xiaoqiang/model`
fi

if [ "$model" = "R1AC" ]; then
        insmod /lib/modules/usb-storage.ko 2>/dev/null
        sleep 3
fi

#support tftpboot from network
if [ "$(nvram get flag_tftp_bootup)" = 'on' -a "$(nvram get flag_tftp_booted)" = 'true' ]
then
        . /tftpboot.init
        tftpboot_mount_rootfs
fi

klogger "Press Ctrl+C to enter RAMFS..."
rd -w 1 && nvram set flag_package_update=5

if [ ! -b /dev/sda ]; then
        klogger "Warning: No Hard Disk"

        nvram set flag_no_hdd=1
        nvram commit

        squashfs
fi

update=`nvram get flag_package_update`
ft_mode=`cat /sys/power/ft_mode`
if [ -n "$update" -a "$update" != "0" ]; then
        #

        klogger "Prepare the system to update..."
        upnet

        if [ $update = "5" ]; then
                exec /sbin/init
        else
                /usr/sbin/autofd
                if [ $? -eq 1 ]; then
                        gpio 2 1
                        gpio 3 1
                        gpio 1 0

                        if [ "$ft_mode" = "1" ]; then
                                exec /sbin/init
                        else
                                while true
                                do
                                                /usr/sbin/update
                                                retcode=$?

                                                [ $retcode -eq 2 ] && exec /sbin/init
                                                [ $retcode -eq 0 ] && updone
                                done
                        fi
                else
                        updone
                fi
        fi
else
        klogger "Bringup the system..."

        flag_try_sys1=`nvram get flag_try_sys1_failed`
        flag_try_sys2=`nvram get flag_try_sys2_failed`

        if [ "$flag_try_sys1" != "1" ] || [ "$flag_try_sys2" != "1" ]; then
                # default: rootfs1
                flag=`nvram get flag_boot_rootfs`
                if [ "$flag" = "1" ]; then
                        mntdev=/dev/sda2
                else
                        mntdev=/dev/sda1
                fi
                [ -b $mntdev ] && mount -o ro $mntdev /mnt

                # flag_boot_type: 1 system in SATA   version
                #                 2 system in SQUASH version
                #                 9 system in tftp version
                nvram set flag_boot_type=1
                if [ -x /mnt/lib/preinit.sh ]; then
                        # boot_status: 1 sata mode
                        #              2 flash mode
                        upmount
                        exec switch_root /mnt /lib/preinit.sh
                else
                        if [ "$ft_mode" = "1" ]; then
                                # skip failed flag setting in FT mode
                                reboot
                        else
                                if [ "$flag" = "1" ]; then
                                        nvram set flag_try_sys2_failed=1
                                else
                                        nvram set flag_try_sys1_failed=1
                                fi
                                nvram set flag_ota_reboot=0
                                nvram commit

                                reboot
                        fi
                fi
        fi

        # reset all failed flag in FT mode
        if [ "$ft_mode" = "1" ]; then
                nvram set flag_try_sys1_failed=0
                nvram set flag_try_sys2_failed=0
                nvram commit

                reboot
        fi

        squashfs
fi

可以看到正常流程下，/dev/sda1 是作为根文件系统，/dev/sda2 作为一个备份的根文件系统。从 "Bringup the system..." 开始，其流程为：

先将 /dev/sda1 挂载到 ramfs 的 /mnt 目录下，然后 switch_root (类 chroot) 到 /mnt 下，同时执行 /mnt/lib/preinit.sh

4 /lib/preinit.sh

#!/bin/sh
# Copyright (C) 2006 OpenWrt.org
# Copyright (C) 2010 Vertical Communications

export PATH=/bin:/sbin:/usr/bin:/usr/sbin

pi_ifname=
pi_ip=192.168.31.1
pi_broadcast=192.168.31.255
pi_netmask=255.255.255.0

fs_failsafe_ifname=
fs_failsafe_ip=192.168.31.1
fs_failsafe_broadcast=192.168.31.255
fs_failsafe_netmask=255.255.255.0

fs_failsafe_wait_timeout=2

pi_suppress_stderr=
pi_init_suppress_stderr=
pi_init_path="/bin:/sbin:/usr/bin:/usr/sbin"
pi_init_cmd="/sbin/init"

. /lib/functions.sh
. /lib/functions/boot.sh

boot_hook_init preinit_essential
boot_hook_init preinit_main
boot_hook_init failsafe
boot_hook_init initramfs
boot_hook_init preinit_mount_root

for pi_source_file in /lib/preinit/*; do
    . $pi_source_file
done

boot_run_hook preinit_essential

pi_jffs2_mount_success=false
pi_failsafe_net_message=false

boot_run_hook preinit_main

ulimit -Hn 50000
ulimit -Sn 50000

可以看到，主要的工作脚本都在 /lib/preinit/ 这个目录下的脚本集：

# ls lib/preinit/
00_extroot.conf                10_indicate_failsafe  30_failsafe_wait       40_run_failsafe_hook         55_determine_extroot_sysupgrade  90_init_console
02_default_set_state           10_indicate_preinit   31_check_for_boottype  41_merge_overlay_hooks       60_init_hotplug                  90_mount_bind_etc
03_init_hotplug_failsafe_brcm  15_mount_proc_brcm    31_restore_nvram       42_format_ext_part           60_pivot_usb_root                90_mount_no_jffs2
05_mount_skip                  20_check_jffs2_ready  40_init_shm            50_choose_console            70_initramfs_test                99_10_failsafe_login
10_check_for_mtd               20_device_fs_mount    40_mount_devpts        50_determine_usb_root        70_pivot_jffs2_root              99_10_mount_no_mtd
10_essential_fs                30_device_fs_daemons  40_mount_jffs2         50_indicate_regular_preinit  80_mount_root                    99_10_run_init

5 最终文件系统结构

文件 lib/preinit/42_format_ext_part 显示：

  /dev/sda3 挂载到 /data
  /dev/sda4 挂载到 /userdisk

文件 lib/preinit/90_mount_bind_etc 显示：

97         mount --bind /data/etc /etc

即：/dev/sda3/etc 是 'mount --bind' 到 /etc 下的

同时当用户在 web 管理界面里格式化磁盘时，实际格式化的是 /dev/sda3 和 /dev/sda4，格式化完了后，其会将 /dev/sda1/etc 同步到 /dev/sda3/etc

最终的目录结构是这样的：

/dev/sda1 on / type ext4 (ro,relatime,barrier=1,data=ordered)
/dev/sda3 on /data type ext4 (rw,noatime,barrier=1,data=ordered)
/dev/sda4 on /userdisk type ext4 (rw,noatime,barrier=1,data=ordered)
/dev/sda3 on /etc type ext4 (rw,noatime,barrier=1,data=ordered)