SMP多核启动
在Linux系统中，对于多核的ARM芯片而言，在Bootrom代码中，每个CPU都会识别自身ID，如果ID是0，则引导Bootloader和Linux内核执行，如果ID不是0，则Bootrom一般在上电时将自身置于WFI或者WFE状态，并等待CPU0给其发CPU核间中断或事件（一般通过SEV指令）来唤醒它。下图是一个典型的多核Linux启动过程。
CPU0唤醒其他CPU的动作在内核中被封装为一个 smp_operations 的结构体，对于ARM而言，它定义在 arch/arm/include/asm/smp.h 中。该结构体的成员函数如下：
1 struct smp_operations { 2 #ifdef CONFIG_SMP 3 /* 4 * Setup the set of possible CPUs (via set_cpu_possible) 5 */ 6 void (*smp_init_cpus)(void); 7 /* 8 * Initialize cpu_possible map, and enable coherency 9 */ 10 void (*smp_prepare_cpus)(unsigned int max_cpus); 11 12 /* 13 * Perform platform specific initialisation of the specified CPU. 14 */ 15 void (*smp_secondary_init)(unsigned int cpu); 16 /* 17 * Boot a secondary CPU, and assign it the specified idle task. 18 * This also gives us the initial stack to use for this CPU. 19 */ 20 int (*smp_boot_secondary)(unsigned int cpu, struct task_struct *idle); 21 #ifdef CONFIG_HOTPLUG_CPU 22 int (*cpu_kill)(unsigned int cpu); 23 void (*cpu_die)(unsigned int cpu); 24 bool (*cpu_can_disable)(unsigned int cpu); 25 int (*cpu_disable)(unsigned int cpu); 26 #endif 27 #endif 28 };
我们从 arch/arm/mach-ambarella/soc/s3.c 中看到S3平台用到的 smp_ops() 为 ambarella_smp_ops ：
DT_MACHINE_START(S3_DT, "Ambarella S3 (Flattened Device Tree)") .l2c_aux_val = L310_AUX_CTRL_DATA_PREFETCH | L310_AUX_CTRL_INSTR_PREFETCH, .l2c_aux_mask = ~0, .smp = smp_ops(ambarella_smp_ops), .map_io = ambarella_map_io, .restart = ambarella_restart_machine, .dt_compat = s3_dt_board_compat, MACHINE_END
通过 arch/arm/mach-ambarella/smp/smp.c 的实现代码可以看出， smp_operations 结构体的成员函数 smp_init_cpus( )，即 ambarella_smp_init_cpus( ) 调用的 scu_get_core_count( ) 会探测 ambarella SoC里CPU核的个数，如果scu_get_core_count( )获取的核的个数ncores 大于 nr_cpu_ids，则会丢出一个警告然后赋值 ncores = nr_cpu_ids，并通过set_cpu_possible( )设置这些CPU（0...ncores-1）可见。如下所示：
1 /* running on CPU0 */ 2 static void __init ambarella_smp_init_cpus(void) 3 { 4 int i; 5 unsigned int ncores; 6 7 ncores = scu_get_core_count(scu_base); 8 if (ncores > nr_cpu_ids) { 9 pr_warning("SMP: cores(%u) greater than maximum(%u), clipping ", 10 ncores, nr_cpu_ids); 11 ncores = nr_cpu_ids; 12 } 13 14 for (i = 0; i < ncores; i++) 15 set_cpu_possible(i, true); 16 }
而smp_operations的成员函数smp_prepare_cpus( )， 即 ambarella_smp_prepare_cpus( ) 则会通过 write_cpux_jump_addr(i, virt_to_phys(ambarella_secondary_startup)) 设置其它CPU的启动地址为 ambarella_secondary_startup，如下代码所示：
1 /* running on CPU0 */ 2 static void __init ambarella_smp_prepare_cpus(unsigned int max_cpus) 3 { 4 u32 cpux_jump, start_limit, end_limit; 5 int i, rval; 6 7 rval = of_property_read_u32(of_chosen, "ambarella,cpux_jump", &cpux_jump); //从设备树里获取compatible为"amabrella,cpux_jump"的物理地址cpux_jump 8 if (rval < 0) { 9 pr_err("No jump address for secondary cpu! "); 10 return; 11 } 12 13 start_limit = get_ambarella_ppm_phys(); 14 end_limit = get_ambarella_ppm_phys() + get_ambarella_ppm_size(); 15 if (cpux_jump < start_limit || cpux_jump > end_limit) { //判断从设备树里解析到的物理地址cpux_jump是否在指定范围内 16 pr_err("Invalid secondary cpu jump address, 0x%08x! ", cpux_jump); 17 return; 18 } 19 20 cpux_jump_virt = (u32 *)ambarella_phys_to_virt(cpux_jump); //把物理地址cpux_jump转换为虚拟地址cpux_jump_virt 21 22 for (i = 0; i < max_cpus; i++) 23 set_cpu_present(i, true); 24 25 scu_enable(scu_base); 26 scu_power_mode(scu_base, SCU_PM_NORMAL); 27 28 for (i = 1; i < max_cpus; i++) 29 write_cpux_jump_addr(i, virt_to_phys(ambarella_secondary_startup)); //设置CPU的启动地址为ambarella_secondary_startup 30 }
注意这部分的实现方法是和具体的SoC相关联的，由芯片的设计及芯片内部的Bootrom决定。对于ambarella S3来讲，设置方法如下：
1 static void write_cpux_jump_addr(unsigned int cpu, int addr) 2 { 3 cpux_jump_virt[cpu] = addr; 4 smp_wmb(); 5 __cpuc_clean_dcache_area( 6 &cpux_jump_virt[cpu], sizeof(cpux_jump_virt[cpu])); 7 outer_clean_range(ambarella_virt_to_phys((u32)&cpux_jump_virt[cpu]), 8 ambarella_virt_to_phys((u32)&cpux_jump_virt[cpu] + 1)); 9 }
填入CPU1...n的起始地址都通过 ambarella_virt_to_phys( ) 转化为物理地址，因为此时CPU1...n的 MMU（Memory Management Unit）尚未开启。
比较关键的是 smp_operations 的成员函数 smp_boot_secondary( )，对于ambarella S3来说是ambarella_smp_boot_secondary( )，它完成CPU的最终唤醒工作。如下代码所示：
1 /* Write pen_release in a way that is guaranteed to be visible to all 2 * observers, irrespective of whether they're taking part in coherency 3 * or not. This is necessary for the hotplug code to work reliably. */ 4 static void write_pen_release(int val) 5 { 6 pen_release = val; 7 smp_wmb(); 8 sync_cache_w(&pen_release); 9 } 10 11 /* running on CPU0 */ 12 static int ambarella_smp_boot_secondary(unsigned int cpu, 13 struct task_struct *idle) 14 { 15 unsigned long flags, timeout; 16 unsigned long phys_cpu = cpu_logical_map(cpu); 17 18 BUG_ON(cpux_jump_virt == NULL); 19 20 scu_enable(scu_base); 21 22 /* Set synchronisation state between this boot processor 23 * and the secondary one */ 24 spin_lock_irqsave(&boot_lock, flags); 25 /* l2 cache has to be disabled, orelse the second core cannot boot up */ 26 outer_disable(); 27 28 /* The secondary processor is waiting to be released from 29 * the holding pen - release it, then wait for it to flag 30 * that it has been released by resetting pen_release. 31 * 32 * Note that "pen_release" is the hardware CPU ID, whereas 33 * "cpu" is Linux's internal ID. */ 34 write_pen_release(phys_cpu); 35 36 write_cpux_jump_addr(cpu, virt_to_phys(ambarella_secondary_startup)); 37 38 #ifdef CONFIG_PLAT_AMBARELLA_SUPPORT_VIC 39 /* IPI interrupt on CPU1 may be unmasked, so this init is necessary */ 40 ambvic_smp_softirq_init(); 41 #endif 42 43 /* Send the secondary CPU a soft interrupt, thereby causing 44 * the boot monitor to read the system wide flags register, 45 * and branch to the address found there. */ 46 timeout = jiffies + (1 * HZ); 47 while (time_before(jiffies, timeout)) { 48 smp_rmb(); 49 50 arch_send_wakeup_ipi_mask(cpumask_of(cpu)); 51 52 if (pen_release == -1) 53 break; 54 55 udelay(10); 56 } 57 58 outer_resume(); 59 spin_unlock_irqrestore(&boot_lock, flags); 60 61 return pen_release != -1 ? -ENOSYS : 0; 62 }
上述第34行代码调用的write_pen_release( ) 会将 pen_release 变量设置为要唤醒的 CPU 核的 CPU 号 cpu_logical_map(cpu)，而后通过 arch_send_wakeup_ipi_mask( ) 给要唤醒的 CPU 发 IPI 中断，这时被唤醒的 CPU 会退出 WFI 状态并从前面 smp_operations 中的 smp_prepare_cpus( ) 成员函数，即 ambarella_smp_prepare_cpus( ) 里通过 write_cpux_jump_addr( )设置的起始地址 ambarella_secondary_startup 开始执行， 如果顺利的话，该 CPU 会将原先为正数的 pen_release 写为 -1，以便 CPU0 从等待 pen_release 成为-1的循环（47~56行）中跳出。
ambarella_secondary_startup( ) 实现于 arch/arm/mach-ambarella/smp/headsmp.S 中，是一段汇编代码，如下所示：
1 /* 2 * ambarella specific entry point for secondary CPUs. This provides 3 * a "holding pen" into which all secondary cores are held until we're 4 * ready for them to initialise. 5 */ 6 ENTRY(ambarella_secondary_startup) 7 mrc p15, 0, r0, c0, c0, 5 8 and r0, r0, #0x00ffffff 9 adr r4, 1f 10 ldmia r4, {r5, r6} 11 sub r4, r4, r5 12 add r6, r6, r4 13 pen: ldr r7, [r6] 14 cmp r7, r0 15 bne pen 16 17 /* 18 * we've been released from the holding pen: secondary_stack 19 * should now contain the SVC stack for this core 20 */ 21 b secondary_startup 22 ENDPROC(ambarella_secondary_startup) 23 24 .align 2 25 1: .long . 26 .long pen_release
上述代码第13~15行的循环是等待 pen_release 变量成为 CPU0 设置的 cpu_logical_map(cpu)，一般直接就成立了。第21行则调用内核通用的 secondary_startup( ) 函数，经过一系列的初始化（如MMU等），最终新的被唤醒的 CPU 将调用 smp_operations 的smp_secondary_init( ) 成员函数，对于 ambarella S3 SoC为 ambarella_smp_secondary_init()。如下所示：
1 /* running on CPU1 */ 2 static void ambarella_smp_secondary_init(unsigned int cpu) 3 { 4 /* let the primary processor know we're out of the 5 * pen, then head off into the C entry point */ 6 write_pen_release(-1); 7 8 /* Synchronise with the boot thread. */ 9 spin_lock(&boot_lock); 10 spin_unlock(&boot_lock); 11 }
上述代码第6行会将 pen_release 写为 -1，于是 CPU0还在执行的代码中的 ambarella_smp_boot_secondary( ) 函数里的如下循环就退出了：
1 while (time_before(jiffies, timeout)) { 2 smp_rmb(); 3 4 arch_send_wakeup_ipi_mask(cpumask_of(cpu)); 5 6 if (pen_release == -1) 7 break; 8 9 udelay(10); 10 }
这样 CPU0 就知道目标 CPU 已经被正确的唤醒，此后 CPU0 和新唤醒的其他 CPU 各自运行。整个系统在运行过程中会进行实时进程和正常进程的动态负载均衡。
下图总结性的描述了上面提到的 ambarella_smp_prepare_cpus( )、ambarella_smp_boot_secondary( )、write_pen_release( )、ambarella_secondary_startup( )、ambarella_smp_secondary_init( )这些函数的执行顺序。
SMP CPU热插拔
被 CPU0 唤醒的 CPUn 可以在运行过程中进行热插拔，譬如运行如下命令即可卸载 CPU1，并且将 CPU1 上的任务全部迁移到其他 CPU 中：
# echo 0 > /sys/devices/system/cpu/cpu1/online
运行如下命令可以再次启动 CPU1：
echo 1 > /sys/devices/system/cpu/cpu1/online
之后 CPU1 会主动参与系统中各个 CPU 之间要运行任务的负载均衡工作。
注意：CPU0 不支持热插拔！！！如下代码所示（arch/arm/mach-ambarella/smp/smp.c）：
1 /* running on CPU1 */ 2 static int ambarella_smp_cpu_disable(unsigned int cpu) 3 { 4 return cpu == 0 ? -EPERM : 0; //若试图移除cpu0，则返回PERM错误 5 } 6 #endif
CPU 热插拔的实现也是与芯片相关的，对于ambarella S3 SoC而言，实现了 smp_operations 的 cpu_die( ) 成员函数，即 ambarella_smp_cpu_die( )。它会在进行 CPUn的拔除操作时将CPUn投入低功耗的 WFI 状态，代码同样位于 arch/arm/mach-ambarella/smp/smp.c中，如下所示：
1 static inline void platform_do_lowpower(unsigned int cpu, int *spurious) 2 { 3 for (;;) { 4 wfi(); // CPUn会睡眠于此，直到接收到CPU0发来的IPI中断，然后返回继续执行 5 6 if (pen_release == cpu_logical_map(cpu)) { //判断该次醒来是否为CPU0唤醒 7 /* OK, proper wakeup, we're done */ 8 break; //如果时CPU0的正常唤醒的话，循环退出，不会执行到下面的代码 9 } 10 11 /* Getting here, means that we have come out of WFI without 12 * having been woken up - this shouldn't happen 13 * 14 * Just note it happening - when we're woken, we can report 15 * its occurrence. */ 16 (*spurious)++; 17 } 18 } 19 20 /* running on CPU1 */ 21 static void ambarella_smp_cpu_die(unsigned int cpu) 22 { 23 int spurious = 0; 24 25 cpu_enter_lowpower(); 26 27 platform_do_lowpower(cpu, &spurious); 28 29 cpu_leave_lowpower(); 30 31 if (spurious) 32 pr_warn("CPU%u: %u spurious wakeup calls ", cpu, spurious); 33 }
CPUn睡眠于wfi( )， 之后再次在线的时候，又会因为 CPU0 给它发出的 IPI 而从 wfi( ) 函数返回继续执行，醒来时 CPUn 也判断 “pen_release == cpu_logical_map(cpu)” 是否成立，以确定该次醒来确实是由 CPU0 唤醒的一次正常醒来。