Linux双核SMP系统启动流程（Zynq-ARM-CortexA9）

转载：http://blog.chinaunix.net/uid-20648445-id-3329217.html

1：资料附录：
    <ug585-Zynq-7000-TRM.pdf>                            xilinx zynq 7000技术参考手册
    <ug821-zynq-7000-swdev.pdf>                          xilinx zynq 7000软件开发手册
    <ug925-zynq-zc702-base-trd.pdf>                      xilinx zynq 7020板级开发手册
    <DDI0406C_arm_architecture_reference_manual.pdf>     ARM v7 cortex A系列和R系列参考手册
    <DDI0407H_cortex_a9_mpcore_r4p0_trm.pdf>             ARM v7 cortex A9 MCORE技术参考手册
    <DDI0388H_cortex_a9_r4p0_trm.pdf>                    ARM v7 cortex A9 技术参考手册
    <IHI0048A_gic_architecture_spec_v1_0.pdf>            ARM 通用中断控制器V1手册
    <IHI0042D_aapcs.pdf>                                 ARM 过程调用ABI约定手册
注：上述手册为本人进行zynq zc702开发板研究时使用参考手册，希望在后续研究过程中能对该手册内容进行概略描述，以便大家查找相关细节。待补充。
2：源码索引：
    Linux源码
3：启动流程：

注：本博文以提出问题，回答问题方式进行记录本人研究双核系统的过程。希望在一步步研究及分析后，能够完整回答各个问题。以下分析基于ARM v7架构Linux代码和XILINX的ZYNQ平台。由于本博文正在更新过程中，还未完成，若对单核启动有兴趣的朋友可以查看如下资源，该资源正是本人前半部分启动需要描述的内容。

    网络资源

问题1-1：双核芯片上电后，是否同时启动的？

答案：双核芯片上电后，并非同时启动，启动代码运行在一个核上，而是一个核处于备用状态。

参见<ug585-Zynq-7000-TRM--P140/1671—6.3.7 Post BootROM State—Starting Code on the CPU 1>

图1：CPU1启动工作状态

图2: CPU1启动流程

问题1-2：根据上述参考资料，第二个核是通过CPU0进行设置并引导启动的，Linux是怎么完成这一步的呢？我们从内核入口函数开始研究启动过程。
答案：本部分从Linux系统启动部分开始分析，BOOT启动引导部分暂不做分析。
    1）入口函数
    linux-xlnxarcharmkernelhead.S

点击(此处)折叠或打开--CodeSegment 1

/*
* Kernel startup entry point.
* ---------------------------
*
* This is normally called from the decompressor code. The requirements
* are: MMU = off, D-cache = off, I-cache = dont care, r0 = 0,
* r1 = machine nr, r2 = atags or dtb pointer.
*
* This code is mostly position independent, so if you link the kernel at
* 0xc0008000, you call this at __pa(0xc0008000).
*
* See linux/arch/arm/tools/mach-types for the complete list of machine
* numbers for r1.
*
* We're trying to keep crap to a minimum; DO NOT add any machine specific
* crap here - that's what the boot loader (or in extreme, well justified
* circumstances, zImage) is for.
*/
.arm

__HEAD
ENTRY(stext)

THUMB( adr r9, BSYM(1f) ) @ Kernel is always entered in ARM.
THUMB( bx r9 ) @ If this is a Thumb-2 kernel,
THUMB( .thumb ) @ switch to Thumb now.
THUMB(1: )

setmode PSR_F_BIT | PSR_I_BIT | SVC_MODE, r9 @ ensure svc mode
@ and irqs disabled
mrc p15, 0, r9, c0, c0 @ get processor id
bl __lookup_processor_type @ r5=procinfo r9=cpuid
movs r10, r5 @ invalid processor (r5=0)?
THUMB( it eq ) @ force fixup-able long branch encoding
beq __error_p @ yes, error 'p'

/* the machine has no way to get setup when we're using a pod so setup it
* up for now this way, if the device tree is expected at the fixed address
* then load R2 to find the device tree at that address
*/
#ifdef CONFIG_ARCH_XILINX
mov r0,#0x0
ldr r1,=MACH_TYPE_XILINX
#endif
#ifdef CONFIG_XILINX_FIXED_DEVTREE_ADDR
mov r2,#0x1000000
#endif
#ifdef CONFIG_ARM_LPAE
mrc p15, 0, r3, c0, c1, 4 @ read ID_MMFR0
and r3, r3, #0xf @ extract VMSA support
cmp r3, #5 @ long-descriptor translation table format?
THUMB( it lo ) @ force fixup-able long branch encoding
blo __error_p @ only classic page table format
#endif

#ifndef CONFIG_XIP_KERNEL
adr r3, 2f
ldmia r3, {r4, r8}
sub r4, r3, r4 @ (PHYS_OFFSET - PAGE_OFFSET)
add r8, r8, r4 @ PHYS_OFFSET
#else
ldr r8, =PHYS_OFFSET @ always constant in this case
#endif

/*
* r1 = machine no, r2 = atags or dtb,
* r8 = phys_offset, r9 = cpuid, r10 = procinfo
*/
bl __vet_atags
#ifdef CONFIG_SMP_ON_UP
bl __fixup_smp
#endif
#ifdef CONFIG_ARM_PATCH_PHYS_VIRT
bl __fixup_pv_table
#endif
bl __create_page_tables

/*
* The following calls CPU specific code in a position independent
* manner. See arch/arm/mm/proc-*.S for details. r10 = base of
* xxx_proc_info structure selected by __lookup_processor_type
* above. On return, the CPU will be ready for the MMU to be
* turned on, and r0 will hold the CPU control register value.
*/
ldr r13, =__mmap_switched @ address to jump to after
@ mmu has been enabled
adr lr, BSYM(1f) @ return (PIC) address
mov r8, r4 @ set TTBR1 to swapper_pg_dir
ARM( add pc, r10, #PROCINFO_INITFUNC )
THUMB( add r12, r10, #PROCINFO_INITFUNC )
THUMB( mov pc, r12 )
1: b __enable_mmu
ENDPROC(stext)
.ltorg
#ifndef CONFIG_XIP_KERNEL
2: .long .
.long PAGE_OFFSET
#endif

注：本部分源码为汇编代码，汇编与C语言相互调用参数传递，返回值等相关约定参见<IHI0042D_aapcs.pdf>
问题1-2-1：U-Boot程序如何跳转到该入口函数？
答案：希望后续章节可以研究分析U-Boot源码进行深入分析。待补充。

问题1-2-2：下列代码为什么意思？THUMB宏是什么定义？ARM宏是什么定义？BSYM宏定义是什么？

点击(此处)折叠或打开--CodeSegment 2

THUMB(	adr	r9, BSYM(1f)	)	@ Kernel is always entered in ARM.
THUMB(	bx	r9	 )	@ If this is a Thumb-2 kernel,
THUMB(	.thumb	 )	@ switch to Thumb now.
THUMB(1:	 )

答案：THUMB宏定义参见文件linux-xlnxarcharmincludeasmunified.h，分析如下宏定义可知，内核在GCC版本大于4.0后，可通过配置宏CONFIG_THUMB2_KERNEL来选择编译成THUMB指令集内核或ARM指令集内核。在编译成ARM指令集内核时，THUMB宏定义的代码行是不可见的。同理编译成THUMB指令集内核时，ARM宏定义的代码行无效。BSYM宏是为在THUMB指令集时访问标号处理而定义的宏。

点击(此处)折叠或打开--CodeSegment 3

#ifdef CONFIG_THUMB2_KERNEL

#if __GNUC__ < 4
#error Thumb-2 kernel requires gcc >= 4
#endif

/* The CPSR bit describing the instruction set (Thumb) */
#define PSR_ISETSTATE	PSR_T_BIT

#define ARM(x...)
#define THUMB(x...)	x
#ifdef __ASSEMBLY__
#define W(instr)	instr.w
#define BSYM(sym)	sym + 1
#endif

#else	/* !CONFIG_THUMB2_KERNEL */

/* The CPSR bit describing the instruction set (ARM) */
#define PSR_ISETSTATE	0

#define ARM(x...)	x
#define THUMB(x...)
#ifdef __ASSEMBLY__
#define W(instr)	instr
#define BSYM(sym)	sym
#endif

#endif	/* CONFIG_THUMB2_KERNEL */

问题1-2-3：THUMB指令集与ARM指令集区别，什么是THUMB-2指令集
答案：待补充

问题1-2-4：setmode是什么定义？
答案：setmode定义参见文件linux-xlnxarcharmincludeasmassembler.h，下面代码为一个汇编宏定义。根据不同的指令集采用不同的宏定义，用于设置CPSR。

点击(此处)折叠或打开--CodeSegment 4

#ifdef CONFIG_THUMB2_KERNEL
.macro	setmode, mode, reg
mov	
eg, #mode
msr	cpsr_c, 
eg
.endm
#else
.macro	setmode, mode, reg
msr	cpsr_c, #mode
.endm
#endif

问题1-2-5： __lookup_processor_type是什么？查找处理器类型？
答案：该汇编函数定义位于文件linux-xlnxarcharmkernelhead-common.S，分析如下汇编代码，通过从__lookup_processor_type_data中获取处理器信息位置，通过轮询查找该内核是否支持该本处理器。其中r9是通过processor id。本代码段会进行详细注释，汇编指令
参见<DDI0406C_arm_architecture_reference_manual--P157/2680—A8 Instruction Details—Alphabetical list of instructions>

点击(此处)折叠或打开--CodeSegment 5

/*
* Read processor ID register (CP#15, CR0), and look up in the linker-built
* supported processor list. Note that we can't use the absolute addresses
* for the __proc_info lists since we aren't running with the MMU on
* (and therefore, we are not in the correct address space). We have to
* calculate the offset.
*
*	r9 = cpuid
* Returns:
*	r3, r4, r6 corrupted
*	r5 = proc_info pointer in physical address space
*	r9 = cpuid (preserved)
*/
__CPUINIT
__lookup_processor_type:
adr	r3, __lookup_processor_type_data   @ R3指向类型数据指针(此处R3是物理地址，可查看反汇编代码，下图3所示)
ldmia	r3, {r4 - r6}                    @ 读取R3指向地址三个WORD数据到R4,R5,R6
sub	r3, r3, r4	                        @ 物理地址-虚拟地址
add	r5, r5, r3	           @ 将R5的虚拟地址转换成物理地址，R5为类型数据起始地址
add	r6, r6, r3            @ 将R6的虚拟地址转换成物理地址，R6为类型数据结束地址
1:	ldmia	r5, {r3, r4}     @ 从R5指向的地址读取两个WORD到R3,R4
and	r4, r4, r9	          	@ 获取判断位
teq	r3, r4                @ 判断本处理器类型是否已找到
beq	2f                    @ 若已找到，跳转到2：执行
add	r5, r5, #PROC_INFO_SZ	@ 若未找到，R5指针增到一个类型数据长度
cmp	r5, r6                @ 是否已查找完所有处理器类型
blo	1b
mov	r5, #0	              	@ 若未找到本处理器类型信息，返回值R5设置为NULL
2:	mov	pc, lr
ENDPROC(__lookup_processor_type)

问题1-2-6： __lookup_processor_type_data是什么？上个问题中R3，R4，R5，R6寄存器是指向什么数据？
答案： __lookup_processor_type_data数据结构定义位于文件linux-xlnxarcharmkernelhead-common.S，参见汇编代码可知，该数据结构有三个数据，数据内容参见代码注释。

点击(此处)折叠或打开--CodeSegment 6

/*
* Look in <asm/procinfo.h> for information about the __proc_info structure.
*/
.align	2
.type	__lookup_processor_type_data, %object
__lookup_processor_type_data:
.long	.                              @ 当前内存地址（此处为虚拟地址）
.long	__proc_info_begin              @ 处理器类型信息起始地址
.long	__proc_info_end                @ 处理器类型信息结束地址
.size	__lookup_processor_type_data, . - __lookup_processor_type_data

通过反汇编，我们可以查看内核编译完成后各个数据结构的具体数据。

图3 反汇编代码

__proc_info_begin，__proc_info_end定义位于linux-xlnxarcharmkernelvmlinux.lds.S，它用于存储.proc.info.init段的起始地址及结束地址。

点击(此处)折叠或打开--CodeSegment 7

VMLINUX_SYMBOL(__proc_info_begin) = .;
*(.proc.info.init)
VMLINUX_SYMBOL(__proc_info_end) = .;

.proc.info.init段定义位于linux-xlnxarcharmmmproc-v7.S，该文件定义了本内核版本支持的ARM v7架构下处理器类型，如下列代码Line 29行表示本版本支持ARM Cortex A5 Processor，Line 39表示本版本支持ARM Cortex A9 Processor等。

点击(此处)折叠或打开--CodeSegment 8

.section ".proc.info.init", #alloc, #execinstr

/*
* Standard v7 proc info content
*/
.macro __v7_proc initfunc, mm_mmuflags = 0, io_mmuflags = 0, hwcaps = 0
ALT_SMP(.long	PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ |
PMD_SECT_AF | PMD_FLAGS_SMP | mm_mmuflags)
ALT_UP(.long	PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ |
PMD_SECT_AF | PMD_FLAGS_UP | mm_mmuflags)
.long	PMD_TYPE_SECT | PMD_SECT_AP_WRITE |
PMD_SECT_AP_READ | PMD_SECT_AF | io_mmuflags
W(b)	initfunc
.long	cpu_arch_name
.long	cpu_elf_name
.long	HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB | HWCAP_FAST_MULT |
HWCAP_EDSP | HWCAP_TLS | hwcaps
.long	cpu_v7_name
.long	v7_processor_functions
.long	v7wbi_tlb_fns
.long	v6_user_fns
.long	v7_cache_fns
.endm

#ifndef CONFIG_ARM_LPAE
/*
* ARM Ltd. Cortex A5 processor.
*/
.type __v7_ca5mp_proc_info, #object
__v7_ca5mp_proc_info:
.long	0x410fc050
.long	0xff0ffff0
__v7_proc __v7_ca5mp_setup
.size	__v7_ca5mp_proc_info, . - __v7_ca5mp_proc_info

/*
* ARM Ltd. Cortex A9 processor.
*/
.type __v7_ca9mp_proc_info, #object
__v7_ca9mp_proc_info:
.long	0x410fc090
.long	0xff0ffff0
__v7_proc __v7_ca9mp_setup
.size	__v7_ca9mp_proc_info, . - __v7_ca9mp_proc_info
#endif	/* CONFIG_ARM_LPAE */

/*
* ARM Ltd. Cortex A7 processor.
*/
.type	__v7_ca7mp_proc_info, #object
__v7_ca7mp_proc_info:
.long	0x410fc070
.long	0xff0ffff0
__v7_proc __v7_ca7mp_setup, hwcaps = HWCAP_IDIV
.size	__v7_ca7mp_proc_info, . - __v7_ca7mp_proc_info
......

问题1-2-7： 在上述问题中我们可以看到支持的处理信息数据，但这些数据是什么样的结构呢？代表什么意义呢？
答案：.proc.info.init段中数据是以proc_info_list结构的方式进行连续存放的。该结构体的定义位于linux-xlnxarcharmincludeasmprocinfo.h，

点击(此处)折叠或打开--CodeSegment 9

/*
* Note! struct processor is always defined if we're
* using MULTI_CPU, otherwise this entry is unused,
* but still exists.
*
* NOTE! The following structure is defined by assembly
* language, NOT C code. For more information, check:
* arch/arm/mm/proc-*.S and arch/arm/kernel/head.S
*/
struct proc_info_list {
unsigned int	 cpu_val;
unsigned int	 cpu_mask;
unsigned long	 __cpu_mm_mmu_flags;	/* used by head.S */
unsigned long	 __cpu_io_mmu_flags;	/* used by head.S */
unsigned long	 __cpu_flush;	 /* used by head.S */
const char	 *arch_name;
const char	 *elf_name;
unsigned int	 elf_hwcap;
const char	 *cpu_name;
struct processor	*proc;
struct cpu_tlb_fns	*tlb;
struct cpu_user_fns	*user;
struct cpu_cache_fns	*cache;
};

问题1-2-8：__vet_atags是什么？
答案：待补充

点击(此处)折叠或打开--CodeSegment 10

/* Determine validity of the r2 atags pointer. The heuristic requires
* that the pointer be aligned, in the first 16k of physical RAM and
* that the ATAG_CORE marker is first and present. If CONFIG_OF_FLATTREE
* is selected, then it will also accept a dtb pointer. Future revisions
* of this function may be more lenient with the physical address and
* may also be able to move the ATAGS block if necessary.
*
* Returns:
* r2 either valid atags pointer, valid dtb pointer, or zero
* r5, r6 corrupted
*/
__vet_atags:
tst	r2, #0x3	 @ aligned?
bne	1f

ldr	r5, [r2, #0]
#ifdef CONFIG_OF_FLATTREE
ldr	r6, =OF_DT_MAGIC	 @ is it a DTB?
cmp	r5, r6
beq	2f
#endif
cmp	r5, #ATAG_CORE_SIZE	 @ is first tag ATAG_CORE?
cmpne	r5, #ATAG_CORE_SIZE_EMPTY
bne	1f
ldr	r5, [r2, #4]
ldr	r6, =ATAG_CORE
cmp	r5, r6
bne	1f

2:	mov	pc, lr	 @ atag/dtb pointer is ok

1:	mov	r2, #0
mov	pc, lr
ENDPROC(__vet_atags)

问题1-2-9：__fixup_smp是什么？