1636 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * linux/init/main.c4 *5 * Copyright (C) 1991, 1992 Linus Torvalds6 *7 * GK 2/5/95 - Changed to support mounting root fs via NFS8 * Added initrd & change_root: Werner Almesberger & Hans Lermen, Feb '969 * Moan early if gcc is old, avoiding bogus kernels - Paul Gortmaker, May '9610 * Simplified starting of init: Michael A. Griffith <grif@acm.org>11 */12 13#define DEBUG /* Enable initcall_debug */14 15#include <linux/types.h>16#include <linux/extable.h>17#include <linux/module.h>18#include <linux/proc_fs.h>19#include <linux/binfmts.h>20#include <linux/kernel.h>21#include <linux/syscalls.h>22#include <linux/stackprotector.h>23#include <linux/string.h>24#include <linux/ctype.h>25#include <linux/delay.h>26#include <linux/ioport.h>27#include <linux/init.h>28#include <linux/initrd.h>29#include <linux/memblock.h>30#include <linux/acpi.h>31#include <linux/bootconfig.h>32#include <linux/console.h>33#include <linux/nmi.h>34#include <linux/percpu.h>35#include <linux/kmod.h>36#include <linux/kprobes.h>37#include <linux/kmsan.h>38#include <linux/vmalloc.h>39#include <linux/kernel_stat.h>40#include <linux/start_kernel.h>41#include <linux/security.h>42#include <linux/smp.h>43#include <linux/profile.h>44#include <linux/kfence.h>45#include <linux/rcupdate.h>46#include <linux/srcu.h>47#include <linux/moduleparam.h>48#include <linux/kallsyms.h>49#include <linux/buildid.h>50#include <linux/writeback.h>51#include <linux/cpu.h>52#include <linux/cpuset.h>53#include <linux/cgroup.h>54#include <linux/efi.h>55#include <linux/tick.h>56#include <linux/sched/isolation.h>57#include <linux/interrupt.h>58#include <linux/taskstats_kern.h>59#include <linux/delayacct.h>60#include <linux/unistd.h>61#include <linux/utsname.h>62#include <linux/rmap.h>63#include <linux/mempolicy.h>64#include <linux/key.h>65#include <linux/debug_locks.h>66#include <linux/debugobjects.h>67#include <linux/lockdep.h>68#include <linux/kmemleak.h>69#include <linux/padata.h>70#include <linux/pid_namespace.h>71#include <linux/device/driver.h>72#include <linux/kthread.h>73#include <linux/sched.h>74#include <linux/sched/init.h>75#include <linux/signal.h>76#include <linux/idr.h>77#include <linux/kgdb.h>78#include <linux/ftrace.h>79#include <linux/async.h>80#include <linux/shmem_fs.h>81#include <linux/slab.h>82#include <linux/perf_event.h>83#include <linux/ptrace.h>84#include <linux/pti.h>85#include <linux/blkdev.h>86#include <linux/sched/clock.h>87#include <linux/sched/task.h>88#include <linux/sched/task_stack.h>89#include <linux/context_tracking.h>90#include <linux/random.h>91#include <linux/moduleloader.h>92#include <linux/list.h>93#include <linux/integrity.h>94#include <linux/proc_ns.h>95#include <linux/io.h>96#include <linux/cache.h>97#include <linux/rodata_test.h>98#include <linux/jump_label.h>99#include <linux/kcsan.h>100#include <linux/init_syscalls.h>101#include <linux/stackdepot.h>102#include <linux/randomize_kstack.h>103#include <linux/pidfs.h>104#include <linux/ptdump.h>105#include <net/net_namespace.h>106 107#include <asm/io.h>108#include <asm/setup.h>109#include <asm/sections.h>110#include <asm/cacheflush.h>111 112#define CREATE_TRACE_POINTS113#include <trace/events/initcall.h>114 115#include <kunit/test.h>116 117/* wasm32/HWJS-1: HWJS_SUSPENDS = published suspend-reachability for the118 * hwjs-cc cross-TU coroutinize seed; macro force-included from119 * arch/wasm32/include/asm/hwjs.h. Static address-taken suspenders must120 * be annotated in-file (no shared prototype to publish). */121static int kernel_init(void *) HWJS_SUSPENDS;122/* arch hook; see patch 0004 */123void arch_kernel_init_pre_run_init(void);124 125/*126 * Debug helper: via this flag we know that we are in 'early bootup code'127 * where only the boot processor is running with IRQ disabled. This means128 * two things - IRQ must not be enabled before the flag is cleared and some129 * operations which are not allowed with IRQ disabled are allowed while the130 * flag is set.131 */132bool early_boot_irqs_disabled __read_mostly;133 134enum system_states system_state __read_mostly;135EXPORT_SYMBOL(system_state);136 137/*138 * Boot command-line arguments139 */140#define MAX_INIT_ARGS CONFIG_INIT_ENV_ARG_LIMIT141#define MAX_INIT_ENVS CONFIG_INIT_ENV_ARG_LIMIT142 143/* Default late time init is NULL. archs can override this later. */144void (*__initdata late_time_init)(void);145 146/* Untouched command line saved by arch-specific code. */147char __initdata boot_command_line[COMMAND_LINE_SIZE];148/* Untouched saved command line (eg. for /proc) */149char *saved_command_line __ro_after_init;150unsigned int saved_command_line_len __ro_after_init;151/* Command line for parameter parsing */152static char *static_command_line;153/* Untouched extra command line */154static char *extra_command_line;155/* Extra init arguments */156static char *extra_init_args;157 158#ifdef CONFIG_BOOT_CONFIG159/* Is bootconfig on command line? */160static bool bootconfig_found;161static size_t initargs_offs;162#else163# define bootconfig_found false164# define initargs_offs 0165#endif166 167static char *execute_command;168static char *ramdisk_execute_command = "/init";169 170/*171 * Used to generate warnings if static_key manipulation functions are used172 * before jump_label_init is called.173 */174bool static_key_initialized __read_mostly;175EXPORT_SYMBOL_GPL(static_key_initialized);176 177/*178 * If set, this is an indication to the drivers that reset the underlying179 * device before going ahead with the initialization otherwise driver might180 * rely on the BIOS and skip the reset operation.181 *182 * This is useful if kernel is booting in an unreliable environment.183 * For ex. kdump situation where previous kernel has crashed, BIOS has been184 * skipped and devices will be in unknown state.185 */186unsigned int reset_devices;187EXPORT_SYMBOL(reset_devices);188 189static int __init set_reset_devices(char *str)190{191 reset_devices = 1;192 return 1;193}194 195__setup("reset_devices", set_reset_devices);196 197static const char *argv_init[MAX_INIT_ARGS+2] = { "init", NULL, };198const char *envp_init[MAX_INIT_ENVS+2] = { "HOME=/", "TERM=linux", NULL, };199static const char *panic_later, *panic_param;200 201static HWJS_SUSPENDS bool __init obsolete_checksetup(char *line)202{203 const struct obs_kernel_param *p;204 bool had_early_param = false;205 206 p = __setup_start;207 do {208 int n = strlen(p->str);209 if (parameqn(line, p->str, n)) {210 if (p->early) {211 /* Already done in parse_early_param?212 * (Needs exact match on param part).213 * Keep iterating, as we can have early214 * params and __setups of same names 8( */215 if (line[n] == '\0' || line[n] == '=')216 had_early_param = true;217 } else if (!p->setup_func) {218 pr_warn("Parameter %s is obsolete, ignored\n",219 p->str);220 return true;221 } else if (p->setup_func(line + n))222 return true;223 }224 p++;225 } while (p < __setup_end);226 227 return had_early_param;228}229 230/*231 * This should be approx 2 Bo*oMips to start (note initial shift), and will232 * still work even if initially too large, it will just take slightly longer233 */234unsigned long loops_per_jiffy = (1<<12);235EXPORT_SYMBOL(loops_per_jiffy);236 237static int __init debug_kernel(char *str)238{239 console_loglevel = CONSOLE_LOGLEVEL_DEBUG;240 return 0;241}242 243static int __init quiet_kernel(char *str)244{245 console_loglevel = CONSOLE_LOGLEVEL_QUIET;246 return 0;247}248 249early_param("debug", debug_kernel);250early_param("quiet", quiet_kernel);251 252static int __init loglevel(char *str)253{254 int newlevel;255 256 /*257 * Only update loglevel value when a correct setting was passed,258 * to prevent blind crashes (when loglevel being set to 0) that259 * are quite hard to debug260 */261 if (get_option(&str, &newlevel)) {262 console_loglevel = newlevel;263 return 0;264 }265 266 return -EINVAL;267}268 269early_param("loglevel", loglevel);270 271#ifdef CONFIG_BLK_DEV_INITRD272static void * __init get_boot_config_from_initrd(size_t *_size)273{274 u32 size, csum;275 char *data;276 u32 *hdr;277 int i;278 279 if (!initrd_end)280 return NULL;281 282 data = (char *)initrd_end - BOOTCONFIG_MAGIC_LEN;283 /*284 * Since Grub may align the size of initrd to 4, we must285 * check the preceding 3 bytes as well.286 */287 for (i = 0; i < 4; i++) {288 if (!memcmp(data, BOOTCONFIG_MAGIC, BOOTCONFIG_MAGIC_LEN))289 goto found;290 data--;291 }292 return NULL;293 294found:295 hdr = (u32 *)(data - 8);296 size = le32_to_cpu(hdr[0]);297 csum = le32_to_cpu(hdr[1]);298 299 data = ((void *)hdr) - size;300 if ((unsigned long)data < initrd_start) {301 pr_err("bootconfig size %d is greater than initrd size %ld\n",302 size, initrd_end - initrd_start);303 return NULL;304 }305 306 if (xbc_calc_checksum(data, size) != csum) {307 pr_err("bootconfig checksum failed\n");308 return NULL;309 }310 311 /* Remove bootconfig from initramfs/initrd */312 initrd_end = (unsigned long)data;313 if (_size)314 *_size = size;315 316 return data;317}318#else319static void * __init get_boot_config_from_initrd(size_t *_size)320{321 return NULL;322}323#endif324 325#ifdef CONFIG_BOOT_CONFIG326 327static char xbc_namebuf[XBC_KEYLEN_MAX] __initdata;328 329#define rest(dst, end) ((end) > (dst) ? (end) - (dst) : 0)330 331static int __init xbc_snprint_cmdline(char *buf, size_t size,332 struct xbc_node *root)333{334 struct xbc_node *knode, *vnode;335 char *end = buf + size;336 const char *val, *q;337 int ret;338 339 xbc_node_for_each_key_value(root, knode, val) {340 ret = xbc_node_compose_key_after(root, knode,341 xbc_namebuf, XBC_KEYLEN_MAX);342 if (ret < 0)343 return ret;344 345 vnode = xbc_node_get_child(knode);346 if (!vnode) {347 ret = snprintf(buf, rest(buf, end), "%s ", xbc_namebuf);348 if (ret < 0)349 return ret;350 buf += ret;351 continue;352 }353 xbc_array_for_each_value(vnode, val) {354 /*355 * For prettier and more readable /proc/cmdline, only356 * quote the value when necessary, i.e. when it contains357 * whitespace.358 */359 q = strpbrk(val, " \t\r\n") ? "\"" : "";360 ret = snprintf(buf, rest(buf, end), "%s=%s%s%s ",361 xbc_namebuf, q, val, q);362 if (ret < 0)363 return ret;364 buf += ret;365 }366 }367 368 return buf - (end - size);369}370#undef rest371 372/* Make an extra command line under given key word */373static char * __init xbc_make_cmdline(const char *key)374{375 struct xbc_node *root;376 char *new_cmdline;377 int ret, len = 0;378 379 root = xbc_find_node(key);380 if (!root)381 return NULL;382 383 /* Count required buffer size */384 len = xbc_snprint_cmdline(NULL, 0, root);385 if (len <= 0)386 return NULL;387 388 new_cmdline = memblock_alloc(len + 1, SMP_CACHE_BYTES);389 if (!new_cmdline) {390 pr_err("Failed to allocate memory for extra kernel cmdline.\n");391 return NULL;392 }393 394 ret = xbc_snprint_cmdline(new_cmdline, len + 1, root);395 if (ret < 0 || ret > len) {396 pr_err("Failed to print extra kernel cmdline.\n");397 memblock_free(new_cmdline, len + 1);398 return NULL;399 }400 401 return new_cmdline;402}403 404static int __init bootconfig_params(char *param, char *val,405 const char *unused, void *arg)406{407 if (strcmp(param, "bootconfig") == 0) {408 bootconfig_found = true;409 }410 return 0;411}412 413static int __init warn_bootconfig(char *str)414{415 /* The 'bootconfig' has been handled by bootconfig_params(). */416 return 0;417}418 419static void __init setup_boot_config(void)420{421 static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata;422 const char *msg, *data;423 int pos, ret;424 size_t size;425 char *err;426 427 /* Cut out the bootconfig data even if we have no bootconfig option */428 data = get_boot_config_from_initrd(&size);429 /* If there is no bootconfig in initrd, try embedded one. */430 if (!data)431 data = xbc_get_embedded_bootconfig(&size);432 433 strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);434 err = parse_args("bootconfig", tmp_cmdline, NULL, 0, 0, 0, NULL,435 bootconfig_params);436 437 if (IS_ERR(err) || !(bootconfig_found || IS_ENABLED(CONFIG_BOOT_CONFIG_FORCE)))438 return;439 440 /* parse_args() stops at the next param of '--' and returns an address */441 if (err)442 initargs_offs = err - tmp_cmdline;443 444 if (!data) {445 /* If user intended to use bootconfig, show an error level message */446 if (bootconfig_found)447 pr_err("'bootconfig' found on command line, but no bootconfig found\n");448 else449 pr_info("No bootconfig data provided, so skipping bootconfig");450 return;451 }452 453 if (size >= XBC_DATA_MAX) {454 pr_err("bootconfig size %ld greater than max size %d\n",455 (long)size, XBC_DATA_MAX);456 return;457 }458 459 ret = xbc_init(data, size, &msg, &pos);460 if (ret < 0) {461 if (pos < 0)462 pr_err("Failed to init bootconfig: %s.\n", msg);463 else464 pr_err("Failed to parse bootconfig: %s at %d.\n",465 msg, pos);466 } else {467 xbc_get_info(&ret, NULL);468 pr_info("Load bootconfig: %ld bytes %d nodes\n", (long)size, ret);469 /* keys starting with "kernel." are passed via cmdline */470 extra_command_line = xbc_make_cmdline("kernel");471 /* Also, "init." keys are init arguments */472 extra_init_args = xbc_make_cmdline("init");473 }474 return;475}476 477static void __init exit_boot_config(void)478{479 xbc_exit();480}481 482#else /* !CONFIG_BOOT_CONFIG */483 484static void __init setup_boot_config(void)485{486 /* Remove bootconfig data from initrd */487 get_boot_config_from_initrd(NULL);488}489 490static int __init warn_bootconfig(char *str)491{492 pr_warn("WARNING: 'bootconfig' found on the kernel command line but CONFIG_BOOT_CONFIG is not set.\n");493 return 0;494}495 496#define exit_boot_config() do {} while (0)497 498#endif /* CONFIG_BOOT_CONFIG */499 500early_param("bootconfig", warn_bootconfig);501 502bool __init cmdline_has_extra_options(void)503{504 return extra_command_line || extra_init_args;505}506 507/* Change NUL term back to "=", to make "param" the whole string. */508static void __init repair_env_string(char *param, char *val)509{510 if (val) {511 /* param=val or param="val"? */512 if (val == param+strlen(param)+1)513 val[-1] = '=';514 else if (val == param+strlen(param)+2) {515 val[-2] = '=';516 memmove(val-1, val, strlen(val)+1);517 } else518 BUG();519 }520}521 522/* Anything after -- gets handed straight to init. */523static int __init set_init_arg(char *param, char *val,524 const char *unused, void *arg)525{526 unsigned int i;527 528 if (panic_later)529 return 0;530 531 repair_env_string(param, val);532 533 for (i = 0; argv_init[i]; i++) {534 if (i == MAX_INIT_ARGS) {535 panic_later = "init";536 panic_param = param;537 return 0;538 }539 }540 argv_init[i] = param;541 return 0;542}543 544/*545 * Unknown boot options get handed to init, unless they look like546 * unused parameters (modprobe will find them in /proc/cmdline).547 */548/* wasm32/HWJS-1: published suspend-reachability (see asm/hwjs.h). */549static int __init HWJS_SUSPENDS unknown_bootoption(char *param, char *val,550 const char *unused, void *arg)551{552 size_t len = strlen(param);553 554 /* Handle params aliased to sysctls */555 if (sysctl_is_alias(param))556 return 0;557 558 repair_env_string(param, val);559 560 /* Handle obsolete-style parameters */561 if (obsolete_checksetup(param))562 return 0;563 564 /* Unused module parameter. */565 if (strnchr(param, len, '.'))566 return 0;567 568 if (panic_later)569 return 0;570 571 if (val) {572 /* Environment option */573 unsigned int i;574 for (i = 0; envp_init[i]; i++) {575 if (i == MAX_INIT_ENVS) {576 panic_later = "env";577 panic_param = param;578 }579 if (!strncmp(param, envp_init[i], len+1))580 break;581 }582 envp_init[i] = param;583 } else {584 /* Command line option */585 unsigned int i;586 for (i = 0; argv_init[i]; i++) {587 if (i == MAX_INIT_ARGS) {588 panic_later = "init";589 panic_param = param;590 }591 }592 argv_init[i] = param;593 }594 return 0;595}596 597static int __init init_setup(char *str)598{599 unsigned int i;600 601 execute_command = str;602 /*603 * In case LILO is going to boot us with default command line,604 * it prepends "auto" before the whole cmdline which makes605 * the shell think it should execute a script with such name.606 * So we ignore all arguments entered _before_ init=... [MJ]607 */608 for (i = 1; i < MAX_INIT_ARGS; i++)609 argv_init[i] = NULL;610 return 1;611}612__setup("init=", init_setup);613 614static int __init rdinit_setup(char *str)615{616 unsigned int i;617 618 ramdisk_execute_command = str;619 /* See "auto" comment in init_setup */620 for (i = 1; i < MAX_INIT_ARGS; i++)621 argv_init[i] = NULL;622 return 1;623}624__setup("rdinit=", rdinit_setup);625 626#ifndef CONFIG_SMP627static inline void setup_nr_cpu_ids(void) { }628static inline void smp_prepare_cpus(unsigned int maxcpus) { }629#endif630 631/*632 * We need to store the untouched command line for future reference.633 * We also need to store the touched command line since the parameter634 * parsing is performed in place, and we should allow a component to635 * store reference of name/value for future reference.636 */637static void __init setup_command_line(char *command_line)638{639 size_t len, xlen = 0, ilen = 0;640 641 if (extra_command_line)642 xlen = strlen(extra_command_line);643 if (extra_init_args) {644 extra_init_args = strim(extra_init_args); /* remove trailing space */645 ilen = strlen(extra_init_args) + 4; /* for " -- " */646 }647 648 len = xlen + strlen(boot_command_line) + ilen + 1;649 650 saved_command_line = memblock_alloc(len, SMP_CACHE_BYTES);651 if (!saved_command_line)652 panic("%s: Failed to allocate %zu bytes\n", __func__, len);653 654 len = xlen + strlen(command_line) + 1;655 656 static_command_line = memblock_alloc(len, SMP_CACHE_BYTES);657 if (!static_command_line)658 panic("%s: Failed to allocate %zu bytes\n", __func__, len);659 660 if (xlen) {661 /*662 * We have to put extra_command_line before boot command663 * lines because there could be dashes (separator of init664 * command line) in the command lines.665 */666 strcpy(saved_command_line, extra_command_line);667 strcpy(static_command_line, extra_command_line);668 }669 strcpy(saved_command_line + xlen, boot_command_line);670 strcpy(static_command_line + xlen, command_line);671 672 if (ilen) {673 /*674 * Append supplemental init boot args to saved_command_line675 * so that user can check what command line options passed676 * to init.677 * The order should always be678 * " -- "[bootconfig init-param][cmdline init-param]679 */680 if (initargs_offs) {681 len = xlen + initargs_offs;682 strcpy(saved_command_line + len, extra_init_args);683 len += ilen - 4; /* strlen(extra_init_args) */684 strcpy(saved_command_line + len,685 boot_command_line + initargs_offs - 1);686 } else {687 len = strlen(saved_command_line);688 strcpy(saved_command_line + len, " -- ");689 len += 4;690 strcpy(saved_command_line + len, extra_init_args);691 }692 }693 694 saved_command_line_len = strlen(saved_command_line);695}696 697/*698 * We need to finalize in a non-__init function or else race conditions699 * between the root thread and the init thread may cause start_kernel to700 * be reaped by free_initmem before the root thread has proceeded to701 * cpu_idle.702 *703 * gcc-3.4 accidentally inlines this function, so use noinline.704 */705 706static __initdata DECLARE_COMPLETION(kthreadd_done);707 708static HWJS_SUSPENDS noinline void __ref __noreturn rest_init(void)709{710 struct task_struct *tsk;711 int pid;712 713 rcu_scheduler_starting();714 /*715 * We need to spawn init first so that it obtains pid 1, however716 * the init task will end up wanting to create kthreads, which, if717 * we schedule it before we create kthreadd, will OOPS.718 */719 pid = user_mode_thread(kernel_init, NULL, CLONE_FS);720 /*721 * Pin init on the boot CPU. Task migration is not properly working722 * until sched_init_smp() has been run. It will set the allowed723 * CPUs for init to the non isolated CPUs.724 */725 rcu_read_lock();726 tsk = find_task_by_pid_ns(pid, &init_pid_ns);727 tsk->flags |= PF_NO_SETAFFINITY;728 set_cpus_allowed_ptr(tsk, cpumask_of(smp_processor_id()));729 rcu_read_unlock();730 731 numa_default_policy();732 pid = kernel_thread(kthreadd, NULL, NULL, CLONE_FS | CLONE_FILES);733 rcu_read_lock();734 kthreadd_task = find_task_by_pid_ns(pid, &init_pid_ns);735 rcu_read_unlock();736 737 /*738 * Enable might_sleep() and smp_processor_id() checks.739 * They cannot be enabled earlier because with CONFIG_PREEMPTION=y740 * kernel_thread() would trigger might_sleep() splats. With741 * CONFIG_PREEMPT_VOLUNTARY=y the init task might have scheduled742 * already, but it's stuck on the kthreadd_done completion.743 */744 system_state = SYSTEM_SCHEDULING;745 746 complete(&kthreadd_done);747 748 /*749 * The boot idle thread must execute schedule()750 * at least once to get things moving:751 */752 schedule_preempt_disabled();753 /* Call into cpu_idle with preempt disabled */754 cpu_startup_entry(CPUHP_ONLINE);755}756 757/* Check for early params. */758/* wasm32/HWJS-1: published suspend-reachability (see asm/hwjs.h). */759static int __init HWJS_SUSPENDS do_early_param(char *param, char *val,760 const char *unused, void *arg)761{762 const struct obs_kernel_param *p;763 764 for (p = __setup_start; p < __setup_end; p++) {765 if ((p->early && parameq(param, p->str)) ||766 (strcmp(param, "console") == 0 &&767 strcmp(p->str, "earlycon") == 0)768 ) {769 if (p->setup_func(val) != 0)770 pr_warn("Malformed early option '%s'\n", param);771 }772 }773 /* We accept everything at this stage. */774 return 0;775}776 777void __init parse_early_options(char *cmdline)778{779 parse_args("early options", cmdline, NULL, 0, 0, 0, NULL,780 do_early_param);781}782 783/* Arch code calls this early on, or if not, just before other parsing. */784void __init parse_early_param(void)785{786 static int done __initdata;787 static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata;788 789 if (done)790 return;791 792 /* All fall through to do_early_param. */793 strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);794 parse_early_options(tmp_cmdline);795 done = 1;796}797 798void __init __weak arch_post_acpi_subsys_init(void) { }799 800void __init __weak smp_setup_processor_id(void)801{802}803 804void __init __weak smp_prepare_boot_cpu(void)805{806}807 808# if THREAD_SIZE >= PAGE_SIZE809void __init __weak thread_stack_cache_init(void)810{811}812#endif813 814void __init __weak poking_init(void) { }815 816void __init __weak pgtable_cache_init(void) { }817 818void __init __weak trap_init(void) { }819 820bool initcall_debug;821core_param(initcall_debug, initcall_debug, bool, 0644);822 823#ifdef TRACEPOINTS_ENABLED824static void __init initcall_debug_enable(void);825#else826static inline void initcall_debug_enable(void)827{828}829#endif830 831#ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET832DEFINE_STATIC_KEY_MAYBE_RO(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT,833 randomize_kstack_offset);834DEFINE_PER_CPU(u32, kstack_offset);835 836static int __init early_randomize_kstack_offset(char *buf)837{838 int ret;839 bool bool_result;840 841 ret = kstrtobool(buf, &bool_result);842 if (ret)843 return ret;844 845 if (bool_result)846 static_branch_enable(&randomize_kstack_offset);847 else848 static_branch_disable(&randomize_kstack_offset);849 return 0;850}851early_param("randomize_kstack_offset", early_randomize_kstack_offset);852#endif853 854static void __init print_unknown_bootoptions(void)855{856 char *unknown_options;857 char *end;858 const char *const *p;859 size_t len;860 861 if (panic_later || (!argv_init[1] && !envp_init[2]))862 return;863 864 /*865 * Determine how many options we have to print out, plus a space866 * before each867 */868 len = 1; /* null terminator */869 for (p = &argv_init[1]; *p; p++) {870 len++;871 len += strlen(*p);872 }873 for (p = &envp_init[2]; *p; p++) {874 len++;875 len += strlen(*p);876 }877 878 unknown_options = memblock_alloc(len, SMP_CACHE_BYTES);879 if (!unknown_options) {880 pr_err("%s: Failed to allocate %zu bytes\n",881 __func__, len);882 return;883 }884 end = unknown_options;885 886 for (p = &argv_init[1]; *p; p++)887 end += sprintf(end, " %s", *p);888 for (p = &envp_init[2]; *p; p++)889 end += sprintf(end, " %s", *p);890 891 /* Start at unknown_options[1] to skip the initial space */892 pr_notice("Unknown kernel command line parameters \"%s\", will be passed to user space.\n",893 &unknown_options[1]);894 memblock_free(unknown_options, len);895}896 897static void __init early_numa_node_init(void)898{899#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID900#ifndef cpu_to_node901 int cpu;902 903 /* The early_cpu_to_node() should be ready here. */904 for_each_possible_cpu(cpu)905 set_cpu_numa_node(cpu, early_cpu_to_node(cpu));906#endif907#endif908}909 910asmlinkage __visible __init __no_sanitize_address __noreturn __no_stack_protector911void start_kernel(void)912{913 char *command_line;914 char *after_dashes;915 916 set_task_stack_end_magic(&init_task);917 smp_setup_processor_id();918 debug_objects_early_init();919 init_vmlinux_build_id();920 921 cgroup_init_early();922 923 local_irq_disable();924 early_boot_irqs_disabled = true;925 926 /*927 * Interrupts are still disabled. Do necessary setups, then928 * enable them.929 */930 boot_cpu_init();931 page_address_init();932 pr_notice("%s", linux_banner);933 setup_arch(&command_line);934 /* Static keys and static calls are needed by LSMs */935 jump_label_init();936 static_call_init();937 early_security_init();938 setup_boot_config();939 setup_command_line(command_line);940 setup_nr_cpu_ids();941 setup_per_cpu_areas();942 smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */943 early_numa_node_init();944 boot_cpu_hotplug_init();945 946 pr_notice("Kernel command line: %s\n", saved_command_line);947 /* parameters may set static keys */948 parse_early_param();949 after_dashes = parse_args("Booting kernel",950 static_command_line, __start___param,951 __stop___param - __start___param,952 -1, -1, NULL, &unknown_bootoption);953 print_unknown_bootoptions();954 if (!IS_ERR_OR_NULL(after_dashes))955 parse_args("Setting init args", after_dashes, NULL, 0, -1, -1,956 NULL, set_init_arg);957 if (extra_init_args)958 parse_args("Setting extra init args", extra_init_args,959 NULL, 0, -1, -1, NULL, set_init_arg);960 961 /* Architectural and non-timekeeping rng init, before allocator init */962 random_init_early(command_line);963 964 /*965 * These use large bootmem allocations and must precede966 * initalization of page allocator967 */968 setup_log_buf(0);969 vfs_caches_init_early();970 sort_main_extable();971 trap_init();972 mm_core_init();973 poking_init();974 ftrace_init();975 976 /* trace_printk can be enabled here */977 early_trace_init();978 979 /*980 * Set up the scheduler prior starting any interrupts (such as the981 * timer interrupt). Full topology setup happens at smp_init()982 * time - but meanwhile we still have a functioning scheduler.983 */984 sched_init();985 986 if (WARN(!irqs_disabled(),987 "Interrupts were enabled *very* early, fixing it\n"))988 local_irq_disable();989 radix_tree_init();990 maple_tree_init();991 992 /*993 * Set up housekeeping before setting up workqueues to allow the unbound994 * workqueue to take non-housekeeping into account.995 */996 housekeeping_init();997 998 /*999 * Allow workqueue creation and work item queueing/cancelling1000 * early. Work item execution depends on kthreads and starts after1001 * workqueue_init().1002 */1003 workqueue_init_early();1004 1005 rcu_init();1006 1007 /* Trace events are available after this */1008 trace_init();1009 1010 if (initcall_debug)1011 initcall_debug_enable();1012 1013 context_tracking_init();1014 /* init some links before init_ISA_irqs() */1015 early_irq_init();1016 init_IRQ();1017 tick_init();1018 rcu_init_nohz();1019 init_timers();1020 srcu_init();1021 hrtimers_init();1022 softirq_init();1023 timekeeping_init();1024 time_init();1025 1026 /* This must be after timekeeping is initialized */1027 random_init();1028 1029 /* These make use of the fully initialized rng */1030 kfence_init();1031 boot_init_stack_canary();1032 1033 perf_event_init();1034 profile_init();1035 call_function_init();1036 WARN(!irqs_disabled(), "Interrupts were enabled early\n");1037 1038 early_boot_irqs_disabled = false;1039 local_irq_enable();1040 1041 kmem_cache_init_late();1042 1043 /*1044 * HACK ALERT! This is early. We're enabling the console before1045 * we've done PCI setups etc, and console_init() must be aware of1046 * this. But we do want output early, in case something goes wrong.1047 */1048 console_init();1049 if (panic_later)1050 panic("Too many boot %s vars at `%s'", panic_later,1051 panic_param);1052 1053 lockdep_init();1054 1055 /*1056 * Need to run this when irqs are enabled, because it wants1057 * to self-test [hard/soft]-irqs on/off lock inversion bugs1058 * too:1059 */1060 locking_selftest();1061 1062#ifdef CONFIG_BLK_DEV_INITRD1063 if (initrd_start && !initrd_below_start_ok &&1064 page_to_pfn(virt_to_page((void *)initrd_start)) < min_low_pfn) {1065 pr_crit("initrd overwritten (0x%08lx < 0x%08lx) - disabling it.\n",1066 page_to_pfn(virt_to_page((void *)initrd_start)),1067 min_low_pfn);1068 initrd_start = 0;1069 }1070#endif1071 setup_per_cpu_pageset();1072 numa_policy_init();1073 acpi_early_init();1074 if (late_time_init)1075 late_time_init();1076 sched_clock_init();1077 calibrate_delay();1078 1079 arch_cpu_finalize_init();1080 1081 pid_idr_init();1082 anon_vma_init();1083#ifdef CONFIG_X861084 if (efi_enabled(EFI_RUNTIME_SERVICES))1085 efi_enter_virtual_mode();1086#endif1087 thread_stack_cache_init();1088 cred_init();1089 fork_init();1090 proc_caches_init();1091 uts_ns_init();1092 key_init();1093 security_init();1094 dbg_late_init();1095 net_ns_init();1096 vfs_caches_init();1097 pagecache_init();1098 signals_init();1099 seq_file_init();1100 proc_root_init();1101 nsfs_init();1102 pidfs_init();1103 cpuset_init();1104 cgroup_init();1105 taskstats_init_early();1106 delayacct_init();1107 1108 acpi_subsystem_init();1109 arch_post_acpi_subsys_init();1110 kcsan_init();1111 1112 /* Do the rest non-__init'ed, we're now alive */1113 rest_init();1114 1115 /*1116 * Avoid stack canaries in callers of boot_init_stack_canary for gcc-101117 * and older.1118 */1119#if !__has_attribute(__no_stack_protector__)1120 prevent_tail_call_optimization();1121#endif1122}1123 1124/* Call all constructor functions linked into the kernel. */1125static void __init do_ctors(void)1126{1127/*1128 * For UML, the constructors have already been called by the1129 * normal setup code as it's just a normal ELF binary, so we1130 * cannot do it again - but we do need CONFIG_CONSTRUCTORS1131 * even on UML for modules.1132 */1133#if defined(CONFIG_CONSTRUCTORS) && !defined(CONFIG_UML)1134 ctor_fn_t *fn = (ctor_fn_t *) __ctors_start;1135 1136 for (; fn < (ctor_fn_t *) __ctors_end; fn++)1137 (*fn)();1138#endif1139}1140 1141#ifdef CONFIG_KALLSYMS1142struct blacklist_entry {1143 struct list_head next;1144 char *buf;1145};1146 1147static __initdata_or_module LIST_HEAD(blacklisted_initcalls);1148 1149static int __init initcall_blacklist(char *str)1150{1151 char *str_entry;1152 struct blacklist_entry *entry;1153 1154 /* str argument is a comma-separated list of functions */1155 do {1156 str_entry = strsep(&str, ",");1157 if (str_entry) {1158 pr_debug("blacklisting initcall %s\n", str_entry);1159 entry = memblock_alloc(sizeof(*entry),1160 SMP_CACHE_BYTES);1161 if (!entry)1162 panic("%s: Failed to allocate %zu bytes\n",1163 __func__, sizeof(*entry));1164 entry->buf = memblock_alloc(strlen(str_entry) + 1,1165 SMP_CACHE_BYTES);1166 if (!entry->buf)1167 panic("%s: Failed to allocate %zu bytes\n",1168 __func__, strlen(str_entry) + 1);1169 strcpy(entry->buf, str_entry);1170 list_add(&entry->next, &blacklisted_initcalls);1171 }1172 } while (str_entry);1173 1174 return 1;1175}1176 1177static bool __init_or_module initcall_blacklisted(initcall_t fn)1178{1179 struct blacklist_entry *entry;1180 char fn_name[KSYM_SYMBOL_LEN];1181 unsigned long addr;1182 1183 if (list_empty(&blacklisted_initcalls))1184 return false;1185 1186 addr = (unsigned long) dereference_function_descriptor(fn);1187 sprint_symbol_no_offset(fn_name, addr);1188 1189 /*1190 * fn will be "function_name [module_name]" where [module_name] is not1191 * displayed for built-in init functions. Strip off the [module_name].1192 */1193 strreplace(fn_name, ' ', '\0');1194 1195 list_for_each_entry(entry, &blacklisted_initcalls, next) {1196 if (!strcmp(fn_name, entry->buf)) {1197 pr_debug("initcall %s blacklisted\n", fn_name);1198 return true;1199 }1200 }1201 1202 return false;1203}1204#else1205static int __init initcall_blacklist(char *str)1206{1207 pr_warn("initcall_blacklist requires CONFIG_KALLSYMS\n");1208 return 0;1209}1210 1211static bool __init_or_module initcall_blacklisted(initcall_t fn)1212{1213 return false;1214}1215#endif1216__setup("initcall_blacklist=", initcall_blacklist);1217 1218static __init_or_module void1219trace_initcall_start_cb(void *data, initcall_t fn)1220{1221 ktime_t *calltime = data;1222 1223 printk(KERN_DEBUG "calling %pS @ %i\n", fn, task_pid_nr(current));1224 *calltime = ktime_get();1225}1226 1227static __init_or_module void1228trace_initcall_finish_cb(void *data, initcall_t fn, int ret)1229{1230 ktime_t rettime, *calltime = data;1231 1232 rettime = ktime_get();1233 printk(KERN_DEBUG "initcall %pS returned %d after %lld usecs\n",1234 fn, ret, (unsigned long long)ktime_us_delta(rettime, *calltime));1235}1236 1237static ktime_t initcall_calltime;1238 1239#ifdef TRACEPOINTS_ENABLED1240static void __init initcall_debug_enable(void)1241{1242 int ret;1243 1244 ret = register_trace_initcall_start(trace_initcall_start_cb,1245 &initcall_calltime);1246 ret |= register_trace_initcall_finish(trace_initcall_finish_cb,1247 &initcall_calltime);1248 WARN(ret, "Failed to register initcall tracepoints\n");1249}1250# define do_trace_initcall_start trace_initcall_start1251# define do_trace_initcall_finish trace_initcall_finish1252#else1253static inline void do_trace_initcall_start(initcall_t fn)1254{1255 if (!initcall_debug)1256 return;1257 trace_initcall_start_cb(&initcall_calltime, fn);1258}1259static inline void do_trace_initcall_finish(initcall_t fn, int ret)1260{1261 if (!initcall_debug)1262 return;1263 trace_initcall_finish_cb(&initcall_calltime, fn, ret);1264}1265#endif /* !TRACEPOINTS_ENABLED */1266 1267int __init_or_module do_one_initcall(initcall_t fn)1268{1269 int count = preempt_count();1270 char msgbuf[64];1271 int ret;1272 1273 if (!fn)1274 return 0;1275 1276 if (initcall_blacklisted(fn))1277 return -EPERM;1278 1279 do_trace_initcall_start(fn);1280 ret = fn();1281 do_trace_initcall_finish(fn, ret);1282 1283 msgbuf[0] = 0;1284 1285 if (preempt_count() != count) {1286 sprintf(msgbuf, "preemption imbalance ");1287 preempt_count_set(count);1288 }1289 if (irqs_disabled()) {1290 strlcat(msgbuf, "disabled interrupts ", sizeof(msgbuf));1291 local_irq_enable();1292 }1293 WARN(msgbuf[0], "initcall %pS returned with %s\n", fn, msgbuf);1294 1295 add_latent_entropy();1296 return ret;1297}1298 1299 1300static initcall_entry_t *initcall_levels[] __initdata = {1301 __initcall0_start,1302 __initcall1_start,1303 __initcall2_start,1304 __initcall3_start,1305 __initcall4_start,1306 __initcall5_start,1307 __initcall6_start,1308 __initcall7_start,1309 __initcall_end,1310};1311 1312/* Keep these in sync with initcalls in include/linux/init.h */1313static const char *initcall_level_names[] __initdata = {1314 "pure",1315 "core",1316 "postcore",1317 "arch",1318 "subsys",1319 "fs",1320 "device",1321 "late",1322};1323 1324static int __init ignore_unknown_bootoption(char *param, char *val,1325 const char *unused, void *arg)1326{1327 return 0;1328}1329 1330static void __init do_initcall_level(int level, char *command_line)1331{1332 initcall_entry_t *fn;1333 1334 parse_args(initcall_level_names[level],1335 command_line, __start___param,1336 __stop___param - __start___param,1337 level, level,1338 NULL, ignore_unknown_bootoption);1339 1340 trace_initcall_level(initcall_level_names[level]);1341 for (fn = initcall_levels[level]; fn < initcall_levels[level+1]; fn++) {1342 if (!initcall_from_entry(fn)) continue;1343 do_one_initcall(initcall_from_entry(fn));1344 }1345}1346 1347static void __init do_initcalls(void)1348{1349 int level;1350 size_t len = saved_command_line_len + 1;1351 char *command_line;1352 1353 command_line = kzalloc(len, GFP_KERNEL);1354 if (!command_line)1355 panic("%s: Failed to allocate %zu bytes\n", __func__, len);1356 1357 for (level = 0; level < ARRAY_SIZE(initcall_levels) - 1; level++) {1358 /* Parser modifies command_line, restore it each time */1359 strcpy(command_line, saved_command_line);1360 do_initcall_level(level, command_line);1361 }1362 1363 kfree(command_line);1364}1365 1366/*1367 * Ok, the machine is now initialized. None of the devices1368 * have been touched yet, but the CPU subsystem is up and1369 * running, and memory and process management works.1370 *1371 * Now we can finally start doing some real work..1372 */1373static void __init do_basic_setup(void)1374{1375 cpuset_init_smp();1376 driver_init();1377 init_irq_proc();1378 do_ctors();1379 do_initcalls();1380}1381 1382static void __init do_pre_smp_initcalls(void)1383{1384 initcall_entry_t *fn;1385 1386 trace_initcall_level("early");1387 for (fn = __initcall_start; fn < __initcall0_start; fn++) {1388 if (!initcall_from_entry(fn)) continue;1389 do_one_initcall(initcall_from_entry(fn));1390 }1391}1392 1393static HWJS_SUSPENDS int run_init_process(const char *init_filename)1394{1395 const char *const *p;1396 1397 argv_init[0] = init_filename;1398 pr_info("Run %s as init process\n", init_filename);1399 pr_debug(" with arguments:\n");1400 for (p = argv_init; *p; p++)1401 pr_debug(" %s\n", *p);1402 pr_debug(" with environment:\n");1403 for (p = envp_init; *p; p++)1404 pr_debug(" %s\n", *p);1405 return kernel_execve(init_filename, argv_init, envp_init);1406}1407 1408static HWJS_SUSPENDS int try_to_run_init_process(const char *init_filename)1409{1410 int ret;1411 1412 ret = run_init_process(init_filename);1413 1414 if (ret && ret != -ENOENT) {1415 pr_err("Starting init: %s exists but couldn't execute it (error %d)\n",1416 init_filename, ret);1417 }1418 1419 return ret;1420}1421 1422static HWJS_SUSPENDS noinline void __init kernel_init_freeable(void);1423 1424#if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_STRICT_MODULE_RWX)1425bool rodata_enabled __ro_after_init = true;1426 1427#ifndef arch_parse_debug_rodata1428static inline bool arch_parse_debug_rodata(char *str) { return false; }1429#endif1430 1431static int __init set_debug_rodata(char *str)1432{1433 if (arch_parse_debug_rodata(str))1434 return 0;1435 1436 if (str && !strcmp(str, "on"))1437 rodata_enabled = true;1438 else if (str && !strcmp(str, "off"))1439 rodata_enabled = false;1440 else1441 pr_warn("Invalid option string for rodata: '%s'\n", str);1442 return 0;1443}1444early_param("rodata", set_debug_rodata);1445#endif1446 1447static void mark_readonly(void)1448{1449 if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX) && rodata_enabled) {1450 /*1451 * load_module() results in W+X mappings, which are cleaned1452 * up with init_free_wq. Let's make sure that queued work is1453 * flushed so that we don't hit false positives looking for1454 * insecure pages which are W+X.1455 */1456 flush_module_init_free_work();1457 jump_label_init_ro();1458 mark_rodata_ro();1459 debug_checkwx();1460 rodata_test();1461 } else if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) {1462 pr_info("Kernel memory protection disabled.\n");1463 } else if (IS_ENABLED(CONFIG_ARCH_HAS_STRICT_KERNEL_RWX)) {1464 pr_warn("Kernel memory protection not selected by kernel config.\n");1465 } else {1466 pr_warn("This architecture does not have kernel memory protection.\n");1467 }1468}1469 1470void __weak free_initmem(void)1471{1472 free_initmem_default(POISON_FREE_INITMEM);1473}1474 1475static int __ref kernel_init(void *unused)1476{1477 int ret;1478 1479 /*1480 * Wait until kthreadd is all set-up.1481 */1482 wait_for_completion(&kthreadd_done);1483 1484 kernel_init_freeable();1485 /* need to finish all async __init code before freeing the memory */1486 async_synchronize_full();1487 1488 system_state = SYSTEM_FREEING_INITMEM;1489 kprobe_free_init_mem();1490 ftrace_free_init_mem();1491 kgdb_free_init_mem();1492 exit_boot_config();1493 free_initmem();1494 mark_readonly();1495 1496 /*1497 * Kernel mappings are now finalized - update the userspace page-table1498 * to finalize PTI.1499 */1500 pti_finalize();1501 1502 system_state = SYSTEM_RUNNING;1503 numa_default_policy();1504 1505 rcu_end_inkernel_boot();1506 /*1507 * arch_kernel_init_pre_run_init - arch-specific hook called1508 * before kernel_init iterates /init candidate paths.1509 *1510 * Default is no-op. Arches without a working filesystem1511 * (wasm32 at K5) may override this to dispatch to an1512 * embedded init image. If the override successfully spawns1513 * the init process, it should not return (typically by1514 * parking current); on return, kernel_init proceeds with1515 * the standard /init iteration.1516 */1517 arch_kernel_init_pre_run_init();1518 1519 do_sysctl_args();1520 1521 if (ramdisk_execute_command) {1522 ret = run_init_process(ramdisk_execute_command);1523 if (!ret)1524 return 0;1525 pr_err("Failed to execute %s (error %d)\n",1526 ramdisk_execute_command, ret);1527 }1528 1529 /*1530 * We try each of these until one succeeds.1531 *1532 * The Bourne shell can be used instead of init if we are1533 * trying to recover a really broken machine.1534 */1535 if (execute_command) {1536 ret = run_init_process(execute_command);1537 if (!ret)1538 return 0;1539 panic("Requested init %s failed (error %d).",1540 execute_command, ret);1541 }1542 1543 if (CONFIG_DEFAULT_INIT[0] != '\0') {1544 ret = run_init_process(CONFIG_DEFAULT_INIT);1545 if (ret)1546 pr_err("Default init %s failed (error %d)\n",1547 CONFIG_DEFAULT_INIT, ret);1548 else1549 return 0;1550 }1551 1552 if (!try_to_run_init_process("/sbin/init") ||1553 !try_to_run_init_process("/etc/init") ||1554 !try_to_run_init_process("/bin/init") ||1555 !try_to_run_init_process("/bin/sh"))1556 return 0;1557 1558 panic("No working init found. Try passing init= option to kernel. "1559 "See Linux Documentation/admin-guide/init.rst for guidance.");1560}1561 1562void __weak arch_kernel_init_pre_run_init(void) { }1563 1564/* Open /dev/console, for stdin/stdout/stderr, this should never fail */1565void __init console_on_rootfs(void)1566{1567 struct file *file = filp_open("/dev/console", O_RDWR, 0);1568 1569 if (IS_ERR(file)) {1570 pr_err("Warning: unable to open an initial console.\n");1571 return;1572 }1573 init_dup(file);1574 init_dup(file);1575 init_dup(file);1576 fput(file);1577}1578 1579static HWJS_SUSPENDS noinline void __init kernel_init_freeable(void)1580{1581 /* Now the scheduler is fully set up and can do blocking allocations */1582 gfp_allowed_mask = __GFP_BITS_MASK;1583 1584 /*1585 * init can allocate pages on any node1586 */1587 set_mems_allowed(node_states[N_MEMORY]);1588 1589 cad_pid = get_pid(task_pid(current));1590 1591 smp_prepare_cpus(setup_max_cpus);1592 1593 workqueue_init();1594 1595 init_mm_internals();1596 1597 rcu_init_tasks_generic();1598 do_pre_smp_initcalls();1599 lockup_detector_init();1600 1601 smp_init();1602 sched_init_smp();1603 1604 workqueue_init_topology();1605 async_init();1606 padata_init();1607 page_alloc_init_late();1608 1609 do_basic_setup();1610 1611 kunit_run_all_tests();1612 1613 wait_for_initramfs();1614 console_on_rootfs();1615 1616 /*1617 * check if there is an early userspace init. If yes, let it do all1618 * the work1619 */1620 if (init_eaccess(ramdisk_execute_command) != 0) {1621 ramdisk_execute_command = NULL;1622 prepare_namespace();1623 }1624 1625 /*1626 * Ok, we have completed the initial bootup, and1627 * we're essentially up and running. Get rid of the1628 * initmem segments and start the user-mode stuff..1629 *1630 * rootfs is available now, try loading the public keys1631 * and default modules1632 */1633 1634 integrity_load_keys();1635}1636