xref: /linux/arch/arm64/kernel/hibernate.c (revision af873fce)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*:
3  * Hibernate support specific for ARM64
4  *
5  * Derived from work on ARM hibernation support by:
6  *
7  * Ubuntu project, hibernation support for mach-dove
8  * Copyright (C) 2010 Nokia Corporation (Hiroshi Doyu)
9  * Copyright (C) 2010 Texas Instruments, Inc. (Teerth Reddy et al.)
10  *  https://lkml.org/lkml/2010/6/18/4
11  *  https://lists.linux-foundation.org/pipermail/linux-pm/2010-June/027422.html
12  *  https://patchwork.kernel.org/patch/96442/
13  *
14  * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
15  */
16 #define pr_fmt(x) "hibernate: " x
17 #include <linux/cpu.h>
18 #include <linux/kvm_host.h>
19 #include <linux/mm.h>
20 #include <linux/pm.h>
21 #include <linux/sched.h>
22 #include <linux/suspend.h>
23 #include <linux/utsname.h>
24 #include <linux/version.h>
25 
26 #include <asm/barrier.h>
27 #include <asm/cacheflush.h>
28 #include <asm/cputype.h>
29 #include <asm/daifflags.h>
30 #include <asm/irqflags.h>
31 #include <asm/kexec.h>
32 #include <asm/memory.h>
33 #include <asm/mmu_context.h>
34 #include <asm/pgalloc.h>
35 #include <asm/pgtable.h>
36 #include <asm/pgtable-hwdef.h>
37 #include <asm/sections.h>
38 #include <asm/smp.h>
39 #include <asm/smp_plat.h>
40 #include <asm/suspend.h>
41 #include <asm/sysreg.h>
42 #include <asm/virt.h>
43 
44 /*
45  * Hibernate core relies on this value being 0 on resume, and marks it
46  * __nosavedata assuming it will keep the resume kernel's '0' value. This
47  * doesn't happen with either KASLR.
48  *
49  * defined as "__visible int in_suspend __nosavedata" in
50  * kernel/power/hibernate.c
51  */
52 extern int in_suspend;
53 
54 /* Do we need to reset el2? */
55 #define el2_reset_needed() (is_hyp_mode_available() && !is_kernel_in_hyp_mode())
56 
57 /* temporary el2 vectors in the __hibernate_exit_text section. */
58 extern char hibernate_el2_vectors[];
59 
60 /* hyp-stub vectors, used to restore el2 during resume from hibernate. */
61 extern char __hyp_stub_vectors[];
62 
63 /*
64  * The logical cpu number we should resume on, initialised to a non-cpu
65  * number.
66  */
67 static int sleep_cpu = -EINVAL;
68 
69 /*
70  * Values that may not change over hibernate/resume. We put the build number
71  * and date in here so that we guarantee not to resume with a different
72  * kernel.
73  */
74 struct arch_hibernate_hdr_invariants {
75 	char		uts_version[__NEW_UTS_LEN + 1];
76 };
77 
78 /* These values need to be know across a hibernate/restore. */
79 static struct arch_hibernate_hdr {
80 	struct arch_hibernate_hdr_invariants invariants;
81 
82 	/* These are needed to find the relocated kernel if built with kaslr */
83 	phys_addr_t	ttbr1_el1;
84 	void		(*reenter_kernel)(void);
85 
86 	/*
87 	 * We need to know where the __hyp_stub_vectors are after restore to
88 	 * re-configure el2.
89 	 */
90 	phys_addr_t	__hyp_stub_vectors;
91 
92 	u64		sleep_cpu_mpidr;
93 } resume_hdr;
94 
95 static inline void arch_hdr_invariants(struct arch_hibernate_hdr_invariants *i)
96 {
97 	memset(i, 0, sizeof(*i));
98 	memcpy(i->uts_version, init_utsname()->version, sizeof(i->uts_version));
99 }
100 
101 int pfn_is_nosave(unsigned long pfn)
102 {
103 	unsigned long nosave_begin_pfn = sym_to_pfn(&__nosave_begin);
104 	unsigned long nosave_end_pfn = sym_to_pfn(&__nosave_end - 1);
105 
106 	return ((pfn >= nosave_begin_pfn) && (pfn <= nosave_end_pfn)) ||
107 		crash_is_nosave(pfn);
108 }
109 
110 void notrace save_processor_state(void)
111 {
112 	WARN_ON(num_online_cpus() != 1);
113 }
114 
115 void notrace restore_processor_state(void)
116 {
117 }
118 
119 int arch_hibernation_header_save(void *addr, unsigned int max_size)
120 {
121 	struct arch_hibernate_hdr *hdr = addr;
122 
123 	if (max_size < sizeof(*hdr))
124 		return -EOVERFLOW;
125 
126 	arch_hdr_invariants(&hdr->invariants);
127 	hdr->ttbr1_el1		= __pa_symbol(swapper_pg_dir);
128 	hdr->reenter_kernel	= _cpu_resume;
129 
130 	/* We can't use __hyp_get_vectors() because kvm may still be loaded */
131 	if (el2_reset_needed())
132 		hdr->__hyp_stub_vectors = __pa_symbol(__hyp_stub_vectors);
133 	else
134 		hdr->__hyp_stub_vectors = 0;
135 
136 	/* Save the mpidr of the cpu we called cpu_suspend() on... */
137 	if (sleep_cpu < 0) {
138 		pr_err("Failing to hibernate on an unknown CPU.\n");
139 		return -ENODEV;
140 	}
141 	hdr->sleep_cpu_mpidr = cpu_logical_map(sleep_cpu);
142 	pr_info("Hibernating on CPU %d [mpidr:0x%llx]\n", sleep_cpu,
143 		hdr->sleep_cpu_mpidr);
144 
145 	return 0;
146 }
147 EXPORT_SYMBOL(arch_hibernation_header_save);
148 
149 int arch_hibernation_header_restore(void *addr)
150 {
151 	int ret;
152 	struct arch_hibernate_hdr_invariants invariants;
153 	struct arch_hibernate_hdr *hdr = addr;
154 
155 	arch_hdr_invariants(&invariants);
156 	if (memcmp(&hdr->invariants, &invariants, sizeof(invariants))) {
157 		pr_crit("Hibernate image not generated by this kernel!\n");
158 		return -EINVAL;
159 	}
160 
161 	sleep_cpu = get_logical_index(hdr->sleep_cpu_mpidr);
162 	pr_info("Hibernated on CPU %d [mpidr:0x%llx]\n", sleep_cpu,
163 		hdr->sleep_cpu_mpidr);
164 	if (sleep_cpu < 0) {
165 		pr_crit("Hibernated on a CPU not known to this kernel!\n");
166 		sleep_cpu = -EINVAL;
167 		return -EINVAL;
168 	}
169 	if (!cpu_online(sleep_cpu)) {
170 		pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n");
171 		ret = cpu_up(sleep_cpu);
172 		if (ret) {
173 			pr_err("Failed to bring hibernate-CPU up!\n");
174 			sleep_cpu = -EINVAL;
175 			return ret;
176 		}
177 	}
178 
179 	resume_hdr = *hdr;
180 
181 	return 0;
182 }
183 EXPORT_SYMBOL(arch_hibernation_header_restore);
184 
185 /*
186  * Copies length bytes, starting at src_start into an new page,
187  * perform cache maintentance, then maps it at the specified address low
188  * address as executable.
189  *
190  * This is used by hibernate to copy the code it needs to execute when
191  * overwriting the kernel text. This function generates a new set of page
192  * tables, which it loads into ttbr0.
193  *
194  * Length is provided as we probably only want 4K of data, even on a 64K
195  * page system.
196  */
197 static int create_safe_exec_page(void *src_start, size_t length,
198 				 unsigned long dst_addr,
199 				 phys_addr_t *phys_dst_addr,
200 				 void *(*allocator)(gfp_t mask),
201 				 gfp_t mask)
202 {
203 	int rc = 0;
204 	pgd_t *pgdp;
205 	pud_t *pudp;
206 	pmd_t *pmdp;
207 	pte_t *ptep;
208 	unsigned long dst = (unsigned long)allocator(mask);
209 
210 	if (!dst) {
211 		rc = -ENOMEM;
212 		goto out;
213 	}
214 
215 	memcpy((void *)dst, src_start, length);
216 	__flush_icache_range(dst, dst + length);
217 
218 	pgdp = pgd_offset_raw(allocator(mask), dst_addr);
219 	if (pgd_none(READ_ONCE(*pgdp))) {
220 		pudp = allocator(mask);
221 		if (!pudp) {
222 			rc = -ENOMEM;
223 			goto out;
224 		}
225 		pgd_populate(&init_mm, pgdp, pudp);
226 	}
227 
228 	pudp = pud_offset(pgdp, dst_addr);
229 	if (pud_none(READ_ONCE(*pudp))) {
230 		pmdp = allocator(mask);
231 		if (!pmdp) {
232 			rc = -ENOMEM;
233 			goto out;
234 		}
235 		pud_populate(&init_mm, pudp, pmdp);
236 	}
237 
238 	pmdp = pmd_offset(pudp, dst_addr);
239 	if (pmd_none(READ_ONCE(*pmdp))) {
240 		ptep = allocator(mask);
241 		if (!ptep) {
242 			rc = -ENOMEM;
243 			goto out;
244 		}
245 		pmd_populate_kernel(&init_mm, pmdp, ptep);
246 	}
247 
248 	ptep = pte_offset_kernel(pmdp, dst_addr);
249 	set_pte(ptep, pfn_pte(virt_to_pfn(dst), PAGE_KERNEL_EXEC));
250 
251 	/*
252 	 * Load our new page tables. A strict BBM approach requires that we
253 	 * ensure that TLBs are free of any entries that may overlap with the
254 	 * global mappings we are about to install.
255 	 *
256 	 * For a real hibernate/resume cycle TTBR0 currently points to a zero
257 	 * page, but TLBs may contain stale ASID-tagged entries (e.g. for EFI
258 	 * runtime services), while for a userspace-driven test_resume cycle it
259 	 * points to userspace page tables (and we must point it at a zero page
260 	 * ourselves). Elsewhere we only (un)install the idmap with preemption
261 	 * disabled, so T0SZ should be as required regardless.
262 	 */
263 	cpu_set_reserved_ttbr0();
264 	local_flush_tlb_all();
265 	write_sysreg(phys_to_ttbr(virt_to_phys(pgdp)), ttbr0_el1);
266 	isb();
267 
268 	*phys_dst_addr = virt_to_phys((void *)dst);
269 
270 out:
271 	return rc;
272 }
273 
274 #define dcache_clean_range(start, end)	__flush_dcache_area(start, (end - start))
275 
276 int swsusp_arch_suspend(void)
277 {
278 	int ret = 0;
279 	unsigned long flags;
280 	struct sleep_stack_data state;
281 
282 	if (cpus_are_stuck_in_kernel()) {
283 		pr_err("Can't hibernate: no mechanism to offline secondary CPUs.\n");
284 		return -EBUSY;
285 	}
286 
287 	flags = local_daif_save();
288 
289 	if (__cpu_suspend_enter(&state)) {
290 		/* make the crash dump kernel image visible/saveable */
291 		crash_prepare_suspend();
292 
293 		sleep_cpu = smp_processor_id();
294 		ret = swsusp_save();
295 	} else {
296 		/* Clean kernel core startup/idle code to PoC*/
297 		dcache_clean_range(__mmuoff_data_start, __mmuoff_data_end);
298 		dcache_clean_range(__idmap_text_start, __idmap_text_end);
299 
300 		/* Clean kvm setup code to PoC? */
301 		if (el2_reset_needed()) {
302 			dcache_clean_range(__hyp_idmap_text_start, __hyp_idmap_text_end);
303 			dcache_clean_range(__hyp_text_start, __hyp_text_end);
304 		}
305 
306 		/* make the crash dump kernel image protected again */
307 		crash_post_resume();
308 
309 		/*
310 		 * Tell the hibernation core that we've just restored
311 		 * the memory
312 		 */
313 		in_suspend = 0;
314 
315 		sleep_cpu = -EINVAL;
316 		__cpu_suspend_exit();
317 
318 		/*
319 		 * Just in case the boot kernel did turn the SSBD
320 		 * mitigation off behind our back, let's set the state
321 		 * to what we expect it to be.
322 		 */
323 		switch (arm64_get_ssbd_state()) {
324 		case ARM64_SSBD_FORCE_ENABLE:
325 		case ARM64_SSBD_KERNEL:
326 			arm64_set_ssbd_mitigation(true);
327 		}
328 	}
329 
330 	local_daif_restore(flags);
331 
332 	return ret;
333 }
334 
335 static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr)
336 {
337 	pte_t pte = READ_ONCE(*src_ptep);
338 
339 	if (pte_valid(pte)) {
340 		/*
341 		 * Resume will overwrite areas that may be marked
342 		 * read only (code, rodata). Clear the RDONLY bit from
343 		 * the temporary mappings we use during restore.
344 		 */
345 		set_pte(dst_ptep, pte_mkwrite(pte));
346 	} else if (debug_pagealloc_enabled() && !pte_none(pte)) {
347 		/*
348 		 * debug_pagealloc will removed the PTE_VALID bit if
349 		 * the page isn't in use by the resume kernel. It may have
350 		 * been in use by the original kernel, in which case we need
351 		 * to put it back in our copy to do the restore.
352 		 *
353 		 * Before marking this entry valid, check the pfn should
354 		 * be mapped.
355 		 */
356 		BUG_ON(!pfn_valid(pte_pfn(pte)));
357 
358 		set_pte(dst_ptep, pte_mkpresent(pte_mkwrite(pte)));
359 	}
360 }
361 
362 static int copy_pte(pmd_t *dst_pmdp, pmd_t *src_pmdp, unsigned long start,
363 		    unsigned long end)
364 {
365 	pte_t *src_ptep;
366 	pte_t *dst_ptep;
367 	unsigned long addr = start;
368 
369 	dst_ptep = (pte_t *)get_safe_page(GFP_ATOMIC);
370 	if (!dst_ptep)
371 		return -ENOMEM;
372 	pmd_populate_kernel(&init_mm, dst_pmdp, dst_ptep);
373 	dst_ptep = pte_offset_kernel(dst_pmdp, start);
374 
375 	src_ptep = pte_offset_kernel(src_pmdp, start);
376 	do {
377 		_copy_pte(dst_ptep, src_ptep, addr);
378 	} while (dst_ptep++, src_ptep++, addr += PAGE_SIZE, addr != end);
379 
380 	return 0;
381 }
382 
383 static int copy_pmd(pud_t *dst_pudp, pud_t *src_pudp, unsigned long start,
384 		    unsigned long end)
385 {
386 	pmd_t *src_pmdp;
387 	pmd_t *dst_pmdp;
388 	unsigned long next;
389 	unsigned long addr = start;
390 
391 	if (pud_none(READ_ONCE(*dst_pudp))) {
392 		dst_pmdp = (pmd_t *)get_safe_page(GFP_ATOMIC);
393 		if (!dst_pmdp)
394 			return -ENOMEM;
395 		pud_populate(&init_mm, dst_pudp, dst_pmdp);
396 	}
397 	dst_pmdp = pmd_offset(dst_pudp, start);
398 
399 	src_pmdp = pmd_offset(src_pudp, start);
400 	do {
401 		pmd_t pmd = READ_ONCE(*src_pmdp);
402 
403 		next = pmd_addr_end(addr, end);
404 		if (pmd_none(pmd))
405 			continue;
406 		if (pmd_table(pmd)) {
407 			if (copy_pte(dst_pmdp, src_pmdp, addr, next))
408 				return -ENOMEM;
409 		} else {
410 			set_pmd(dst_pmdp,
411 				__pmd(pmd_val(pmd) & ~PMD_SECT_RDONLY));
412 		}
413 	} while (dst_pmdp++, src_pmdp++, addr = next, addr != end);
414 
415 	return 0;
416 }
417 
418 static int copy_pud(pgd_t *dst_pgdp, pgd_t *src_pgdp, unsigned long start,
419 		    unsigned long end)
420 {
421 	pud_t *dst_pudp;
422 	pud_t *src_pudp;
423 	unsigned long next;
424 	unsigned long addr = start;
425 
426 	if (pgd_none(READ_ONCE(*dst_pgdp))) {
427 		dst_pudp = (pud_t *)get_safe_page(GFP_ATOMIC);
428 		if (!dst_pudp)
429 			return -ENOMEM;
430 		pgd_populate(&init_mm, dst_pgdp, dst_pudp);
431 	}
432 	dst_pudp = pud_offset(dst_pgdp, start);
433 
434 	src_pudp = pud_offset(src_pgdp, start);
435 	do {
436 		pud_t pud = READ_ONCE(*src_pudp);
437 
438 		next = pud_addr_end(addr, end);
439 		if (pud_none(pud))
440 			continue;
441 		if (pud_table(pud)) {
442 			if (copy_pmd(dst_pudp, src_pudp, addr, next))
443 				return -ENOMEM;
444 		} else {
445 			set_pud(dst_pudp,
446 				__pud(pud_val(pud) & ~PMD_SECT_RDONLY));
447 		}
448 	} while (dst_pudp++, src_pudp++, addr = next, addr != end);
449 
450 	return 0;
451 }
452 
453 static int copy_page_tables(pgd_t *dst_pgdp, unsigned long start,
454 			    unsigned long end)
455 {
456 	unsigned long next;
457 	unsigned long addr = start;
458 	pgd_t *src_pgdp = pgd_offset_k(start);
459 
460 	dst_pgdp = pgd_offset_raw(dst_pgdp, start);
461 	do {
462 		next = pgd_addr_end(addr, end);
463 		if (pgd_none(READ_ONCE(*src_pgdp)))
464 			continue;
465 		if (copy_pud(dst_pgdp, src_pgdp, addr, next))
466 			return -ENOMEM;
467 	} while (dst_pgdp++, src_pgdp++, addr = next, addr != end);
468 
469 	return 0;
470 }
471 
472 /*
473  * Setup then Resume from the hibernate image using swsusp_arch_suspend_exit().
474  *
475  * Memory allocated by get_safe_page() will be dealt with by the hibernate code,
476  * we don't need to free it here.
477  */
478 int swsusp_arch_resume(void)
479 {
480 	int rc = 0;
481 	void *zero_page;
482 	size_t exit_size;
483 	pgd_t *tmp_pg_dir;
484 	phys_addr_t phys_hibernate_exit;
485 	void __noreturn (*hibernate_exit)(phys_addr_t, phys_addr_t, void *,
486 					  void *, phys_addr_t, phys_addr_t);
487 
488 	/*
489 	 * Restoring the memory image will overwrite the ttbr1 page tables.
490 	 * Create a second copy of just the linear map, and use this when
491 	 * restoring.
492 	 */
493 	tmp_pg_dir = (pgd_t *)get_safe_page(GFP_ATOMIC);
494 	if (!tmp_pg_dir) {
495 		pr_err("Failed to allocate memory for temporary page tables.\n");
496 		rc = -ENOMEM;
497 		goto out;
498 	}
499 	rc = copy_page_tables(tmp_pg_dir, PAGE_OFFSET, 0);
500 	if (rc)
501 		goto out;
502 
503 	/*
504 	 * We need a zero page that is zero before & after resume in order to
505 	 * to break before make on the ttbr1 page tables.
506 	 */
507 	zero_page = (void *)get_safe_page(GFP_ATOMIC);
508 	if (!zero_page) {
509 		pr_err("Failed to allocate zero page.\n");
510 		rc = -ENOMEM;
511 		goto out;
512 	}
513 
514 	/*
515 	 * Locate the exit code in the bottom-but-one page, so that *NULL
516 	 * still has disastrous affects.
517 	 */
518 	hibernate_exit = (void *)PAGE_SIZE;
519 	exit_size = __hibernate_exit_text_end - __hibernate_exit_text_start;
520 	/*
521 	 * Copy swsusp_arch_suspend_exit() to a safe page. This will generate
522 	 * a new set of ttbr0 page tables and load them.
523 	 */
524 	rc = create_safe_exec_page(__hibernate_exit_text_start, exit_size,
525 				   (unsigned long)hibernate_exit,
526 				   &phys_hibernate_exit,
527 				   (void *)get_safe_page, GFP_ATOMIC);
528 	if (rc) {
529 		pr_err("Failed to create safe executable page for hibernate_exit code.\n");
530 		goto out;
531 	}
532 
533 	/*
534 	 * The hibernate exit text contains a set of el2 vectors, that will
535 	 * be executed at el2 with the mmu off in order to reload hyp-stub.
536 	 */
537 	__flush_dcache_area(hibernate_exit, exit_size);
538 
539 	/*
540 	 * KASLR will cause the el2 vectors to be in a different location in
541 	 * the resumed kernel. Load hibernate's temporary copy into el2.
542 	 *
543 	 * We can skip this step if we booted at EL1, or are running with VHE.
544 	 */
545 	if (el2_reset_needed()) {
546 		phys_addr_t el2_vectors = phys_hibernate_exit;  /* base */
547 		el2_vectors += hibernate_el2_vectors -
548 			       __hibernate_exit_text_start;     /* offset */
549 
550 		__hyp_set_vectors(el2_vectors);
551 	}
552 
553 	hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1,
554 		       resume_hdr.reenter_kernel, restore_pblist,
555 		       resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page));
556 
557 out:
558 	return rc;
559 }
560 
561 int hibernate_resume_nonboot_cpu_disable(void)
562 {
563 	if (sleep_cpu < 0) {
564 		pr_err("Failing to resume from hibernate on an unknown CPU.\n");
565 		return -ENODEV;
566 	}
567 
568 	return freeze_secondary_cpus(sleep_cpu);
569 }
570