From 0e827b4f741e57538c24d90b63659dbdb0992cb0 Mon Sep 17 00:00:00 2001
From: Jorrit Jongma <jorrit@jongma.org>
Date: Tue, 12 Apr 2022 17:09:34 +0000
Subject: [PATCH] [Backport] [PA] Support 16kb pagesize on Linux+ARM64
This makes the system pagesize a run-time property.
ARM64 supports 4kb, 16kb, and 64kb page sizes. Previously, only 4kb
was supported by Chromium. This patch adds 16kb support, as is used
for example by Asahi Linux on M1 Macs. The rare 64kb case is still
not supported due to further changes needed to SlotSpanMetadata.
The implementation follows the changes made to support run-time page
size on macOS. On macOS, the required constants are conveniently
injected before any code runs, while on Linux a function call is
needed, complicating initialization.
The new PageCharacteristics structure holds the page size and shift
as std::atomic<int> which are initialized on first use.
Bug: 1301788
Change-Id: I8ceead40de53ba7a2ec248bd6ef46f2a521dd29c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3545665
Reviewed-by: Benoit Lize <lizeb@chromium.org>
Reviewed-by: Mark Mentovai <mark@chromium.org>
Commit-Queue: Mark Mentovai <mark@chromium.org>
Cr-Commit-Position: refs/heads/main@{#991588}
(adapted from commit a704c3a94179e6f3abb28f090d096ced72657d7c)
Bug-Fedora: 2144200
Task-number: QTBUG-108674
---
.../address_space_randomization.h | 15 ++++++
.../page_allocator_constants.h | 51 +++++++++++++++++-
.../partition_address_space.cc | 6 +++
.../partition_allocator/partition_alloc.cc | 2 +-
.../partition_alloc_constants.h | 5 +-
.../address_space_randomization.h | 15 ++++++
.../partition_allocator/page_allocator.cc | 8 +++
.../page_allocator_constants.h | 52 ++++++++++++++++++-
.../partition_allocator/partition_alloc.cc | 2 +-
.../partition_alloc_constants.h | 5 +-
10 files changed, 153 insertions(+), 8 deletions(-)
diff --git a/src/3rdparty/chromium/base/allocator/partition_allocator/address_space_randomization.h b/src/3rdparty/chromium/base/allocator/partition_allocator/address_space_randomization.h
index e77003eab25..31ac05b86f5 100644
--- a/src/3rdparty/chromium/base/allocator/partition_allocator/address_space_randomization.h
+++ b/src/3rdparty/chromium/base/allocator/partition_allocator/address_space_randomization.h
@@ -119,6 +119,21 @@ AslrMask(uintptr_t bits) {
return AslrAddress(0x20000000ULL);
}
+ #elif defined(OS_LINUX)
+
+ // Linux on arm64 can use 39, 42, 48, or 52-bit user space, depending on
+ // page size and number of levels of translation pages used. We use
+ // 39-bit as base as all setups should support this, lowered to 38-bit
+ // as ASLROffset() could cause a carry.
+ PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE uintptr_t
+ ASLRMask() {
+ return AslrMask(38);
+ }
+ PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE uintptr_t
+ ASLROffset() {
+ return AslrAddress(0x1000000000ULL);
+ }
+
#else
// ARM64 on Linux has 39-bit user space. Use 38 bits since ASLROffset()
diff --git a/src/3rdparty/chromium/base/allocator/partition_allocator/page_allocator_constants.h b/src/3rdparty/chromium/base/allocator/partition_allocator/page_allocator_constants.h
index c42fe2835ff..dc7486608b9 100644
--- a/src/3rdparty/chromium/base/allocator/partition_allocator/page_allocator_constants.h
+++ b/src/3rdparty/chromium/base/allocator/partition_allocator/page_allocator_constants.h
@@ -24,6 +24,31 @@
// elimination.
#define PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR __attribute__((const))
+#elif defined(OS_LINUX) && defined(ARCH_CPU_ARM64)
+// This should work for all POSIX (if needed), but currently all other
+// supported OS/architecture combinations use either hard-coded values
+// (such as x86) or have means to determine these values without needing
+// atomics (such as macOS on arm64).
+
+// Page allocator constants are run-time constant
+#define PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR __attribute__((const))
+
+#include <unistd.h>
+#include <atomic>
+
+namespace base::internal {
+
+// Holds the current page size and shift, where size = 1 << shift
+// Use PageAllocationGranularity(), PageAllocationGranularityShift()
+// to initialize and retrieve these values safely.
+struct PageCharacteristics {
+ std::atomic<int> size;
+ std::atomic<int> shift;
+};
+extern PageCharacteristics page_characteristics;
+
+} // namespace base::internal
+
#else
// When defined, page size constants are fixed at compile time. When not
@@ -36,11 +61,17 @@
#endif
+namespace base {
+// Forward declaration, implementation below
+PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE size_t
+PageAllocationGranularity();
+}
+
namespace {
#if !defined(OS_APPLE)
-constexpr ALWAYS_INLINE int PageAllocationGranularityShift() {
+PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE int PageAllocationGranularityShift() {
#if defined(OS_WIN) || defined(ARCH_CPU_PPC64)
// Modern ppc64 systems support 4kB (shift = 12) and 64kB (shift = 16) page
// sizes. Since 64kB is the de facto standard on the platform and binaries
@@ -49,6 +80,15 @@ constexpr ALWAYS_INLINE int PageAllocationGranularityShift() {
return 16; // 64kB
#elif defined(_MIPS_ARCH_LOONGSON)
return 14; // 16kB
+#elif defined(OS_LINUX) && defined(ARCH_CPU_ARM64)
+ // arm64 supports 4kb (shift = 12), 16kb (shift = 14), and 64kb (shift = 16)
+ // page sizes. Retrieve from or initialize cache.
+ int shift = base::internal::page_characteristics.shift.load(std::memory_order_relaxed);
+ if (UNLIKELY(shift == 0)) {
+ shift = __builtin_ctz((int)base::PageAllocationGranularity());
+ base::internal::page_characteristics.shift.store(shift, std::memory_order_relaxed);
+ }
+ return shift;
#else
return 12; // 4kB
#endif
@@ -64,6 +104,15 @@ PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE size_t
PageAllocationGranularity() {
#if defined(OS_APPLE)
return vm_page_size;
+#elif defined(OS_LINUX) && defined(ARCH_CPU_ARM64)
+ // arm64 supports 4kb, 16kb, and 64kb page sizes. Retrieve from or
+ // initialize cache.
+ int size = internal::page_characteristics.size.load(std::memory_order_relaxed);
+ if (UNLIKELY(size == 0)) {
+ size = getpagesize();
+ internal::page_characteristics.size.store(size, std::memory_order_relaxed);
+ }
+ return size;
#else
return 1ULL << PageAllocationGranularityShift();
#endif
diff --git a/src/3rdparty/chromium/base/allocator/partition_allocator/partition_address_space.cc b/src/3rdparty/chromium/base/allocator/partition_allocator/partition_address_space.cc
index 03883bcb113..90efc51c838 100644
--- a/src/3rdparty/chromium/base/allocator/partition_allocator/partition_address_space.cc
+++ b/src/3rdparty/chromium/base/allocator/partition_allocator/partition_address_space.cc
@@ -75,6 +75,12 @@ void PartitionAddressSpace::UninitForTesting() {
internal::AddressPoolManager::GetInstance()->ResetForTesting();
}
+#if defined(OS_LINUX) && defined(ARCH_CPU_ARM64)
+
+PageCharacteristics page_characteristics;
+
+#endif // defined(OS_LINUX) && defined(ARCH_CPU_ARM64)
+
#endif // defined(PA_HAS_64_BITS_POINTERS)
} // namespace internal
diff --git a/src/3rdparty/chromium/base/allocator/partition_allocator/partition_alloc.cc b/src/3rdparty/chromium/base/allocator/partition_allocator/partition_alloc.cc
index daeb6d5cb17..7c434b5e697 100644
--- a/src/3rdparty/chromium/base/allocator/partition_allocator/partition_alloc.cc
+++ b/src/3rdparty/chromium/base/allocator/partition_allocator/partition_alloc.cc
@@ -522,7 +522,7 @@ static size_t PartitionPurgePage(internal::PartitionPage<thread_safe>* page,
#if defined(PAGE_ALLOCATOR_CONSTANTS_ARE_CONSTEXPR)
constexpr size_t kMaxSlotCount =
(PartitionPageSize() * kMaxPartitionPagesPerSlotSpan) / SystemPageSize();
-#elif defined(OS_APPLE)
+#elif defined(OS_APPLE) || (defined(OS_LINUX) && defined(ARCH_CPU_ARM64))
// It's better for slot_usage to be stack-allocated and fixed-size, which
// demands that its size be constexpr. On OS_APPLE, PartitionPageSize() is
// always SystemPageSize() << 2, so regardless of what the run time page size
diff --git a/src/3rdparty/chromium/base/allocator/partition_allocator/partition_alloc_constants.h b/src/3rdparty/chromium/base/allocator/partition_allocator/partition_alloc_constants.h
index c8268ec30a0..f03ba1e4ab4 100644
--- a/src/3rdparty/chromium/base/allocator/partition_allocator/partition_alloc_constants.h
+++ b/src/3rdparty/chromium/base/allocator/partition_allocator/partition_alloc_constants.h
@@ -57,10 +57,11 @@ PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE int
PartitionPageShift() {
return 18; // 256 KiB
}
-#elif defined(OS_APPLE)
+#elif defined(OS_APPLE) || \
+ (defined(OS_LINUX) && defined(ARCH_CPU_ARM64))
PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE int
PartitionPageShift() {
- return vm_page_shift + 2;
+ return PageAllocationGranularityShift() + 2;
}
#else
PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE int
diff --git a/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/address_space_randomization.h b/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/address_space_randomization.h
index 28c8271fd68..3957e0cdf76 100644
--- a/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/address_space_randomization.h
+++ b/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/address_space_randomization.h
@@ -120,6 +120,21 @@ AslrMask(uintptr_t bits) {
return AslrAddress(0x20000000ULL);
}
+ #elif defined(OS_LINUX)
+
+ // Linux on arm64 can use 39, 42, 48, or 52-bit user space, depending on
+ // page size and number of levels of translation pages used. We use
+ // 39-bit as base as all setups should support this, lowered to 38-bit
+ // as ASLROffset() could cause a carry.
+ PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE uintptr_t
+ ASLRMask() {
+ return AslrMask(38);
+ }
+ PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE uintptr_t
+ ASLROffset() {
+ return AslrAddress(0x1000000000ULL);
+ }
+
#else
// ARM64 on Linux has 39-bit user space. Use 38 bits since kASLROffset
diff --git a/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/page_allocator.cc b/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/page_allocator.cc
index 91d00d2fbca..597d5f84cb1 100644
--- a/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/page_allocator.cc
+++ b/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/page_allocator.cc
@@ -255,5 +255,13 @@ uint32_t GetAllocPageErrorCode() {
return s_allocPageErrorCode;
}
+#if defined(OS_LINUX) && defined(ARCH_CPU_ARM64)
+
+namespace internal {
+PageCharacteristics page_characteristics;
+}
+
+#endif // defined(OS_LINUX) && defined(ARCH_CPU_ARM64)
+
} // namespace base
} // namespace pdfium
diff --git a/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/page_allocator_constants.h b/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/page_allocator_constants.h
index fdc65ac47b7..f826308839d 100644
--- a/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/page_allocator_constants.h
+++ b/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/page_allocator_constants.h
@@ -24,6 +24,31 @@
// elimination.
#define PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR __attribute__((const))
+#elif defined(OS_LINUX) && defined(ARCH_CPU_ARM64)
+// This should work for all POSIX (if needed), but currently all other
+// supported OS/architecture combinations use either hard-coded values
+// (such as x86) or have means to determine these values without needing
+// atomics (such as macOS on arm64).
+
+// Page allocator constants are run-time constant
+#define PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR __attribute__((const))
+
+#include <unistd.h>
+#include <atomic>
+
+namespace pdfium::base::internal {
+
+// Holds the current page size and shift, where size = 1 << shift
+// Use PageAllocationGranularity(), PageAllocationGranularityShift()
+// to initialize and retrieve these values safely.
+struct PageCharacteristics {
+ std::atomic<int> size;
+ std::atomic<int> shift;
+};
+extern PageCharacteristics page_characteristics;
+
+} // namespace base::internal
+
#else
// When defined, page size constants are fixed at compile time. When not
@@ -37,11 +62,18 @@
#endif
namespace pdfium {
+
+namespace base {
+// Forward declaration, implementation below
+PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE size_t
+PageAllocationGranularity();
+}
+
namespace {
#if !defined(OS_APPLE)
-constexpr ALWAYS_INLINE int PageAllocationGranularityShift() {
+PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE int PageAllocationGranularityShift() {
#if defined(OS_WIN) || defined(ARCH_CPU_PPC64)
// Modern ppc64 systems support 4kB (shift = 12) and 64kB (shift = 16) page
// sizes. Since 64kB is the de facto standard on the platform and binaries
@@ -50,6 +82,15 @@ constexpr ALWAYS_INLINE int PageAllocationGranularityShift() {
return 16; // 64kB
#elif defined(_MIPS_ARCH_LOONGSON)
return 14; // 16kB
+#elif defined(OS_LINUX) && defined(ARCH_CPU_ARM64)
+ // arm64 supports 4kb (shift = 12), 16kb (shift = 14), and 64kb (shift = 16)
+ // page sizes. Retrieve from or initialize cache.
+ int shift = base::internal::page_characteristics.shift.load(std::memory_order_relaxed);
+ if (UNLIKELY(shift == 0)) {
+ shift = __builtin_ctz((int)base::PageAllocationGranularity());
+ base::internal::page_characteristics.shift.store(shift, std::memory_order_relaxed);
+ }
+ return shift;
#else
return 12; // 4kB
#endif
@@ -65,6 +106,15 @@ PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE size_t
PageAllocationGranularity() {
#if defined(OS_APPLE)
return vm_page_size;
+#elif defined(OS_LINUX) && defined(ARCH_CPU_ARM64)
+ // arm64 supports 4kb, 16kb, and 64kb page sizes. Retrieve from or
+ // initialize cache.
+ int size = internal::page_characteristics.size.load(std::memory_order_relaxed);
+ if (UNLIKELY(size == 0)) {
+ size = getpagesize();
+ internal::page_characteristics.size.store(size, std::memory_order_relaxed);
+ }
+ return size;
#else
return 1ULL << PageAllocationGranularityShift();
#endif
diff --git a/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/partition_alloc.cc b/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/partition_alloc.cc
index 2e5e87fa7e6..89b9f6217a6 100644
--- a/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/partition_alloc.cc
+++ b/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/partition_alloc.cc
@@ -486,7 +486,7 @@ static size_t PartitionPurgePage(internal::PartitionPage* page, bool discard) {
#if defined(PAGE_ALLOCATOR_CONSTANTS_ARE_CONSTEXPR)
constexpr size_t kMaxSlotCount =
(PartitionPageSize() * kMaxPartitionPagesPerSlotSpan) / SystemPageSize();
-#elif defined(OS_APPLE)
+#elif defined(OS_APPLE) || (defined(OS_LINUX) && defined(ARCH_CPU_ARM64))
// It's better for slot_usage to be stack-allocated and fixed-size, which
// demands that its size be constexpr. On OS_APPLE, PartitionPageSize() is
// always SystemPageSize() << 2, so regardless of what the run time page size
diff --git a/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/partition_alloc_constants.h b/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/partition_alloc_constants.h
index 71d63ba4146..a6d83626741 100644
--- a/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/partition_alloc_constants.h
+++ b/src/3rdparty/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/partition_alloc_constants.h
@@ -50,10 +50,11 @@ PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE int
PartitionPageShift() {
return 18; // 256 KiB
}
-#elif defined(OS_APPLE)
+#elif defined(OS_APPLE) || \
+ (defined(OS_LINUX) && defined(ARCH_CPU_ARM64))
PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE int
PartitionPageShift() {
- return vm_page_shift + 2;
+ return PageAllocationGranularityShift() + 2;
}
#else
PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE int
--
2.38.1