ring/cpu/arm.rs
1// Copyright 2016-2024 Brian Smith.
2//
3// Permission to use, copy, modify, and/or distribute this software for any
4// purpose with or without fee is hereby granted, provided that the above
5// copyright notice and this permission notice appear in all copies.
6//
7// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
8// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
9// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
10// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
11// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
12// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
13// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
14
15use super::CAPS_STATIC;
16
17mod abi_assumptions {
18 use core::mem::size_of;
19
20 // TODO: Support ARM64_32; see
21 // https://github.com/briansmith/ring/issues/1832#issuecomment-1892928147. This also requires
22 // replacing all `cfg(target_pointer_width)` logic for non-pointer/reference things
23 // (`N0`, `Limb`, `LimbMask`, `crypto_word_t` etc.).
24 #[cfg(target_arch = "aarch64")]
25 const _ASSUMED_POINTER_SIZE: usize = 8;
26 #[cfg(target_arch = "arm")]
27 const _ASSUMED_POINTER_SIZE: usize = 4;
28 const _ASSUMED_USIZE_SIZE: () = assert!(size_of::<usize>() == _ASSUMED_POINTER_SIZE);
29 const _ASSUMED_REF_SIZE: () = assert!(size_of::<&'static u8>() == _ASSUMED_POINTER_SIZE);
30
31 // To support big-endian, we'd need to make several changes as described in
32 // https://github.com/briansmith/ring/issues/1832.
33 const _ASSUMED_ENDIANNESS: () = assert!(cfg!(target_endian = "little"));
34}
35
36// uclibc: When linked statically, uclibc doesn't provide getauxval.
37// When linked dynamically, recent versions do provide it, but we
38// want to support older versions too. Assume that if uclibc is being
39// used, this is an embedded target where the user cares a lot about
40// minimizing code size and also that they know in advance exactly
41// what target features are supported, so rely only on static feature
42// detection.
43
44cfg_if::cfg_if! {
45 if #[cfg(all(all(target_arch = "aarch64", target_endian = "little"),
46 any(target_os = "ios", target_os = "macos", target_os = "tvos", target_os = "visionos", target_os = "watchos")))] {
47 mod darwin;
48 use darwin as detect;
49 } else if #[cfg(all(all(target_arch = "aarch64", target_endian = "little"), target_os = "fuchsia"))] {
50 mod fuchsia;
51 use fuchsia as detect;
52 } else if #[cfg(any(target_os = "android", target_os = "linux"))] {
53 mod linux;
54 use linux as detect;
55 } else if #[cfg(all(all(target_arch = "aarch64", target_endian = "little"), target_os = "windows"))] {
56 mod windows;
57 use windows as detect;
58 } else {
59 mod detect {
60 pub const FORCE_DYNAMIC_DETECTION: u32 = 0;
61 pub fn detect_features() -> u32 { 0 }
62 }
63 }
64}
65
66impl_get_feature! {
67 features: [
68 // TODO(MSRV): 32-bit ARM doesn't have `target_feature = "neon"` yet.
69 { ("aarch64", "arm") => Neon },
70
71 // TODO(MSRV): There is no "pmull" feature listed from
72 // `rustc --print cfg --target=aarch64-apple-darwin`. Originally ARMv8 tied
73 // PMULL detection into AES detection, but later versions split it; see
74 // https://developer.arm.com/downloads/-/exploration-tools/feature-names-for-a-profile
75 // "Features introduced prior to 2020." Change this to use "pmull" when
76 // that is supported.
77 { ("aarch64") => PMull },
78
79 { ("aarch64") => Aes },
80
81 { ("aarch64") => Sha256 },
82
83 // Keep in sync with `ARMV8_SHA512`.
84
85 // "sha3" is overloaded for both SHA-3 and SHA-512.
86 { ("aarch64") => Sha512 },
87 ],
88}
89
90pub(super) mod featureflags {
91 pub(in super::super) use super::detect::FORCE_DYNAMIC_DETECTION;
92 use super::*;
93 use crate::{
94 cpu,
95 polyfill::{once_cell::race, usize_from_u32},
96 };
97 use core::num::NonZeroUsize;
98 #[cfg(all(target_arch = "arm", target_endian = "little"))]
99 use core::sync::atomic::{AtomicU32, Ordering};
100
101 pub(in super::super) fn get_or_init() -> cpu::Features {
102 fn init() -> NonZeroUsize {
103 let detected = detect::detect_features();
104 let filtered = (if cfg!(feature = "unstable-testing-arm-no-hw") {
105 !Neon::mask()
106 } else {
107 0
108 }) | (if cfg!(feature = "unstable-testing-arm-no-neon") {
109 Neon::mask()
110 } else {
111 0
112 });
113 let detected = detected & !filtered;
114 let merged = CAPS_STATIC | detected;
115
116 #[cfg(all(
117 target_arch = "arm",
118 target_endian = "little",
119 target_has_atomic = "32"
120 ))]
121 if (merged & Neon::mask()) == Neon::mask() {
122 // `neon_available` is declared as `alignas(4) uint32_t` in the C code.
123 // AtomicU32 is `#[repr(C, align(4))]`.
124 prefixed_extern! {
125 static neon_available: AtomicU32;
126 }
127 // SAFETY: The C code only reads `neon_available`, and its
128 // reads are synchronized through the `OnceNonZeroUsize`
129 // Acquire/Release semantics as we ensure we have a
130 // `cpu::Features` instance before calling into the C code.
131 let p = unsafe { &neon_available };
132 p.store(1, Ordering::Relaxed);
133 }
134
135 let merged = usize_from_u32(merged) | (1 << (Shift::Initialized as u32));
136 NonZeroUsize::new(merged).unwrap() // Can't fail because we just set a bit.
137 }
138
139 // SAFETY: This is the only caller. Any concurrent reading doesn't
140 // affect the safety of the writing.
141 let _: NonZeroUsize = FEATURES.get_or_init(init);
142
143 // SAFETY: We initialized the CPU features as required.
144 unsafe { cpu::Features::new_after_feature_flags_written_and_synced_unchecked() }
145 }
146
147 pub(in super::super) fn get(_cpu_features: cpu::Features) -> u32 {
148 // SAFETY: Since only `get_or_init()` could have created
149 // `_cpu_features`, and it only does so after `FEATURES.get_or_init()`,
150 // we know we are reading from `FEATURES` after initializing it.
151 //
152 // Also, 0 means "no features detected" to users, which is designed to
153 // be a safe configuration.
154 let features = FEATURES.get().map(NonZeroUsize::get).unwrap_or(0);
155
156 // The truncation is lossless, as we set the value with a u32.
157 #[allow(clippy::cast_possible_truncation)]
158 let features = features as u32;
159
160 features
161 }
162
163 static FEATURES: race::OnceNonZeroUsize = race::OnceNonZeroUsize::new();
164
165 // TODO(MSRV): There is no "pmull" feature listed from
166 // `rustc --print cfg --target=aarch64-apple-darwin`. Originally ARMv8 tied
167 // PMULL detection into AES detection, but later versions split it; see
168 // https://developer.arm.com/downloads/-/exploration-tools/feature-names-for-a-profile
169 // "Features introduced prior to 2020." Change this to use "pmull" when
170 // that is supported.
171 //
172 // "sha3" is overloaded for both SHA-3 and SHA-512.
173 #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
174 #[rustfmt::skip]
175 pub(in super::super) const STATIC_DETECTED: u32 = 0
176 | (if cfg!(target_feature = "neon") { Neon::mask() } else { 0 })
177 | (if cfg!(target_feature = "aes") { Aes::mask() } else { 0 })
178 | (if cfg!(target_feature = "aes") { PMull::mask() } else { 0 })
179 | (if cfg!(target_feature = "sha2") { Sha256::mask() } else { 0 })
180 | (if cfg!(target_feature = "sha3") { Sha512::mask() } else { 0 })
181 ;
182
183 // TODO(MSRV): 32-bit ARM doesn't support any static feature detection yet.
184 #[cfg(all(target_arch = "arm", target_endian = "little"))]
185 pub(in super::super) const STATIC_DETECTED: u32 = 0;
186}
187
188#[allow(clippy::assertions_on_constants)]
189const _AARCH64_HAS_NEON: () = assert!(
190 ((CAPS_STATIC & Neon::mask()) == Neon::mask())
191 || !cfg!(all(target_arch = "aarch64", target_endian = "little"))
192);