11//! Run-time feature detection for RISC-V on Linux.
2+ //!
3+ //! On RISC-V, detection using auxv only supports single-letter extensions.
4+ //! So, we use riscv_hwprobe that supports multi-letter extensions if available.
5+ //! <https://www.kernel.org/doc/html/latest/arch/riscv/hwprobe.html>
6+
7+ use core:: ptr;
28
39use super :: super :: riscv:: imply_features;
410use super :: auxvec;
511use crate :: detect:: { Feature , bit, cache} ;
612
7- /// Read list of supported features from the auxiliary vector.
13+ // See <https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/uapi/linux/prctl.h?h=v6.14>
14+ // for runtime status query constants.
15+ const PR_RISCV_V_GET_CONTROL : libc:: c_int = 70 ;
16+ const PR_RISCV_V_VSTATE_CTRL_ON : libc:: c_int = 2 ;
17+ const PR_RISCV_V_VSTATE_CTRL_CUR_MASK : libc:: c_int = 3 ;
18+
19+ // See <https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/riscv/include/uapi/asm/hwprobe.h?h=v6.14>
20+ // for riscv_hwprobe struct and hardware probing constants.
21+
22+ #[ repr( C ) ]
23+ struct riscv_hwprobe {
24+ key : i64 ,
25+ value : u64 ,
26+ }
27+
28+ #[ allow( non_upper_case_globals) ]
29+ const __NR_riscv_hwprobe: libc:: c_long = 258 ;
30+
31+ const RISCV_HWPROBE_KEY_BASE_BEHAVIOR : i64 = 3 ;
32+ const RISCV_HWPROBE_BASE_BEHAVIOR_IMA : u64 = 1 << 0 ;
33+
34+ const RISCV_HWPROBE_KEY_IMA_EXT_0 : i64 = 4 ;
35+ const RISCV_HWPROBE_IMA_FD : u64 = 1 << 0 ;
36+ const RISCV_HWPROBE_IMA_C : u64 = 1 << 1 ;
37+ const RISCV_HWPROBE_IMA_V : u64 = 1 << 2 ;
38+ const RISCV_HWPROBE_EXT_ZBA : u64 = 1 << 3 ;
39+ const RISCV_HWPROBE_EXT_ZBB : u64 = 1 << 4 ;
40+ const RISCV_HWPROBE_EXT_ZBS : u64 = 1 << 5 ;
41+ const RISCV_HWPROBE_EXT_ZICBOZ : u64 = 1 << 6 ;
42+ const RISCV_HWPROBE_EXT_ZBC : u64 = 1 << 7 ;
43+ const RISCV_HWPROBE_EXT_ZBKB : u64 = 1 << 8 ;
44+ const RISCV_HWPROBE_EXT_ZBKC : u64 = 1 << 9 ;
45+ const RISCV_HWPROBE_EXT_ZBKX : u64 = 1 << 10 ;
46+ const RISCV_HWPROBE_EXT_ZKND : u64 = 1 << 11 ;
47+ const RISCV_HWPROBE_EXT_ZKNE : u64 = 1 << 12 ;
48+ const RISCV_HWPROBE_EXT_ZKNH : u64 = 1 << 13 ;
49+ const RISCV_HWPROBE_EXT_ZKSED : u64 = 1 << 14 ;
50+ const RISCV_HWPROBE_EXT_ZKSH : u64 = 1 << 15 ;
51+ const RISCV_HWPROBE_EXT_ZKT : u64 = 1 << 16 ;
52+ const RISCV_HWPROBE_EXT_ZVBB : u64 = 1 << 17 ;
53+ const RISCV_HWPROBE_EXT_ZVBC : u64 = 1 << 18 ;
54+ const RISCV_HWPROBE_EXT_ZVKB : u64 = 1 << 19 ;
55+ const RISCV_HWPROBE_EXT_ZVKG : u64 = 1 << 20 ;
56+ const RISCV_HWPROBE_EXT_ZVKNED : u64 = 1 << 21 ;
57+ const RISCV_HWPROBE_EXT_ZVKNHA : u64 = 1 << 22 ;
58+ const RISCV_HWPROBE_EXT_ZVKNHB : u64 = 1 << 23 ;
59+ const RISCV_HWPROBE_EXT_ZVKSED : u64 = 1 << 24 ;
60+ const RISCV_HWPROBE_EXT_ZVKSH : u64 = 1 << 25 ;
61+ const RISCV_HWPROBE_EXT_ZVKT : u64 = 1 << 26 ;
62+ const RISCV_HWPROBE_EXT_ZFH : u64 = 1 << 27 ;
63+ const RISCV_HWPROBE_EXT_ZFHMIN : u64 = 1 << 28 ;
64+ const RISCV_HWPROBE_EXT_ZIHINTNTL : u64 = 1 << 29 ;
65+ const RISCV_HWPROBE_EXT_ZVFH : u64 = 1 << 30 ;
66+ const RISCV_HWPROBE_EXT_ZVFHMIN : u64 = 1 << 31 ;
67+ const RISCV_HWPROBE_EXT_ZFA : u64 = 1 << 32 ;
68+ const RISCV_HWPROBE_EXT_ZTSO : u64 = 1 << 33 ;
69+ const RISCV_HWPROBE_EXT_ZACAS : u64 = 1 << 34 ;
70+ const RISCV_HWPROBE_EXT_ZICOND : u64 = 1 << 35 ;
71+ const RISCV_HWPROBE_EXT_ZIHINTPAUSE : u64 = 1 << 36 ;
72+ const RISCV_HWPROBE_EXT_ZVE32X : u64 = 1 << 37 ;
73+ const RISCV_HWPROBE_EXT_ZVE32F : u64 = 1 << 38 ;
74+ const RISCV_HWPROBE_EXT_ZVE64X : u64 = 1 << 39 ;
75+ const RISCV_HWPROBE_EXT_ZVE64F : u64 = 1 << 40 ;
76+ const RISCV_HWPROBE_EXT_ZVE64D : u64 = 1 << 41 ;
77+ const RISCV_HWPROBE_EXT_ZIMOP : u64 = 1 << 42 ;
78+ const RISCV_HWPROBE_EXT_ZCA : u64 = 1 << 43 ;
79+ const RISCV_HWPROBE_EXT_ZCB : u64 = 1 << 44 ;
80+ const RISCV_HWPROBE_EXT_ZCD : u64 = 1 << 45 ;
81+ const RISCV_HWPROBE_EXT_ZCF : u64 = 1 << 46 ;
82+ const RISCV_HWPROBE_EXT_ZCMOP : u64 = 1 << 47 ;
83+ const RISCV_HWPROBE_EXT_ZAWRS : u64 = 1 << 48 ;
84+ // Excluded because it only reports the existence of `prctl`-based pointer masking control.
85+ // const RISCV_HWPROBE_EXT_SUPM: u64 = 1 << 49;
86+
87+ const RISCV_HWPROBE_KEY_CPUPERF_0 : i64 = 5 ;
88+ const RISCV_HWPROBE_MISALIGNED_FAST : u64 = 3 ;
89+ const RISCV_HWPROBE_MISALIGNED_MASK : u64 = 7 ;
90+
91+ const RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF : i64 = 9 ;
92+ const RISCV_HWPROBE_MISALIGNED_SCALAR_FAST : u64 = 3 ;
93+
94+ const RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF : i64 = 10 ;
95+ const RISCV_HWPROBE_MISALIGNED_VECTOR_FAST : u64 = 3 ;
96+
97+ // syscall returns an unsupported error if riscv_hwprobe is not supported,
98+ // so we can safely use this function on older versions of Linux.
99+ fn _riscv_hwprobe ( out : & mut [ riscv_hwprobe ] ) -> bool {
100+ unsafe fn __riscv_hwprobe (
101+ pairs : * mut riscv_hwprobe ,
102+ pair_count : libc:: size_t ,
103+ cpu_set_size : libc:: size_t ,
104+ cpus : * mut libc:: c_ulong ,
105+ flags : libc:: c_uint ,
106+ ) -> libc:: c_long {
107+ unsafe {
108+ libc:: syscall (
109+ __NR_riscv_hwprobe,
110+ pairs,
111+ pair_count,
112+ cpu_set_size,
113+ cpus,
114+ flags,
115+ )
116+ }
117+ }
118+
119+ let len = out. len ( ) ;
120+ unsafe { __riscv_hwprobe ( out. as_mut_ptr ( ) , len, 0 , ptr:: null_mut ( ) , 0 ) == 0 }
121+ }
122+
123+ /// Read list of supported features from (1) the auxiliary vector
124+ /// and (2) the results of `riscv_hwprobe` and `prctl` system calls.
8125pub ( crate ) fn detect_features ( ) -> cache:: Initializer {
9126 let mut value = cache:: Initializer :: default ( ) ;
10127 let mut enable_feature = |feature, enable| {
@@ -18,16 +135,175 @@ pub(crate) fn detect_features() -> cache::Initializer {
18135 //
19136 // [hwcap]: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/riscv/include/uapi/asm/hwcap.h?h=v6.14
20137 let auxv = auxvec:: auxv ( ) . expect ( "read auxvec" ) ; // should not fail on RISC-V platform
138+ let mut has_i = bit:: test ( auxv. hwcap , ( b'i' - b'a' ) . into ( ) ) ;
21139 #[ allow( clippy:: eq_op) ]
22140 enable_feature ( Feature :: a, bit:: test ( auxv. hwcap , ( b'a' - b'a' ) . into ( ) ) ) ;
23141 enable_feature ( Feature :: c, bit:: test ( auxv. hwcap , ( b'c' - b'a' ) . into ( ) ) ) ;
24142 enable_feature ( Feature :: d, bit:: test ( auxv. hwcap , ( b'd' - b'a' ) . into ( ) ) ) ;
25143 enable_feature ( Feature :: f, bit:: test ( auxv. hwcap , ( b'f' - b'a' ) . into ( ) ) ) ;
26144 enable_feature ( Feature :: h, bit:: test ( auxv. hwcap , ( b'h' - b'a' ) . into ( ) ) ) ;
27145 enable_feature ( Feature :: m, bit:: test ( auxv. hwcap , ( b'm' - b'a' ) . into ( ) ) ) ;
146+ let has_v = bit:: test ( auxv. hwcap , ( b'v' - b'a' ) . into ( ) ) ;
147+ let mut is_v_set = false ;
148+
149+ // Use riscv_hwprobe syscall to query more extensions and
150+ // performance-related capabilities.
151+ ' hwprobe: {
152+ let mut out = [
153+ riscv_hwprobe {
154+ key : RISCV_HWPROBE_KEY_BASE_BEHAVIOR ,
155+ value : 0 ,
156+ } ,
157+ riscv_hwprobe {
158+ key : RISCV_HWPROBE_KEY_IMA_EXT_0 ,
159+ value : 0 ,
160+ } ,
161+ riscv_hwprobe {
162+ key : RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF ,
163+ value : 0 ,
164+ } ,
165+ riscv_hwprobe {
166+ key : RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF ,
167+ value : 0 ,
168+ } ,
169+ riscv_hwprobe {
170+ key : RISCV_HWPROBE_KEY_CPUPERF_0 ,
171+ value : 0 ,
172+ } ,
173+ ] ;
174+ if !_riscv_hwprobe ( & mut out) {
175+ break ' hwprobe;
176+ }
177+
178+ // Query scalar / vector misaligned behavior.
179+ if out[ 2 ] . key != -1 {
180+ enable_feature (
181+ Feature :: unaligned_scalar_mem,
182+ out[ 2 ] . value == RISCV_HWPROBE_MISALIGNED_SCALAR_FAST ,
183+ ) ;
184+ } else if out[ 4 ] . key != -1 {
185+ // Deprecated method for fallback
186+ enable_feature (
187+ Feature :: unaligned_scalar_mem,
188+ out[ 4 ] . value & RISCV_HWPROBE_MISALIGNED_MASK == RISCV_HWPROBE_MISALIGNED_FAST ,
189+ ) ;
190+ }
191+ if out[ 3 ] . key != -1 {
192+ enable_feature (
193+ Feature :: unaligned_vector_mem,
194+ out[ 3 ] . value == RISCV_HWPROBE_MISALIGNED_VECTOR_FAST ,
195+ ) ;
196+ }
197+
198+ // Query whether "I" base and extensions "M" and "A" (as in the ISA
199+ // manual version 2.2) are enabled. "I" base at that time corresponds
200+ // to "I", "Zicsr", "Zicntr" and "Zifencei" (as in the ISA manual version
201+ // 20240411) and we chose to imply "Zicsr" and "Zifencei" (not "Zicntr")
202+ // because there will be a separate RISCV_HWPROBE_EXT_ZICNTR constant to
203+ // determine existence of the "Zicntr" extension in Linux 6.15 (as of rc1).
204+ // "fence.i" ("Zifencei") is conditionally valid on the Linux userland
205+ // (valid when CMODX is enabled).
206+ // This is a requirement of `RISCV_HWPROBE_KEY_IMA_EXT_0`-based tests.
207+ let has_ima = ( out[ 0 ] . key != -1 ) && ( out[ 0 ] . value & RISCV_HWPROBE_BASE_BEHAVIOR_IMA != 0 ) ;
208+ if !has_ima {
209+ break ' hwprobe;
210+ }
211+ has_i |= has_ima;
212+ enable_feature ( Feature :: zicsr, has_ima) ;
213+ enable_feature ( Feature :: zifencei, has_ima) ;
214+ enable_feature ( Feature :: m, has_ima) ;
215+ enable_feature ( Feature :: a, has_ima) ;
216+
217+ // Enable features based on `RISCV_HWPROBE_KEY_IMA_EXT_0`.
218+ if out[ 1 ] . key == -1 {
219+ break ' hwprobe;
220+ }
221+ let ima_ext_0 = out[ 1 ] . value ;
222+ let test = |mask| ( ima_ext_0 & mask) != 0 ;
223+
224+ enable_feature ( Feature :: d, test ( RISCV_HWPROBE_IMA_FD ) ) ; // F is implied.
225+ enable_feature ( Feature :: c, test ( RISCV_HWPROBE_IMA_C ) ) ;
226+
227+ enable_feature ( Feature :: zihintntl, test ( RISCV_HWPROBE_EXT_ZIHINTNTL ) ) ;
228+ enable_feature ( Feature :: zihintpause, test ( RISCV_HWPROBE_EXT_ZIHINTPAUSE ) ) ;
229+ enable_feature ( Feature :: zimop, test ( RISCV_HWPROBE_EXT_ZIMOP ) ) ;
230+ enable_feature ( Feature :: zicboz, test ( RISCV_HWPROBE_EXT_ZICBOZ ) ) ;
231+ enable_feature ( Feature :: zicond, test ( RISCV_HWPROBE_EXT_ZICOND ) ) ;
232+
233+ enable_feature ( Feature :: zawrs, test ( RISCV_HWPROBE_EXT_ZAWRS ) ) ;
234+ enable_feature ( Feature :: zacas, test ( RISCV_HWPROBE_EXT_ZACAS ) ) ;
235+ enable_feature ( Feature :: ztso, test ( RISCV_HWPROBE_EXT_ZTSO ) ) ;
236+
237+ enable_feature ( Feature :: zba, test ( RISCV_HWPROBE_EXT_ZBA ) ) ;
238+ enable_feature ( Feature :: zbb, test ( RISCV_HWPROBE_EXT_ZBB ) ) ;
239+ enable_feature ( Feature :: zbs, test ( RISCV_HWPROBE_EXT_ZBS ) ) ;
240+ enable_feature ( Feature :: zbc, test ( RISCV_HWPROBE_EXT_ZBC ) ) ;
241+
242+ enable_feature ( Feature :: zbkb, test ( RISCV_HWPROBE_EXT_ZBKB ) ) ;
243+ enable_feature ( Feature :: zbkc, test ( RISCV_HWPROBE_EXT_ZBKC ) ) ;
244+ enable_feature ( Feature :: zbkx, test ( RISCV_HWPROBE_EXT_ZBKX ) ) ;
245+ enable_feature ( Feature :: zknd, test ( RISCV_HWPROBE_EXT_ZKND ) ) ;
246+ enable_feature ( Feature :: zkne, test ( RISCV_HWPROBE_EXT_ZKNE ) ) ;
247+ enable_feature ( Feature :: zknh, test ( RISCV_HWPROBE_EXT_ZKNH ) ) ;
248+ enable_feature ( Feature :: zksed, test ( RISCV_HWPROBE_EXT_ZKSED ) ) ;
249+ enable_feature ( Feature :: zksh, test ( RISCV_HWPROBE_EXT_ZKSH ) ) ;
250+ enable_feature ( Feature :: zkt, test ( RISCV_HWPROBE_EXT_ZKT ) ) ;
251+
252+ enable_feature ( Feature :: zcmop, test ( RISCV_HWPROBE_EXT_ZCMOP ) ) ;
253+ enable_feature ( Feature :: zca, test ( RISCV_HWPROBE_EXT_ZCA ) ) ;
254+ enable_feature ( Feature :: zcf, test ( RISCV_HWPROBE_EXT_ZCF ) ) ;
255+ enable_feature ( Feature :: zcd, test ( RISCV_HWPROBE_EXT_ZCD ) ) ;
256+ enable_feature ( Feature :: zcb, test ( RISCV_HWPROBE_EXT_ZCB ) ) ;
257+
258+ enable_feature ( Feature :: zfh, test ( RISCV_HWPROBE_EXT_ZFH ) ) ;
259+ enable_feature ( Feature :: zfhmin, test ( RISCV_HWPROBE_EXT_ZFHMIN ) ) ;
260+ enable_feature ( Feature :: zfa, test ( RISCV_HWPROBE_EXT_ZFA ) ) ;
261+
262+ // Use prctl (if any) to determine whether the vector extension
263+ // is enabled on the current thread (assuming the entire process
264+ // share the same status). If prctl fails (e.g. QEMU userland emulator
265+ // as of version 9.2.3), use auxiliary vector to retrieve the default
266+ // vector status on the process startup.
267+ let has_vectors = {
268+ let v_status = unsafe { libc:: prctl ( PR_RISCV_V_GET_CONTROL ) } ;
269+ if v_status >= 0 {
270+ ( v_status & PR_RISCV_V_VSTATE_CTRL_CUR_MASK ) == PR_RISCV_V_VSTATE_CTRL_ON
271+ } else {
272+ has_v
273+ }
274+ } ;
275+ if has_vectors {
276+ enable_feature ( Feature :: v, test ( RISCV_HWPROBE_IMA_V ) ) ;
277+ enable_feature ( Feature :: zve32x, test ( RISCV_HWPROBE_EXT_ZVE32X ) ) ;
278+ enable_feature ( Feature :: zve32f, test ( RISCV_HWPROBE_EXT_ZVE32F ) ) ;
279+ enable_feature ( Feature :: zve64x, test ( RISCV_HWPROBE_EXT_ZVE64X ) ) ;
280+ enable_feature ( Feature :: zve64f, test ( RISCV_HWPROBE_EXT_ZVE64F ) ) ;
281+ enable_feature ( Feature :: zve64d, test ( RISCV_HWPROBE_EXT_ZVE64D ) ) ;
282+
283+ enable_feature ( Feature :: zvbb, test ( RISCV_HWPROBE_EXT_ZVBB ) ) ;
284+ enable_feature ( Feature :: zvbc, test ( RISCV_HWPROBE_EXT_ZVBC ) ) ;
285+ enable_feature ( Feature :: zvkb, test ( RISCV_HWPROBE_EXT_ZVKB ) ) ;
286+ enable_feature ( Feature :: zvkg, test ( RISCV_HWPROBE_EXT_ZVKG ) ) ;
287+ enable_feature ( Feature :: zvkned, test ( RISCV_HWPROBE_EXT_ZVKNED ) ) ;
288+ enable_feature ( Feature :: zvknha, test ( RISCV_HWPROBE_EXT_ZVKNHA ) ) ;
289+ enable_feature ( Feature :: zvknhb, test ( RISCV_HWPROBE_EXT_ZVKNHB ) ) ;
290+ enable_feature ( Feature :: zvksed, test ( RISCV_HWPROBE_EXT_ZVKSED ) ) ;
291+ enable_feature ( Feature :: zvksh, test ( RISCV_HWPROBE_EXT_ZVKSH ) ) ;
292+ enable_feature ( Feature :: zvkt, test ( RISCV_HWPROBE_EXT_ZVKT ) ) ;
293+
294+ enable_feature ( Feature :: zvfh, test ( RISCV_HWPROBE_EXT_ZVFH ) ) ;
295+ enable_feature ( Feature :: zvfhmin, test ( RISCV_HWPROBE_EXT_ZVFHMIN ) ) ;
296+ }
297+ is_v_set = true ;
298+ } ;
299+
300+ // Set V purely depending on the auxiliary vector
301+ // only if no fine-grained vector extension detection is available.
302+ if !is_v_set {
303+ enable_feature ( Feature :: v, has_v) ;
304+ }
28305
29306 // Handle base ISA.
30- let has_i = bit:: test ( auxv. hwcap , ( b'i' - b'a' ) . into ( ) ) ;
31307 // If future RV128I is supported, implement with `enable_feature` here.
32308 // Note that we should use `target_arch` instead of `target_pointer_width`
33309 // to avoid misdetection caused by experimental ABIs such as RV64ILP32.
0 commit comments