From 59ae90bf2098e51219f1a8762411708c3631ccfc Mon Sep 17 00:00:00 2001 From: usamoi Date: Thu, 21 Aug 2025 17:40:08 +0800 Subject: [PATCH] fix compilation on aarch64_be Signed-off-by: usamoi --- Cargo.lock | 131 +++++++++++++++++----------------- Cargo.toml | 2 +- crates/k_means/Cargo.toml | 2 +- crates/make/Cargo.toml | 5 +- crates/make/src/main.rs | 64 +++++++++++------ crates/simd/Cargo.toml | 2 +- crates/simd/cshim/aarch64.c | 18 +++++ crates/simd/src/f16.rs | 18 +++-- crates/simd/src/f32.rs | 54 +++++++------- crates/simd/src/fast_scan.rs | 7 +- crates/simd/src/quantize.rs | 6 ++ crates/simd_macros/src/lib.rs | 34 +++++---- src/lib.rs | 3 - 13 files changed, 194 insertions(+), 152 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 858b22cd..f0b41cd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -48,9 +48,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.19" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" +checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" dependencies = [ "anstyle", "anstyle-parse", @@ -78,29 +78,29 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" +checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.9" +version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" +checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] name = "anyhow" -version = "1.0.98" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" [[package]] name = "bindgen" @@ -123,9 +123,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +checksum = "6a65b545ab31d687cff52899d4890855fec459eb6afe0da6417b8a18da87aa29" [[package]] name = "bitvec" @@ -157,9 +157,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.31" +version = "1.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a42d84bb6b69d3a8b3eaacf0d88f179e1929695e1ad012b6cf64d9caaa5fd2" +checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f" dependencies = [ "shlex", ] @@ -185,9 +185,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" [[package]] name = "clang-sys" @@ -202,9 +202,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.42" +version = "4.5.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed87a9d530bb41a67537289bafcac159cb3ee28460e0a4571123d2a778a6a882" +checksum = "1fc0e74a703892159f5ae7d3aac52c8e6c392f5ae5f359c70b5881d60aaac318" dependencies = [ "clap_builder", "clap_derive", @@ -212,9 +212,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.42" +version = "4.5.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64f4f3f3c77c94aff3c7e9aac9a2ca1974a5adf392a8bb751e827d6d127ab966" +checksum = "b3e7f4214277f3c7aa526a59dd3fbe306a370daee1f8b7b8c987069cd8e888a8" dependencies = [ "anstream", "anstyle", @@ -224,9 +224,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.41" +version = "4.5.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" +checksum = "14cb31bb0a7d536caef2639baa7fad459e15c3144efefa6dbd1c84562c4739f6" dependencies = [ "heck", "proc-macro2", @@ -441,9 +441,9 @@ checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" [[package]] name = "form_urlencoded" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ "percent-encoding", ] @@ -468,9 +468,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "half" @@ -491,9 +491,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.4" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", @@ -615,9 +615,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "1.0.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ "idna_adapter", "smallvec", @@ -636,9 +636,9 @@ dependencies = [ [[package]] name = "indenter" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" +checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" [[package]] name = "indexmap" @@ -710,9 +710,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.174" +version = "0.2.175" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" [[package]] name = "libloading" @@ -758,6 +758,7 @@ version = "0.0.0" dependencies = [ "clap", "object", + "shlex", "target-triple", ] @@ -800,9 +801,9 @@ dependencies = [ [[package]] name = "object" -version = "0.37.2" +version = "0.37.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e3d0a7419f081f4a808147e845310313a39f322d7ae1f996b7f001d6cbed04" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" dependencies = [ "memchr", "wasmparser", @@ -848,9 +849,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "petgraph" @@ -1041,9 +1042,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.95" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" dependencies = [ "unicode-ident", ] @@ -1110,9 +1111,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" dependencies = [ "either", "rayon-core", @@ -1120,9 +1121,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -1178,9 +1179,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" @@ -1241,9 +1242,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.142" +version = "1.0.143" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" dependencies = [ "itoa", "memchr", @@ -1346,9 +1347,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.104" +version = "2.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" dependencies = [ "proc-macro2", "quote", @@ -1380,18 +1381,18 @@ checksum = "1ac9aa371f599d22256307c24a9d748c041e548cbf599f35d890f9d365361790" [[package]] name = "thiserror" -version = "2.0.12" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.12" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" dependencies = [ "proc-macro2", "quote", @@ -1514,9 +1515,9 @@ checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" [[package]] name = "url" -version = "2.5.4" +version = "2.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +checksum = "ec961601b32b6f5d14ae8dabd35ff2ff2e2c6cb4c0e6641845ff105abe96d958" dependencies = [ "form_urlencoded", "idna", @@ -1537,9 +1538,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" +checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" dependencies = [ "getrandom", "js-sys", @@ -1728,9 +1729,9 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.236.0" +version = "0.236.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d1eee846a705f6f3cb9d7b9f79b54583810f1fb57a1e3aea76d1742db2e3d2" +checksum = "a9b1e81f3eb254cf7404a82cee6926a4a3ccc5aad80cc3d43608a070c67aa1d7" dependencies = [ "bitflags", ] @@ -1764,11 +1765,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +checksum = "0978bf7171b3d90bac376700cb56d606feb40f251a475a5d6634613564460b22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -2047,9 +2048,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.2" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" dependencies = [ "yoke", "zerofrom", diff --git a/Cargo.toml b/Cargo.toml index 7902d33a..15e8d202 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,7 @@ mimalloc = { version = "0.1.47", features = ["local_dynamic_tls"] } workspace = true [workspace] -resolver = "2" +resolver = "3" members = ["crates/*"] [workspace.package] diff --git a/crates/k_means/Cargo.toml b/crates/k_means/Cargo.toml index fd3b43a7..6475cd6e 100644 --- a/crates/k_means/Cargo.toml +++ b/crates/k_means/Cargo.toml @@ -9,7 +9,7 @@ rabitq = { path = "../rabitq" } simd = { path = "../simd" } rand.workspace = true -rayon = "1.10.0" +rayon = "1.11.0" [lints] workspace = true diff --git a/crates/make/Cargo.toml b/crates/make/Cargo.toml index 0d7d6414..d4565466 100644 --- a/crates/make/Cargo.toml +++ b/crates/make/Cargo.toml @@ -5,9 +5,10 @@ edition.workspace = true publish = false [dependencies] -clap = { version = "4.5.42", features = ["derive"] } -object = { version = "0.37.2", default-features = false, features = [ +clap = { version = "4.5.45", features = ["derive"] } +object = { version = "0.37.3", default-features = false, features = [ "read", "wasm", ] } +shlex = "1.3.0" target-triple = "0.1.4" diff --git a/crates/make/src/main.rs b/crates/make/src/main.rs index b9c426cf..8e942920 100644 --- a/crates/make/src/main.rs +++ b/crates/make/src/main.rs @@ -40,6 +40,8 @@ struct BuildArgs { profile: String, #[arg(long, default_value = target_triple::TARGET)] target: String, + #[arg(long)] + runner: Option, } struct TargetSpecificInformation { @@ -225,6 +227,7 @@ fn parse( } fn generate( + runner: &Option>, pg_config: impl AsRef, pg_version: &str, tsi: &TargetSpecificInformation, @@ -263,11 +266,14 @@ fn generate( }); result.push(format!("pgrx_embed_vchord{}", tsi.exe_suffix()?)); let mut command; - if !(tsi.is_unix && tsi.is_emscripten) { - command = Command::new(result); - } else { - command = Command::new("node"); + if let Some(runner) = runner { + command = Command::new(&runner[0]); + for arg in runner[1..].iter() { + command.arg(arg); + } command.arg(result); + } else { + command = Command::new(result); } command.stderr(Stdio::inherit()); eprintln!("Running {command:?}"); @@ -315,31 +321,35 @@ fn main() -> Result<(), Box> { if !std::fs::exists("./vchord.control")? { return Err("The script must be run from the VectorChord source directory.".into()); } - let path = if let Some(value) = var_os("PGRX_PG_CONFIG_PATH") { - PathBuf::from(value) - } else { - return Err("Environment variable `PGRX_PG_CONFIG_PATH` is not set.".into()); - }; - let pg_config = pg_config(&path)?; - let pg_version = { - let version = pg_config["VERSION"].clone(); - if let Some(prefix_stripped) = version.strip_prefix("PostgreSQL ") { - if let Some((stripped, _)) = prefix_stripped.split_once(|c: char| !c.is_ascii_digit()) { - format!("pg{stripped}",) - } else { - format!("pg{prefix_stripped}",) - } - } else { - return Err("PostgreSQL version is invalid.".into()); - } - }; let vchord_version = control_file("./vchord.control")?["default_version"].clone(); match cli.command { Commands::Build(BuildArgs { output, profile, target, + runner, }) => { + let runner = runner.and_then(|runner| shlex::split(&runner)); + let path = if let Some(value) = var_os("PGRX_PG_CONFIG_PATH") { + PathBuf::from(value) + } else { + return Err("Environment variable `PGRX_PG_CONFIG_PATH` is not set.".into()); + }; + let pg_config = pg_config(&path)?; + let pg_version = { + let version = pg_config["VERSION"].clone(); + if let Some(prefix_stripped) = version.strip_prefix("PostgreSQL ") { + if let Some((stripped, _)) = + prefix_stripped.split_once(|c: char| !c.is_ascii_digit()) + { + format!("pg{stripped}",) + } else { + format!("pg{prefix_stripped}",) + } + } else { + return Err("PostgreSQL version is invalid.".into()); + } + }; let tsi = target_specific_information(&target)?; let obj = build(&path, &pg_version, &tsi, &profile, &target)?; let pkglibdir = format!("{output}/pkglibdir"); @@ -378,7 +388,15 @@ fn main() -> Result<(), Box> { } else { let exports = parse(&tsi, obj)?; install_by_writing( - generate(&path, &pg_version, &tsi, &profile, &target, exports)?, + generate( + &runner, + &path, + &pg_version, + &tsi, + &profile, + &target, + exports, + )?, format!("{sharedir_extension}/vchord--0.0.0.sql"), false, )?; diff --git a/crates/simd/Cargo.toml b/crates/simd/Cargo.toml index 69ed8684..98a9bd5c 100644 --- a/crates/simd/Cargo.toml +++ b/crates/simd/Cargo.toml @@ -18,7 +18,7 @@ zerocopy.workspace = true rand.workspace = true [build-dependencies] -cc = "1.2.31" +cc = "1.2.33" which = "8.0.0" [lints] diff --git a/crates/simd/cshim/aarch64.c b/crates/simd/cshim/aarch64.c index 8e2d9336..5af96c3b 100644 --- a/crates/simd/cshim/aarch64.c +++ b/crates/simd/cshim/aarch64.c @@ -25,7 +25,9 @@ #endif #include +#if defined(__AARCH64EL__) #include +#endif #include #include @@ -81,6 +83,7 @@ fp16_reduce_sum_of_xy_a2_fp16(f16 *restrict a, f16 *restrict b, size_t n) { return vaddvq_f32(lo) + vaddvq_f32(hi); } +#if defined(__AARCH64EL__) __attribute__((target("+sve"))) float fp16_reduce_sum_of_xy_a3_512(f16 *restrict a, f16 *restrict b, size_t n) { svfloat16_t xy = svdup_f16(0.0); @@ -92,6 +95,7 @@ fp16_reduce_sum_of_xy_a3_512(f16 *restrict a, f16 *restrict b, size_t n) { } return svaddv_f16(svptrue_b16(), xy); } +#endif __attribute__((target("+fp16"))) float fp16_reduce_sum_of_d2_a2_fp16(f16 *restrict a, f16 *restrict b, size_t n) { @@ -150,6 +154,7 @@ fp16_reduce_sum_of_d2_a2_fp16(f16 *restrict a, f16 *restrict b, size_t n) { return vaddvq_f32(lo) + vaddvq_f32(hi); } +#if defined(__AARCH64EL__) __attribute__((target("+sve"))) float fp16_reduce_sum_of_d2_a3_512(f16 *restrict a, f16 *restrict b, size_t n) { svfloat16_t d2 = svdup_f16(0.0); @@ -162,7 +167,9 @@ fp16_reduce_sum_of_d2_a3_512(f16 *restrict a, f16 *restrict b, size_t n) { } return svaddv_f16(svptrue_b16(), d2); } +#endif +#if defined(__AARCH64EL__) __attribute__((target("+sve"))) float fp32_reduce_sum_of_x_a3_256(float *restrict this, size_t n) { svfloat32_t sum = svdup_f32(0.0); @@ -173,7 +180,9 @@ fp32_reduce_sum_of_x_a3_256(float *restrict this, size_t n) { } return svaddv_f32(svptrue_b32(), sum); } +#endif +#if defined(__AARCH64EL__) __attribute__((target("+sve"))) float fp32_reduce_sum_of_abs_x_a3_256(float *restrict this, size_t n) { svfloat32_t sum = svdup_f32(0.0); @@ -184,7 +193,9 @@ fp32_reduce_sum_of_abs_x_a3_256(float *restrict this, size_t n) { } return svaddv_f32(svptrue_b32(), sum); } +#endif +#if defined(__AARCH64EL__) __attribute__((target("+sve"))) float fp32_reduce_sum_of_x2_a3_256(float *restrict this, size_t n) { svfloat32_t sum = svdup_f32(0.0); @@ -195,7 +206,9 @@ fp32_reduce_sum_of_x2_a3_256(float *restrict this, size_t n) { } return svaddv_f32(svptrue_b32(), sum); } +#endif +#if defined(__AARCH64EL__) __attribute__((target("+sve"))) void fp32_reduce_min_max_of_x_a3_256(float *restrict this, size_t n, float *out_min, float *out_max) { @@ -210,7 +223,9 @@ fp32_reduce_min_max_of_x_a3_256(float *restrict this, size_t n, float *out_min, *out_min = svminv_f32(svptrue_b32(), min); *out_max = svmaxv_f32(svptrue_b32(), max); } +#endif +#if defined(__AARCH64EL__) __attribute__((target("+sve"))) float fp32_reduce_sum_of_xy_a3_256(float *restrict lhs, float *restrict rhs, size_t n) { @@ -223,7 +238,9 @@ fp32_reduce_sum_of_xy_a3_256(float *restrict lhs, float *restrict rhs, } return svaddv_f32(svptrue_b32(), sum); } +#endif +#if defined(__AARCH64EL__) __attribute__((target("+sve"))) float fp32_reduce_sum_of_d2_a3_256(float *restrict lhs, float *restrict rhs, size_t n) { @@ -237,3 +254,4 @@ fp32_reduce_sum_of_d2_a3_256(float *restrict lhs, float *restrict rhs, } return svaddv_f32(svptrue_b32(), sum); } +#endif diff --git a/crates/simd/src/f16.rs b/crates/simd/src/f16.rs index e6828907..95686e0c 100644 --- a/crates/simd/src/f16.rs +++ b/crates/simd/src/f16.rs @@ -458,9 +458,8 @@ mod reduce_sum_of_xy { } #[inline] - #[cfg(target_arch = "aarch64")] - #[crate::target_cpu(enable = "a2")] - #[target_feature(enable = "sve")] + #[cfg(all(target_arch = "aarch64", target_endian = "little"))] + #[crate::target_cpu(enable = "a3.512")] pub fn reduce_sum_of_xy_a3_512(lhs: &[f16], rhs: &[f16]) -> f32 { assert!(lhs.len() == rhs.len()); unsafe { @@ -472,7 +471,7 @@ mod reduce_sum_of_xy { } } - #[cfg(all(target_arch = "aarch64", test, not(miri)))] + #[cfg(all(target_arch = "aarch64", target_endian = "little", test, not(miri)))] #[test] fn reduce_sum_of_xy_a3_512_test() { use rand::Rng; @@ -503,7 +502,7 @@ mod reduce_sum_of_xy { } } - #[crate::multiversion(@"v4:avx512fp16", @"v4", @"v3", @"a3.512", @"a2:fp16")] + #[crate::multiversion(@"v4:avx512fp16", @"v4", @"v3", #[cfg(target_endian = "little")] @"a3.512", @"a2:fp16")] pub fn reduce_sum_of_xy(lhs: &[f16], rhs: &[f16]) -> f32 { assert!(lhs.len() == rhs.len()); let n = lhs.len(); @@ -742,9 +741,8 @@ mod reduce_sum_of_d2 { } #[inline] - #[cfg(target_arch = "aarch64")] - #[crate::target_cpu(enable = "a2")] - #[target_feature(enable = "sve")] + #[cfg(all(target_arch = "aarch64", target_endian = "little"))] + #[crate::target_cpu(enable = "a3.512")] pub fn reduce_sum_of_d2_a3_512(lhs: &[f16], rhs: &[f16]) -> f32 { assert!(lhs.len() == rhs.len()); unsafe { @@ -756,7 +754,7 @@ mod reduce_sum_of_d2 { } } - #[cfg(all(target_arch = "aarch64", test, not(miri)))] + #[cfg(all(target_arch = "aarch64", target_endian = "little", test, not(miri)))] #[test] fn reduce_sum_of_d2_a3_512_test() { use rand::Rng; @@ -787,7 +785,7 @@ mod reduce_sum_of_d2 { } } - #[crate::multiversion(@"v4:avx512fp16", @"v4", @"v3", @"a3.512", @"a2:fp16")] + #[crate::multiversion(@"v4:avx512fp16", @"v4", @"v3", #[cfg(target_endian = "little")] @"a3.512", @"a2:fp16")] pub fn reduce_sum_of_d2(lhs: &[f16], rhs: &[f16]) -> f32 { assert!(lhs.len() == rhs.len()); let n = lhs.len(); diff --git a/crates/simd/src/f32.rs b/crates/simd/src/f32.rs index 159bb949..7e534922 100644 --- a/crates/simd/src/f32.rs +++ b/crates/simd/src/f32.rs @@ -373,9 +373,8 @@ mod reduce_sum_of_x { } #[inline] - #[cfg(target_arch = "aarch64")] - #[crate::target_cpu(enable = "a2")] - #[target_feature(enable = "sve")] + #[cfg(all(target_arch = "aarch64", target_endian = "little"))] + #[crate::target_cpu(enable = "a3.256")] fn reduce_sum_of_x_a3_256(this: &[f32]) -> f32 { unsafe { unsafe extern "C" { @@ -385,7 +384,7 @@ mod reduce_sum_of_x { } } - #[cfg(all(target_arch = "aarch64", test, not(miri)))] + #[cfg(all(target_arch = "aarch64", target_endian = "little", test, not(miri)))] #[test] fn reduce_sum_of_x_a3_256_test() { use rand::Rng; @@ -412,7 +411,7 @@ mod reduce_sum_of_x { } } - #[crate::multiversion(@"v4", @"v3", @"v2", @"a3.256", @"a2")] + #[crate::multiversion(@"v4", @"v3", @"v2", #[cfg(target_endian = "little")] @"a3.256", @"a2")] pub fn reduce_sum_of_x(this: &[f32]) -> f32 { let n = this.len(); let mut sum = 0.0f32; @@ -649,9 +648,8 @@ mod reduce_sum_of_abs_x { } #[inline] - #[cfg(target_arch = "aarch64")] - #[crate::target_cpu(enable = "a2")] - #[target_feature(enable = "sve")] + #[cfg(all(target_arch = "aarch64", target_endian = "little"))] + #[crate::target_cpu(enable = "a3.256")] fn reduce_sum_of_abs_x_a3_256(this: &[f32]) -> f32 { unsafe { unsafe extern "C" { @@ -661,7 +659,7 @@ mod reduce_sum_of_abs_x { } } - #[cfg(all(target_arch = "aarch64", test, not(miri)))] + #[cfg(all(target_arch = "aarch64", target_endian = "little", test, not(miri)))] #[test] fn reduce_sum_of_abs_x_a3_256_test() { use rand::Rng; @@ -688,7 +686,7 @@ mod reduce_sum_of_abs_x { } } - #[crate::multiversion(@"v4", @"v3", @"v2", @"a3.256", @"a2")] + #[crate::multiversion(@"v4", @"v3", @"v2", #[cfg(target_endian = "little")] @"a3.256", @"a2")] pub fn reduce_sum_of_abs_x(this: &[f32]) -> f32 { let n = this.len(); let mut sum = 0.0f32; @@ -915,9 +913,8 @@ mod reduce_sum_of_x2 { } #[inline] - #[cfg(target_arch = "aarch64")] - #[crate::target_cpu(enable = "a2")] - #[target_feature(enable = "sve")] + #[cfg(all(target_arch = "aarch64", target_endian = "little"))] + #[crate::target_cpu(enable = "a3.256")] fn reduce_sum_of_x2_a3_256(this: &[f32]) -> f32 { unsafe { unsafe extern "C" { @@ -927,7 +924,7 @@ mod reduce_sum_of_x2 { } } - #[cfg(all(target_arch = "aarch64", test, not(miri)))] + #[cfg(all(target_arch = "aarch64", target_endian = "little", test, not(miri)))] #[test] fn reduce_sum_of_x2_a3_256_test() { use rand::Rng; @@ -954,7 +951,7 @@ mod reduce_sum_of_x2 { } } - #[crate::multiversion(@"v4", @"v3", @"v2:fma", @"a3.256", @"a2")] + #[crate::multiversion(@"v4", @"v3", @"v2:fma", #[cfg(target_endian = "little")] @"a3.256", @"a2")] pub fn reduce_sum_of_x2(this: &[f32]) -> f32 { let n = this.len(); let mut x2 = 0.0f32; @@ -1182,9 +1179,8 @@ mod reduce_min_max_of_x { } #[inline] - #[cfg(target_arch = "aarch64")] - #[crate::target_cpu(enable = "a2")] - #[target_feature(enable = "sve")] + #[cfg(all(target_arch = "aarch64", target_endian = "little"))] + #[crate::target_cpu(enable = "a3.256")] fn reduce_min_max_of_x_a3_256(this: &[f32]) -> (f32, f32) { let mut min = f32::INFINITY; let mut max = -f32::INFINITY; @@ -1202,7 +1198,7 @@ mod reduce_min_max_of_x { (min, max) } - #[cfg(all(target_arch = "aarch64", test, not(miri)))] + #[cfg(all(target_arch = "aarch64", target_endian = "little", test, not(miri)))] #[test] fn reduce_min_max_of_x_a3_256_test() { use rand::Rng; @@ -1226,7 +1222,7 @@ mod reduce_min_max_of_x { } } - #[crate::multiversion(@"v4", @"v3", @"v2", @"a3.256", @"a2")] + #[crate::multiversion(@"v4", @"v3", @"v2", #[cfg(target_endian = "little")] @"a3.256", @"a2")] pub fn reduce_min_max_of_x(this: &[f32]) -> (f32, f32) { let mut min = f32::INFINITY; let mut max = f32::NEG_INFINITY; @@ -1496,9 +1492,8 @@ mod reduce_sum_of_xy { } #[inline] - #[cfg(target_arch = "aarch64")] - #[crate::target_cpu(enable = "a2")] - #[target_feature(enable = "sve")] + #[cfg(all(target_arch = "aarch64", target_endian = "little"))] + #[crate::target_cpu(enable = "a3.256")] fn reduce_sum_of_xy_a3_256(lhs: &[f32], rhs: &[f32]) -> f32 { assert!(lhs.len() == rhs.len()); unsafe { @@ -1513,7 +1508,7 @@ mod reduce_sum_of_xy { } } - #[cfg(all(target_arch = "aarch64", test, not(miri)))] + #[cfg(all(target_arch = "aarch64", target_endian = "little", test, not(miri)))] #[test] fn reduce_sum_of_xy_a3_256_test() { use rand::Rng; @@ -1544,7 +1539,7 @@ mod reduce_sum_of_xy { } } - #[crate::multiversion(@"v4", @"v3", @"v2:fma", @"a3.256", @"a2")] + #[crate::multiversion(@"v4", @"v3", @"v2:fma", #[cfg(target_endian = "little")] @"a3.256", @"a2")] pub fn reduce_sum_of_xy(lhs: &[f32], rhs: &[f32]) -> f32 { assert!(lhs.len() == rhs.len()); let n = lhs.len(); @@ -1822,9 +1817,8 @@ mod reduce_sum_of_d2 { } #[inline] - #[cfg(target_arch = "aarch64")] - #[crate::target_cpu(enable = "a2")] - #[target_feature(enable = "sve")] + #[cfg(all(target_arch = "aarch64", target_endian = "little"))] + #[crate::target_cpu(enable = "a3.256")] fn reduce_sum_of_d2_a3_256(lhs: &[f32], rhs: &[f32]) -> f32 { assert!(lhs.len() == rhs.len()); unsafe { @@ -1839,7 +1833,7 @@ mod reduce_sum_of_d2 { } } - #[cfg(all(target_arch = "aarch64", test, not(miri)))] + #[cfg(all(target_arch = "aarch64", target_endian = "little", test, not(miri)))] #[test] fn reduce_sum_of_d2_a3_256_test() { use rand::Rng; @@ -1870,7 +1864,7 @@ mod reduce_sum_of_d2 { } } - #[crate::multiversion(@"v4", @"v3", @"v2:fma", @"a3.256", @"a2")] + #[crate::multiversion(@"v4", @"v3", @"v2:fma", #[cfg(target_endian = "little")] @"a3.256", @"a2")] pub fn reduce_sum_of_d2(lhs: &[f32], rhs: &[f32]) -> f32 { assert!(lhs.len() == rhs.len()); let n = lhs.len(); diff --git a/crates/simd/src/fast_scan.rs b/crates/simd/src/fast_scan.rs index 43673381..c766cda5 100644 --- a/crates/simd/src/fast_scan.rs +++ b/crates/simd/src/fast_scan.rs @@ -435,6 +435,9 @@ mod scan { fn shuffle(a: [u8; 16], b: [u8; 16]) -> [u8; 16] { std::array::from_fn(|i| a[b[i] as usize]) } + fn transmute(x: [u8; 16]) -> [u16; 8] { + std::array::from_fn(|i| u16::from_le_bytes([x[i << 1 | 0], x[i << 1 | 1]])) + } assert_eq!(code.len(), lut.len()); let n = code.len(); @@ -451,10 +454,10 @@ mod scan { let chi = code.map(|x| x >> 4); let lut = lut[i]; - let res_lo = zerocopy::transmute!(shuffle(lut, clo)); + let res_lo = transmute(shuffle(lut, clo)); a_0 = binary(u16::wrapping_add, a_0, res_lo); a_1 = binary(u16::wrapping_add, a_1, res_lo.map(|x| x >> 8)); - let res_hi = zerocopy::transmute!(shuffle(lut, chi)); + let res_hi = transmute(shuffle(lut, chi)); a_2 = binary(u16::wrapping_add, a_2, res_hi); a_3 = binary(u16::wrapping_add, a_3, res_hi.map(|x| x >> 8)); } diff --git a/crates/simd/src/quantize.rs b/crates/simd/src/quantize.rs index cb363df0..874f66f7 100644 --- a/crates/simd/src/quantize.rs +++ b/crates/simd/src/quantize.rs @@ -216,10 +216,16 @@ mod mul_add_round { fn mul_add_round_a2(this: &[f32], k: f32, b: f32) -> Vec { let mut r = Vec::::with_capacity(this.len()); use std::arch::aarch64::*; + #[cfg(target_endian = "little")] const CONS: [u8; 16] = [ 0, 4, 8, 12, 0xff, 0xff, 0xff, 0xff, // 0..8 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, // 8..15 ]; + #[cfg(target_endian = "big")] + const CONS: [u8; 16] = [ + 12, 8, 4, 0, 0xff, 0xff, 0xff, 0xff, // 0..8 + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, // 8..15 + ]; let cons = unsafe { vld1q_u8(CONS.as_ptr()) }; let lk = vdupq_n_f32(k); let lb = vdupq_n_f32(b); diff --git a/crates/simd_macros/src/lib.rs b/crates/simd_macros/src/lib.rs index a085260c..75f7b12d 100644 --- a/crates/simd_macros/src/lib.rs +++ b/crates/simd_macros/src/lib.rs @@ -15,27 +15,30 @@ mod target; struct MultiversionVersion { + attrs: Vec, target: String, import: bool, } impl syn::parse::Parse for MultiversionVersion { fn parse(input: syn::parse::ParseStream) -> syn::Result { - let lookahead1 = input.lookahead1(); - if lookahead1.peek(syn::Token![@]) { + let attrs = if input.lookahead1().peek(syn::Token![#]) { + syn::Attribute::parse_outer(input)? + } else { + Vec::new() + }; + let import = if input.lookahead1().peek(syn::Token![@]) { let _: syn::Token![@] = input.parse()?; - let target: syn::LitStr = input.parse()?; - Ok(Self { - target: target.value(), - import: true, - }) + true } else { - let target: syn::LitStr = input.parse()?; - Ok(Self { - target: target.value(), - import: false, - }) - } + false + }; + let target = input.parse::()?.value(); + Ok(Self { + attrs, + target, + import, + }) } } @@ -101,6 +104,7 @@ pub fn multiversion( let mut versions = quote::quote! {}; let mut cold = quote::quote! {}; for version in attr.versions { + let attrs = version.attrs.clone(); let target = version.target.clone(); let name = syn::Ident::new( &format!("{name}_{}", target.replace(":", "_").replace(".", "_")), @@ -116,14 +120,16 @@ pub fn multiversion( let target_cpu = target_cpu.target_cpu; if !version.import { versions.extend(quote::quote! { + #(#attrs)* #[inline] - #[cfg(any(target_arch = #target_arch))] + #[cfg(target_arch = #target_arch)] #[crate::target_cpu(enable = #target_cpu)] #(#[target_feature(enable = #additional_target_features)])* fn #name < #generics_params > (#inputs) #output #generics_where { #block } }); } cold.extend(quote::quote! { + #(#attrs)* #[cfg(target_arch = #target_arch)] if crate::is_cpu_detected!(#target_cpu) #(&& crate::is_feature_detected!(#additional_target_features))* { let ptr = unsafe { std::mem::transmute::(#name) }; diff --git a/src/lib.rs b/src/lib.rs index 4447d88b..b458d51c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -73,9 +73,6 @@ fn is_main() -> bool { #[cfg(not(panic = "unwind"))] compile_error!("This crate must be compiled with `-Cpanic=unwind`."); -#[cfg(not(target_endian = "little"))] -compile_error!("Target architecture is not supported."); - #[cfg(not(miri))] #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] #[cfg(any(target_os = "linux", target_os = "macos"))]