Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,8 @@ jobs:
if: startsWith(matrix.os, 'windows')
shell: bash
run: echo "RUSTFLAGS=-C debuginfo=0" >> "${GITHUB_ENV}"
- name: Install llvm-tools
run: rustup component add llvm-tools-preview
- name: cargo test
run: cargo nextest run --features password-storage
- name: test cross compiling with zig
Expand Down
17 changes: 5 additions & 12 deletions src/build_orchestrator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -75,24 +75,17 @@ impl<'a> BuildOrchestrator<'a> {
fn build_wheels_pgo_single_pass(&self, pgo_command: String) -> Result<Vec<BuiltWheelMetadata>> {
let pgo_ctx = PgoContext::new(pgo_command)?;

let instrumentation_python = self
.context
.python
.interpreter
.first()
.context(
"PGO builds require a Python interpreter. \
let instrumentation_python = self.context.python.interpreter.first().context(
"PGO builds require a Python interpreter. \
Please specify one with `--interpreter`.",
)?
.executable
.clone();
)?;

// Phase 1: Build a single instrumented wheel for training.
eprintln!("📊 Phase 1/3: Building instrumented wheel...");
let mut instrumented_ctx = self.clone_context_for_pgo(PgoPhase::Generate(
pgo_ctx.profdata_dir_path().to_path_buf(),
));
instrumented_ctx.python.interpreter = vec![self.context.python.interpreter[0].clone()];
instrumented_ctx.python.interpreter = vec![instrumentation_python.clone()];
let instrumented_out =
tempfile::TempDir::new().context("Failed to create temp dir for instrumented wheel")?;
instrumented_ctx.artifact.out = instrumented_out.path().to_path_buf();
Expand Down Expand Up @@ -163,7 +156,7 @@ impl<'a> BuildOrchestrator<'a> {
// Phase 2: Run instrumentation with this interpreter
eprintln!(" 🔬 Phase 2/3: Running PGO instrumentation...");
pgo_ctx.run_instrumentation(
&python_interpreter.executable,
python_interpreter,
&instrumented_wheel_path,
self.context,
)?;
Expand Down
21 changes: 15 additions & 6 deletions src/pgo.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use crate::PythonInterpreter;
use crate::develop::install_backend::{find_uv_bin, find_uv_python};
use anyhow::{Context, Result, bail};
use fs_err as fs;
Expand Down Expand Up @@ -130,38 +131,45 @@ impl PgoContext {
/// 4. Run the instrumentation command
pub(crate) fn run_instrumentation(
&self,
python: &Path,
python: &PythonInterpreter,
wheel_path: &Path,
build_context: &crate::BuildContext,
) -> Result<()> {
let venv_dir = TempDir::new().context("Failed to create temporary venv directory")?;
let venv_path = venv_dir.path();

// Detect uv: try the binary first, then the Python module
let uv = find_uv_python(python).or_else(|_| find_uv_bin()).ok();
let uv = find_uv_python(&python.executable)
.or_else(|_| find_uv_bin())
.ok();

// Create venv
if let Some((uv_path, uv_args)) = &uv {
debug!("Creating venv with uv");
let status = Command::new(uv_path)
.args(uv_args.iter().copied())
.args(["venv", "--python"])
.arg(python)
.arg(python.as_uv_python_arg())
.arg(venv_path)
.status()
.context("Failed to create virtual environment with uv")?;
if !status.success() {
bail!("Failed to create virtual environment with uv (exit status: {status})");
}
} else {
let status = Command::new(python)
} else if python.runnable {
let status = Command::new(&python.executable)
.args(["-m", "venv"])
.arg(venv_path)
.status()
.context("Failed to create virtual environment")?;
if !status.success() {
bail!("Failed to create virtual environment (exit status: {status})");
}
} else {
bail!(
"Python interpreter {} is not runnable, cannot create virtual environment for PGO instrumentation",
python.executable.display()
);
}
debug!("Created temporary venv at {}", venv_path.display());

Expand Down Expand Up @@ -263,7 +271,8 @@ impl PgoContext {
cmd.current_dir(project_dir)
.env("LLVM_PROFILE_FILE", &profraw_pattern)
.env("PATH", &path_env)
.env("VIRTUAL_ENV", venv_path);
.env("VIRTUAL_ENV", venv_path)
.env("UV_PYTHON", &venv_python);

let status = cmd.status().with_context(|| {
format!(
Expand Down
108 changes: 108 additions & 0 deletions src/python_interpreter/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ use crate::Target;
use crate::auditwheel::PlatformTag;
use crate::bridge::{ABI3T_MINIMUM_PYTHON_MINOR, StableAbiKind};
use anyhow::{Result, bail};
use std::borrow::Cow;
use std::ffi::OsStr;
use std::fmt;
use std::ops::Deref;
use std::path::{Path, PathBuf};
Expand Down Expand Up @@ -150,6 +152,35 @@ impl PythonInterpreter {
}
}

/// Formats this interpreter as a value for uv's `--python` argument.
///
/// A runnable interpreter is passed by path. Otherwise the interpreter
/// is passed as a version request for uv to resolve (e.g. if maturin
/// received `-i 3.10`, and no runnable interpreter was found, maturin
/// will pass `cpython@3.10` to uv).
pub fn as_uv_python_arg(&self) -> Cow<'_, OsStr> {
if self.runnable && !self.executable.as_os_str().is_empty() {
return self.executable.as_os_str().into();
}
let implementation = match self.interpreter_kind {
InterpreterKind::CPython => "cpython",
InterpreterKind::PyPy => "pypy",
InterpreterKind::GraalPy => "graalpy",
};
let mut request = format!(
"{implementation}@{major}.{minor}{abiflags}",
major = self.major,
minor = self.minor,
abiflags = self.abiflags,
);
// `3.14` might resolve to free-threaded python depending what is on the users'
// path (see https://github.com/astral-sh/uv/issues/16722), explicitly request `+gil` if needed
if self.interpreter_kind == InterpreterKind::CPython && !self.gil_disabled {
request.push_str("+gil");
}
Cow::Owned(request.into())
}

/// Returns the supported python environment in the PEP 425 format used for the wheel filename:
/// {python tag}-{abi tag}-{platform tag}
///
Expand Down Expand Up @@ -478,3 +509,80 @@ impl fmt::Display for PythonInterpreter {
}
}
}

#[cfg(test)]
mod tests {
use super::*;

fn interpreter(
kind: InterpreterKind,
minor: usize,
abiflags: &str,
gil_disabled: bool,
) -> PythonInterpreter {
PythonInterpreter {
config: InterpreterConfig {
major: 3,
minor,
interpreter_kind: kind,
abiflags: abiflags.to_string(),
ext_suffix: String::new(),
pointer_width: None,
gil_disabled,
},
executable: PathBuf::new(),
platform: None,
runnable: false,
implementation_name: kind.to_string().to_ascii_lowercase(),
soabi: None,
}
}

#[test]
fn uv_python_arg_runnable_uses_executable() {
let mut interp = interpreter(InterpreterKind::CPython, 10, "", false);
interp.executable = PathBuf::from("/usr/bin/python3.10");
interp.runnable = true;
assert_eq!(
interp.as_uv_python_arg(),
Cow::Borrowed(OsStr::new("/usr/bin/python3.10"))
);
}

#[test]
fn uv_python_arg_runnable_but_empty_executable_falls_back() {
let mut interp = interpreter(InterpreterKind::CPython, 10, "", false);
interp.runnable = true;
assert_eq!(interp.as_uv_python_arg(), OsStr::new("cpython@3.10+gil"));
}

#[test]
fn uv_python_arg_cpython_requests_gil() {
let interp = interpreter(InterpreterKind::CPython, 14, "", false);
assert_eq!(interp.as_uv_python_arg(), OsStr::new("cpython@3.14+gil"));
}

#[test]
fn uv_python_arg_free_threaded_uses_abiflags() {
let interp = interpreter(InterpreterKind::CPython, 14, "t", true);
assert_eq!(interp.as_uv_python_arg(), OsStr::new("cpython@3.14t"));
}

#[test]
fn uv_python_arg_debug_keeps_abiflags_and_gil() {
let interp = interpreter(InterpreterKind::CPython, 13, "d", false);
assert_eq!(interp.as_uv_python_arg(), OsStr::new("cpython@3.13d+gil"));
}

#[test]
fn uv_python_arg_pypy_omits_gil() {
let interp = interpreter(InterpreterKind::PyPy, 10, "", false);
assert_eq!(interp.as_uv_python_arg(), OsStr::new("pypy@3.10"));
}

#[test]
fn uv_python_arg_graalpy_omits_gil() {
let interp = interpreter(InterpreterKind::GraalPy, 11, "", false);
assert_eq!(interp.as_uv_python_arg(), OsStr::new("graalpy@3.11"));
}
}
1 change: 1 addition & 0 deletions test-crates/pyo3-mixed/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,4 @@ print_cli_args = "pyo3_mixed:print_cli_args"

[tool.maturin]
include = ["pyo3_mixed/assets/*"]
pgo-command = "print_cli_args"
9 changes: 9 additions & 0 deletions tests/common/integration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ pub struct IntegrationCase<'a> {
pub target: Option<&'a str>,
/// Do introspection to generate type stubs
pub generate_stubs: bool,
/// Build with `--pgo` flag
pub pgo: bool,
}

impl<'a> IntegrationCase<'a> {
Expand All @@ -49,6 +51,7 @@ impl<'a> IntegrationCase<'a> {
zig: false,
target: None,
generate_stubs: false,
pgo: false,
}
}

Expand All @@ -71,6 +74,11 @@ impl<'a> IntegrationCase<'a> {
self.generate_stubs = true;
self
}

pub fn pgo(mut self) -> Self {
self.pgo = true;
self
}
}

// The zig wrappers consult CARGO_BIN_EXE_cargo-zigbuild via process env, so zig builds run in a
Expand Down Expand Up @@ -297,6 +305,7 @@ pub fn test_integration(case: &IntegrationCase<'_>) -> Result<()> {
.into_build_context()
.strip(Some(cfg!(feature = "faster-tests")))
.editable(false)
.pgo(case.pgo)
.build()?;
let wheels = BuildOrchestrator::new(&build_context).build_wheels()?;

Expand Down
2 changes: 2 additions & 0 deletions tests/run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ mod errors;
mod integration;
#[path = "run/pep517.rs"]
mod pep517;
#[path = "run/pgo.rs"]
mod pgo;
#[path = "run/sdist.rs"]
mod sdist;
#[path = "run/wheel.rs"]
Expand Down
9 changes: 9 additions & 0 deletions tests/run/pgo.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
use crate::common::handle_result;
use crate::common::integration::{self, IntegrationCase};

#[test]
fn pgo_pyo3_mixed() {
handle_result(integration::test_integration(
&IntegrationCase::new("pgo-pyo3-mixed", "test-crates/pyo3-mixed").pgo(),
));
}
Loading