diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt index 8acb1468f6..190de62eea 100644 --- a/.cspell.dict/cpython.txt +++ b/.cspell.dict/cpython.txt @@ -42,6 +42,7 @@ numer orelse pathconfig patma +platstdlib posonlyarg posonlyargs prec diff --git a/crates/stdlib/src/syslog.rs b/crates/stdlib/src/syslog.rs index adba6f297c..d0ed3f6094 100644 --- a/crates/stdlib/src/syslog.rs +++ b/crates/stdlib/src/syslog.rs @@ -26,7 +26,7 @@ mod syslog { use libc::{LOG_AUTHPRIV, LOG_CRON, LOG_PERROR}; fn get_argv(vm: &VirtualMachine) -> Option { - if let Some(argv) = vm.state.settings.argv.first() + if let Some(argv) = vm.state.config.settings.argv.first() && !argv.is_empty() { return Some( diff --git a/crates/vm/src/getpath.rs b/crates/vm/src/getpath.rs new file mode 100644 index 0000000000..423b9b5413 --- /dev/null +++ b/crates/vm/src/getpath.rs @@ -0,0 +1,385 @@ +//! Path configuration for RustPython (ref: Modules/getpath.py) +//! +//! This module implements Python path calculation logic following getpath.py. +//! It uses landmark-based search to locate prefix, exec_prefix, and stdlib directories. +//! +//! The main entry point is `init_path_config()` which computes Paths from Settings. + +use crate::vm::{Paths, Settings}; +use std::env; +use std::path::{Path, PathBuf}; + +// Platform-specific landmarks (ref: getpath.py PLATFORM CONSTANTS) + +#[cfg(not(windows))] +mod platform { + use crate::version; + + pub const BUILDDIR_TXT: &str = "pybuilddir.txt"; + pub const BUILD_LANDMARK: &str = "Modules/Setup.local"; + pub const VENV_LANDMARK: &str = "pyvenv.cfg"; + pub const BUILDSTDLIB_LANDMARK: &str = "Lib/os.py"; + + pub fn stdlib_subdir() -> String { + format!("lib/python{}.{}", version::MAJOR, version::MINOR) + } + + pub fn stdlib_landmarks() -> [String; 2] { + let subdir = stdlib_subdir(); + [format!("{}/os.py", subdir), format!("{}/os.pyc", subdir)] + } + + pub fn platstdlib_landmark() -> String { + format!( + "lib/python{}.{}/lib-dynload", + version::MAJOR, + version::MINOR + ) + } + + pub fn zip_landmark() -> String { + format!("lib/python{}{}.zip", version::MAJOR, version::MINOR) + } +} + +#[cfg(windows)] +mod platform { + use crate::version; + + pub const BUILDDIR_TXT: &str = "pybuilddir.txt"; + pub const BUILD_LANDMARK: &str = "Modules\\Setup.local"; + pub const VENV_LANDMARK: &str = "pyvenv.cfg"; + pub const BUILDSTDLIB_LANDMARK: &str = "Lib\\os.py"; + pub const STDLIB_SUBDIR: &str = "Lib"; + + pub fn stdlib_landmarks() -> [String; 2] { + ["Lib\\os.py".into(), "Lib\\os.pyc".into()] + } + + pub fn platstdlib_landmark() -> String { + "DLLs".into() + } + + pub fn zip_landmark() -> String { + format!("python{}{}.zip", version::MAJOR, version::MINOR) + } +} + +// Helper functions (ref: getpath.py HELPER FUNCTIONS) + +/// Search upward from a directory for landmark files/directories +/// Returns the directory where a landmark was found +fn search_up(start: P, landmarks: &[&str], test: F) -> Option +where + P: AsRef, + F: Fn(&Path) -> bool, +{ + let mut current = start.as_ref().to_path_buf(); + loop { + for landmark in landmarks { + let path = current.join(landmark); + if test(&path) { + return Some(current); + } + } + if !current.pop() { + return None; + } + } +} + +/// Search upward for a file landmark +fn search_up_file>(start: P, landmarks: &[&str]) -> Option { + search_up(start, landmarks, |p| p.is_file()) +} + +/// Search upward for a directory landmark +#[cfg(not(windows))] +fn search_up_dir>(start: P, landmarks: &[&str]) -> Option { + search_up(start, landmarks, |p| p.is_dir()) +} + +// Path computation functions + +/// Compute path configuration from Settings +/// +/// This function should be called before interpreter initialization. +/// It returns a Paths struct with all computed path values. +pub fn init_path_config(settings: &Settings) -> Paths { + let mut paths = Paths::default(); + + // Step 0: Get executable path + let executable = get_executable_path(); + paths.executable = executable + .as_ref() + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_default(); + + let exe_dir = executable + .as_ref() + .and_then(|p| p.parent().map(PathBuf::from)); + + // Step 1: Check for __PYVENV_LAUNCHER__ environment variable + if let Ok(launcher) = env::var("__PYVENV_LAUNCHER__") { + paths.base_executable = launcher; + } + + // Step 2: Check for venv (pyvenv.cfg) and get 'home' + let (venv_prefix, home_dir) = detect_venv(&exe_dir); + let search_dir = home_dir.clone().or(exe_dir.clone()); + + // Step 3: Check for build directory + let build_prefix = detect_build_directory(&search_dir); + + // Step 4: Calculate prefix via landmark search + // When in venv, search_dir is home_dir, so this gives us the base Python's prefix + let calculated_prefix = calculate_prefix(&search_dir, &build_prefix); + + // Step 5: Set prefix and base_prefix + if venv_prefix.is_some() { + // In venv: prefix = venv directory, base_prefix = original Python's prefix + paths.prefix = venv_prefix + .as_ref() + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_else(|| calculated_prefix.clone()); + paths.base_prefix = calculated_prefix; + } else { + // Not in venv: prefix == base_prefix + paths.prefix = calculated_prefix.clone(); + paths.base_prefix = calculated_prefix; + } + + // Step 6: Calculate exec_prefix + paths.exec_prefix = if venv_prefix.is_some() { + // In venv: exec_prefix = prefix (venv directory) + paths.prefix.clone() + } else { + calculate_exec_prefix(&search_dir, &paths.prefix) + }; + paths.base_exec_prefix = paths.base_prefix.clone(); + + // Step 7: Calculate base_executable (if not already set by __PYVENV_LAUNCHER__) + if paths.base_executable.is_empty() { + paths.base_executable = calculate_base_executable(executable.as_ref(), &home_dir); + } + + // Step 8: Build module_search_paths + paths.module_search_paths = + build_module_search_paths(settings, &paths.prefix, &paths.exec_prefix); + + paths +} + +/// Get default prefix value +fn default_prefix() -> String { + std::option_env!("RUSTPYTHON_PREFIX") + .map(String::from) + .unwrap_or_else(|| { + if cfg!(windows) { + "C:".to_owned() + } else { + "/usr/local".to_owned() + } + }) +} + +/// Detect virtual environment by looking for pyvenv.cfg +/// Returns (venv_prefix, home_dir from pyvenv.cfg) +fn detect_venv(exe_dir: &Option) -> (Option, Option) { + // Try exe_dir/../pyvenv.cfg first (standard venv layout: venv/bin/python) + if let Some(dir) = exe_dir + && let Some(venv_dir) = dir.parent() + { + let cfg = venv_dir.join(platform::VENV_LANDMARK); + if cfg.exists() + && let Some(home) = parse_pyvenv_home(&cfg) + { + return (Some(venv_dir.to_path_buf()), Some(PathBuf::from(home))); + } + } + + // Try exe_dir/pyvenv.cfg (alternative layout) + if let Some(dir) = exe_dir { + let cfg = dir.join(platform::VENV_LANDMARK); + if cfg.exists() + && let Some(home) = parse_pyvenv_home(&cfg) + { + return (Some(dir.clone()), Some(PathBuf::from(home))); + } + } + + (None, None) +} + +/// Detect if running from a build directory +fn detect_build_directory(exe_dir: &Option) -> Option { + let dir = exe_dir.as_ref()?; + + // Check for pybuilddir.txt (indicates build directory) + if dir.join(platform::BUILDDIR_TXT).exists() { + return Some(dir.clone()); + } + + // Check for Modules/Setup.local (build landmark) + if dir.join(platform::BUILD_LANDMARK).exists() { + return Some(dir.clone()); + } + + // Search up for Lib/os.py (build stdlib landmark) + search_up_file(dir, &[platform::BUILDSTDLIB_LANDMARK]) +} + +/// Calculate prefix by searching for landmarks +fn calculate_prefix(exe_dir: &Option, build_prefix: &Option) -> String { + // 1. If build directory detected, use it + if let Some(bp) = build_prefix { + return bp.to_string_lossy().into_owned(); + } + + if let Some(dir) = exe_dir { + // 2. Search for ZIP landmark + let zip = platform::zip_landmark(); + if let Some(prefix) = search_up_file(dir, &[&zip]) { + return prefix.to_string_lossy().into_owned(); + } + + // 3. Search for stdlib landmarks (os.py) + let landmarks = platform::stdlib_landmarks(); + let refs: Vec<&str> = landmarks.iter().map(|s| s.as_str()).collect(); + if let Some(prefix) = search_up_file(dir, &refs) { + return prefix.to_string_lossy().into_owned(); + } + } + + // 4. Fallback to default + default_prefix() +} + +/// Calculate exec_prefix +fn calculate_exec_prefix(exe_dir: &Option, prefix: &str) -> String { + #[cfg(windows)] + { + // Windows: exec_prefix == prefix + let _ = exe_dir; // silence unused warning + prefix.to_owned() + } + + #[cfg(not(windows))] + { + // POSIX: search for lib-dynload directory + if let Some(dir) = exe_dir { + let landmark = platform::platstdlib_landmark(); + if let Some(exec_prefix) = search_up_dir(dir, &[&landmark]) { + return exec_prefix.to_string_lossy().into_owned(); + } + } + // Fallback: same as prefix + prefix.to_owned() + } +} + +/// Calculate base_executable +fn calculate_base_executable(executable: Option<&PathBuf>, home_dir: &Option) -> String { + // If in venv and we have home, construct base_executable from home + if let (Some(exe), Some(home)) = (executable, home_dir) + && let Some(exe_name) = exe.file_name() + { + let base = home.join(exe_name); + return base.to_string_lossy().into_owned(); + } + + // Otherwise, base_executable == executable + executable + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_default() +} + +/// Build the complete module_search_paths (sys.path) +fn build_module_search_paths(settings: &Settings, prefix: &str, exec_prefix: &str) -> Vec { + let mut paths = Vec::new(); + + // 1. PYTHONPATH/RUSTPYTHONPATH from settings + paths.extend(settings.path_list.iter().cloned()); + + // 2. ZIP file path + let zip_path = PathBuf::from(prefix).join(platform::zip_landmark()); + paths.push(zip_path.to_string_lossy().into_owned()); + + // 3. stdlib and platstdlib directories + #[cfg(not(windows))] + { + // POSIX: stdlib first, then lib-dynload + let stdlib_dir = PathBuf::from(prefix).join(platform::stdlib_subdir()); + paths.push(stdlib_dir.to_string_lossy().into_owned()); + + let platstdlib = PathBuf::from(exec_prefix).join(platform::platstdlib_landmark()); + paths.push(platstdlib.to_string_lossy().into_owned()); + } + + #[cfg(windows)] + { + // Windows: DLLs first, then Lib + let platstdlib = PathBuf::from(exec_prefix).join(platform::platstdlib_landmark()); + paths.push(platstdlib.to_string_lossy().into_owned()); + + let stdlib_dir = PathBuf::from(prefix).join(platform::STDLIB_SUBDIR); + paths.push(stdlib_dir.to_string_lossy().into_owned()); + } + + paths +} + +/// Get the current executable path +fn get_executable_path() -> Option { + #[cfg(not(target_arch = "wasm32"))] + { + let exec_arg = env::args_os().next()?; + which::which(exec_arg).ok() + } + #[cfg(target_arch = "wasm32")] + { + let exec_arg = env::args().next()?; + Some(PathBuf::from(exec_arg)) + } +} + +/// Parse pyvenv.cfg and extract the 'home' key value +fn parse_pyvenv_home(pyvenv_cfg: &Path) -> Option { + let content = std::fs::read_to_string(pyvenv_cfg).ok()?; + + for line in content.lines() { + if let Some((key, value)) = line.split_once('=') + && key.trim().to_lowercase() == "home" + { + return Some(value.trim().to_string()); + } + } + + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_init_path_config() { + let settings = Settings::default(); + let paths = init_path_config(&settings); + // Just verify it doesn't panic and returns valid paths + assert!(!paths.prefix.is_empty()); + } + + #[test] + fn test_search_up() { + // Test with a path that doesn't have any landmarks + let result = search_up_file(std::env::temp_dir(), &["nonexistent_landmark_xyz"]); + assert!(result.is_none()); + } + + #[test] + fn test_default_prefix() { + let prefix = default_prefix(); + assert!(!prefix.is_empty()); + } +} diff --git a/crates/vm/src/import.rs b/crates/vm/src/import.rs index 3f4a437c59..39748655e0 100644 --- a/crates/vm/src/import.rs +++ b/crates/vm/src/import.rs @@ -206,7 +206,7 @@ fn remove_importlib_frames_inner( // TODO: This function should do nothing on verbose mode. // TODO: Fix this function after making PyTraceback.next mutable pub fn remove_importlib_frames(vm: &VirtualMachine, exc: &Py) { - if vm.state.settings.verbose != 0 { + if vm.state.config.settings.verbose != 0 { return; } diff --git a/crates/vm/src/lib.rs b/crates/vm/src/lib.rs index 923b33d2ac..f461c61295 100644 --- a/crates/vm/src/lib.rs +++ b/crates/vm/src/lib.rs @@ -60,6 +60,7 @@ pub mod exceptions; pub mod format; pub mod frame; pub mod function; +pub mod getpath; pub mod import; mod intern; pub mod iter; diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index 442bb79b94..72d2c72415 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -132,7 +132,7 @@ mod builtins { let optimize: i32 = args.optimize.map_or(Ok(-1), |v| v.try_to_primitive(vm))?; let optimize: u8 = if optimize == -1 { - vm.state.settings.optimize + vm.state.config.settings.optimize } else { optimize .try_into() @@ -1080,7 +1080,7 @@ pub fn init_module(vm: &VirtualMachine, module: &Py) { builtins::extend_module(vm, module).unwrap(); - let debug_mode: bool = vm.state.settings.optimize == 0; + let debug_mode: bool = vm.state.config.settings.optimize == 0; // Create dynamic ExceptionGroup with multiple inheritance (BaseExceptionGroup + Exception) let exception_group = crate::exception_group::exception_group(); diff --git a/crates/vm/src/stdlib/imp.rs b/crates/vm/src/stdlib/imp.rs index 596847776f..76b3bfd124 100644 --- a/crates/vm/src/stdlib/imp.rs +++ b/crates/vm/src/stdlib/imp.rs @@ -91,7 +91,7 @@ mod _imp { #[pyattr] fn check_hash_based_pycs(vm: &VirtualMachine) -> PyStrRef { vm.ctx - .new_str(vm.state.settings.check_hash_pycs_mode.to_string()) + .new_str(vm.state.config.settings.check_hash_pycs_mode.to_string()) } #[pyfunction] diff --git a/crates/vm/src/stdlib/io.rs b/crates/vm/src/stdlib/io.rs index 547e482f13..ba3576a176 100644 --- a/crates/vm/src/stdlib/io.rs +++ b/crates/vm/src/stdlib/io.rs @@ -2398,7 +2398,9 @@ mod _io { *data = None; let encoding = match args.encoding { - None if vm.state.settings.utf8_mode > 0 => identifier_utf8!(vm, utf_8).to_owned(), + None if vm.state.config.settings.utf8_mode > 0 => { + identifier_utf8!(vm, utf_8).to_owned() + } Some(enc) if enc.as_str() != "locale" => enc, _ => { // None without utf8_mode or "locale" encoding diff --git a/crates/vm/src/stdlib/signal.rs b/crates/vm/src/stdlib/signal.rs index 4eacb10154..6771a95040 100644 --- a/crates/vm/src/stdlib/signal.rs +++ b/crates/vm/src/stdlib/signal.rs @@ -114,7 +114,7 @@ pub(crate) mod _signal { module: &Py, vm: &VirtualMachine, ) { - if vm.state.settings.install_signal_handlers { + if vm.state.config.settings.install_signal_handlers { let sig_dfl = vm.new_pyobj(SIG_DFL as u8); let sig_ign = vm.new_pyobj(SIG_IGN as u8); diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index f9bd2b5945..cfe6f9f5e6 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -25,7 +25,6 @@ mod sys { use std::{ env::{self, VarError}, io::Read, - path, sync::atomic::Ordering, }; @@ -96,25 +95,20 @@ mod sys { const DLLHANDLE: usize = 0; #[pyattr] - const fn default_prefix(_vm: &VirtualMachine) -> &'static str { - // TODO: the windows one doesn't really make sense - if cfg!(windows) { "C:" } else { "/usr/local" } + fn prefix(vm: &VirtualMachine) -> String { + vm.state.config.paths.prefix.clone() } #[pyattr] - fn prefix(vm: &VirtualMachine) -> &'static str { - option_env!("RUSTPYTHON_PREFIX").unwrap_or_else(|| default_prefix(vm)) + fn base_prefix(vm: &VirtualMachine) -> String { + vm.state.config.paths.base_prefix.clone() } #[pyattr] - fn base_prefix(vm: &VirtualMachine) -> &'static str { - option_env!("RUSTPYTHON_BASEPREFIX").unwrap_or_else(|| prefix(vm)) + fn exec_prefix(vm: &VirtualMachine) -> String { + vm.state.config.paths.exec_prefix.clone() } #[pyattr] - fn exec_prefix(vm: &VirtualMachine) -> &'static str { - option_env!("RUSTPYTHON_BASEPREFIX").unwrap_or_else(|| prefix(vm)) - } - #[pyattr] - fn base_exec_prefix(vm: &VirtualMachine) -> &'static str { - option_env!("RUSTPYTHON_BASEPREFIX").unwrap_or_else(|| exec_prefix(vm)) + fn base_exec_prefix(vm: &VirtualMachine) -> String { + vm.state.config.paths.base_exec_prefix.clone() } #[pyattr] fn platlibdir(_vm: &VirtualMachine) -> &'static str { @@ -126,6 +120,7 @@ mod sys { #[pyattr] fn argv(vm: &VirtualMachine) -> Vec { vm.state + .config .settings .argv .iter() @@ -162,117 +157,18 @@ mod sys { } #[pyattr] - fn _base_executable(vm: &VirtualMachine) -> PyObjectRef { - let ctx = &vm.ctx; - // First check __PYVENV_LAUNCHER__ environment variable - if let Ok(var) = env::var("__PYVENV_LAUNCHER__") { - return ctx.new_str(var).into(); - } - - // Try to detect if we're running from a venv by looking for pyvenv.cfg - if let Some(base_exe) = get_venv_base_executable() { - return ctx.new_str(base_exe).into(); - } - - executable(vm) - } - - /// Try to find base executable from pyvenv.cfg (see getpath.py) - fn get_venv_base_executable() -> Option { - // TODO: This is a minimal implementation of getpath.py - // To fully support all cases, `getpath.py` should be placed in @crates/vm/Lib/python_builtins/ - - // Get current executable path - #[cfg(not(target_arch = "wasm32"))] - let exe_path = { - let exec_arg = env::args_os().next()?; - which::which(exec_arg).ok()? - }; - #[cfg(target_arch = "wasm32")] - let exe_path = { - let exec_arg = env::args().next()?; - path::PathBuf::from(exec_arg) - }; - - let exe_dir = exe_path.parent()?; - let exe_name = exe_path.file_name()?; - - // Look for pyvenv.cfg in parent directory (typical venv layout: venv/bin/python) - let venv_dir = exe_dir.parent()?; - let pyvenv_cfg = venv_dir.join("pyvenv.cfg"); - - if !pyvenv_cfg.exists() { - return None; - } - - // Parse pyvenv.cfg and extract home directory - let content = std::fs::read_to_string(&pyvenv_cfg).ok()?; - - for line in content.lines() { - if let Some((key, value)) = line.split_once('=') { - let key = key.trim().to_lowercase(); - let value = value.trim(); - - if key == "home" { - // First try to resolve symlinks (getpath.py line 373-377) - if let Ok(resolved) = std::fs::canonicalize(&exe_path) - && resolved != exe_path - { - return Some(resolved.to_string_lossy().into_owned()); - } - // Fallback: home_dir + executable_name (getpath.py line 381) - let base_exe = path::Path::new(value).join(exe_name); - return Some(base_exe.to_string_lossy().into_owned()); - } - } - } - - None + fn _base_executable(vm: &VirtualMachine) -> String { + vm.state.config.paths.base_executable.clone() } #[pyattr] fn dont_write_bytecode(vm: &VirtualMachine) -> bool { - !vm.state.settings.write_bytecode + !vm.state.config.settings.write_bytecode } #[pyattr] - fn executable(vm: &VirtualMachine) -> PyObjectRef { - let ctx = &vm.ctx; - #[cfg(not(target_arch = "wasm32"))] - { - if let Some(exec_path) = env::args_os().next() - && let Ok(path) = which::which(exec_path) - { - return ctx - .new_str( - path.into_os_string() - .into_string() - .unwrap_or_else(|p| p.to_string_lossy().into_owned()), - ) - .into(); - } - } - if let Some(exec_path) = env::args().next() { - let path = path::Path::new(&exec_path); - if !path.exists() { - return ctx.new_str(ascii!("")).into(); - } - if path.is_absolute() { - return ctx.new_str(exec_path).into(); - } - if let Ok(dir) = env::current_dir() - && let Ok(dir) = dir.into_os_string().into_string() - { - return ctx - .new_str(format!( - "{}/{}", - dir, - exec_path.strip_prefix("./").unwrap_or(&exec_path) - )) - .into(); - } - } - ctx.none() + fn executable(vm: &VirtualMachine) -> String { + vm.state.config.paths.executable.clone() } #[pyattr] @@ -312,8 +208,9 @@ mod sys { #[pyattr] fn path(vm: &VirtualMachine) -> Vec { vm.state - .settings - .path_list + .config + .paths + .module_search_paths .iter() .map(|path| vm.ctx.new_str(path.clone()).into()) .collect() @@ -350,7 +247,7 @@ mod sys { fn _xoptions(vm: &VirtualMachine) -> PyDictRef { let ctx = &vm.ctx; let xopts = ctx.new_dict(); - for (key, value) in &vm.state.settings.xoptions { + for (key, value) in &vm.state.config.settings.xoptions { let value = value.as_ref().map_or_else( || ctx.new_bool(true).into(), |s| ctx.new_str(s.clone()).into(), @@ -363,6 +260,7 @@ mod sys { #[pyattr] fn warnoptions(vm: &VirtualMachine) -> Vec { vm.state + .config .settings .warnoptions .iter() @@ -507,7 +405,7 @@ mod sys { #[pyattr] fn flags(vm: &VirtualMachine) -> PyTupleRef { - PyFlags::from_data(FlagsData::from_settings(&vm.state.settings), vm) + PyFlags::from_data(FlagsData::from_settings(&vm.state.config.settings), vm) } #[pyattr] diff --git a/crates/vm/src/vm/compile.rs b/crates/vm/src/vm/compile.rs index 6f1ea73492..44332cda83 100644 --- a/crates/vm/src/vm/compile.rs +++ b/crates/vm/src/vm/compile.rs @@ -38,7 +38,7 @@ impl VirtualMachine { } // TODO: check if this is proper place - if !self.state.settings.safe_path { + if !self.state.config.settings.safe_path { let dir = std::path::Path::new(path) .parent() .unwrap() diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index 05613d4338..503feb3dc7 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -1,5 +1,5 @@ -use super::{Context, VirtualMachine, setting::Settings, thread}; -use crate::{PyResult, stdlib::atexit, vm::PyBaseExceptionRef}; +use super::{Context, PyConfig, VirtualMachine, setting::Settings, thread}; +use crate::{PyResult, getpath, stdlib::atexit, vm::PyBaseExceptionRef}; use std::sync::atomic::Ordering; /// The general interface for the VM @@ -47,10 +47,14 @@ impl Interpreter { where F: FnOnce(&mut VirtualMachine), { + // Compute path configuration from settings + let paths = getpath::init_path_config(&settings); + let config = PyConfig::new(settings, paths); + let ctx = Context::genesis(); crate::types::TypeZoo::extend(ctx); crate::exceptions::ExceptionZoo::extend(ctx); - let mut vm = VirtualMachine::new(settings, ctx.clone()); + let mut vm = VirtualMachine::new(config, ctx.clone()); init(&mut vm); vm.initialize(); Self { vm } diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 4574b2de37..3409245405 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -50,7 +50,7 @@ use std::{ pub use context::Context; pub use interpreter::Interpreter; pub(crate) use method::PyMethod; -pub use setting::{CheckHashPycsMode, Settings}; +pub use setting::{CheckHashPycsMode, Paths, PyConfig, Settings}; pub const MAX_MEMORY_SIZE: usize = isize::MAX as usize; @@ -87,7 +87,7 @@ struct ExceptionStack { } pub struct PyGlobalState { - pub settings: Settings, + pub config: PyConfig, pub module_inits: stdlib::StdlibMap, pub frozen: HashMap<&'static str, FrozenModule, ahash::RandomState>, pub stacksize: AtomicCell, @@ -114,7 +114,7 @@ pub fn process_hash_secret_seed() -> u32 { impl VirtualMachine { /// Create a new `VirtualMachine` structure. - fn new(settings: Settings, ctx: PyRc) -> Self { + fn new(config: PyConfig, ctx: PyRc) -> Self { flame_guard!("new VirtualMachine"); // make a new module without access to the vm; doesn't @@ -141,7 +141,7 @@ impl VirtualMachine { let module_inits = stdlib::get_module_inits(); - let seed = match settings.hash_seed { + let seed = match config.settings.hash_seed { Some(seed) => seed, None => process_hash_secret_seed(), }; @@ -151,7 +151,7 @@ impl VirtualMachine { let warnings = WarningsState::init_state(&ctx); - let int_max_str_digits = AtomicCell::new(match settings.int_max_str_digits { + let int_max_str_digits = AtomicCell::new(match config.settings.int_max_str_digits { -1 => 4300, other => other, } as usize); @@ -171,7 +171,7 @@ impl VirtualMachine { signal_rx: None, repr_guards: RefCell::default(), state: PyRc::new(PyGlobalState { - settings, + config, module_inits, frozen: HashMap::default(), stacksize: AtomicCell::new(0), @@ -227,7 +227,7 @@ impl VirtualMachine { let rustpythonpath_env = std::env::var("RUSTPYTHONPATH").ok(); let pythonpath_env = std::env::var("PYTHONPATH").ok(); let env_set = rustpythonpath_env.as_ref().is_some() || pythonpath_env.as_ref().is_some(); - let path_contains_env = self.state.settings.path_list.iter().any(|s| { + let path_contains_env = self.state.config.paths.module_search_paths.iter().any(|s| { Some(s.as_str()) == rustpythonpath_env.as_deref() || Some(s.as_str()) == pythonpath_env.as_deref() }); @@ -238,7 +238,7 @@ impl VirtualMachine { } else if path_contains_env { "RUSTPYTHONPATH or PYTHONPATH is set, but it doesn't contain the encodings library. If you are customizing the RustPython vm/interpreter, try adding the stdlib directory to the path. If you are developing the RustPython interpreter, it might be a bug during development." } else { - "RUSTPYTHONPATH or PYTHONPATH is set, but it wasn't loaded to `Settings::path_list`. If you are going to customize the RustPython vm/interpreter, those environment variables are not loaded in the Settings struct by default. Please try creating a customized instance of the Settings struct. If you are developing the RustPython interpreter, it might be a bug during development." + "RUSTPYTHONPATH or PYTHONPATH is set, but it wasn't loaded to `PyConfig::paths::module_search_paths`. If you are going to customize the RustPython vm/interpreter, those environment variables are not loaded in the Settings struct by default. Please try creating a customized instance of the Settings struct. If you are developing the RustPython interpreter, it might be a bug during development." }; let mut msg = format!( @@ -303,7 +303,7 @@ impl VirtualMachine { let io = import::import_builtin(self, "_io")?; #[cfg(feature = "stdio")] let make_stdio = |name, fd, write| { - let buffered_stdio = self.state.settings.buffered_stdio; + let buffered_stdio = self.state.config.settings.buffered_stdio; let unbuffered = write && !buffered_stdio; let buf = crate::stdlib::io::open( self.ctx.new_int(fd).into(), @@ -364,7 +364,7 @@ impl VirtualMachine { let res = essential_init(); let importlib = self.expect_pyresult(res, "essential initialization failed"); - if self.state.settings.allow_external_library + if self.state.config.settings.allow_external_library && cfg!(feature = "rustpython-compiler") && let Err(e) = import::init_importlib_package(self, importlib) { @@ -374,8 +374,8 @@ impl VirtualMachine { self.print_exception(e); } - let _expect_stdlib = - cfg!(feature = "freeze-stdlib") || !self.state.settings.path_list.is_empty(); + let _expect_stdlib = cfg!(feature = "freeze-stdlib") + || !self.state.config.paths.module_search_paths.is_empty(); #[cfg(feature = "encodings")] if _expect_stdlib { @@ -389,7 +389,7 @@ impl VirtualMachine { // Here may not be the best place to give general `path_list` advice, // but bare rustpython_vm::VirtualMachine users skipped proper settings must hit here while properly setup vm never enters here. eprintln!( - "feature `encodings` is enabled but `settings.path_list` is empty. \ + "feature `encodings` is enabled but `paths.module_search_paths` is empty. \ Please add the library path to `settings.path_list`. If you intended to disable the entire standard library (including the `encodings` feature), please also make sure to disable the `encodings` feature.\n\ Tip: You may also want to add `\"\"` to `settings.path_list` in order to enable importing from the current working directory." ); @@ -505,7 +505,7 @@ impl VirtualMachine { #[cfg(feature = "rustpython-codegen")] pub fn compile_opts(&self) -> crate::compiler::CompileOpts { crate::compiler::CompileOpts { - optimize: self.state.settings.optimize, + optimize: self.state.config.settings.optimize, } } diff --git a/crates/vm/src/vm/setting.rs b/crates/vm/src/vm/setting.rs index deaca705c4..53e2cef116 100644 --- a/crates/vm/src/vm/setting.rs +++ b/crates/vm/src/vm/setting.rs @@ -1,8 +1,40 @@ #[cfg(feature = "flame-it")] use std::ffi::OsString; -/// Struct containing all kind of settings for the python vm. -/// Mostly `PyConfig` in CPython. +/// Path configuration computed at runtime (like PyConfig path outputs) +#[derive(Debug, Clone, Default)] +pub struct Paths { + /// sys.executable + pub executable: String, + /// sys._base_executable (original interpreter in venv) + pub base_executable: String, + /// sys.prefix + pub prefix: String, + /// sys.base_prefix + pub base_prefix: String, + /// sys.exec_prefix + pub exec_prefix: String, + /// sys.base_exec_prefix + pub base_exec_prefix: String, + /// Computed module_search_paths (complete sys.path) + pub module_search_paths: Vec, +} + +/// Combined configuration: user settings + computed paths +/// CPython directly exposes every fields under both of them. +/// We separate them to maintain better ownership discipline. +pub struct PyConfig { + pub settings: Settings, + pub paths: Paths, +} + +impl PyConfig { + pub fn new(settings: Settings, paths: Paths) -> Self { + Self { settings, paths } + } +} + +/// User-configurable settings for the python vm. #[non_exhaustive] pub struct Settings { /// -I diff --git a/src/interpreter.rs b/src/interpreter.rs index b79a1a0ffb..b4fd319cda 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -104,23 +104,25 @@ pub fn init_stdlib(vm: &mut VirtualMachine) { use rustpython_vm::common::rc::PyRc; let state = PyRc::get_mut(&mut vm.state).unwrap(); - let settings = &mut state.settings; - let path_list = std::mem::take(&mut settings.path_list); + // Collect additional paths to add + let mut additional_paths = Vec::new(); // BUILDTIME_RUSTPYTHONPATH should be set when distributing if let Some(paths) = option_env!("BUILDTIME_RUSTPYTHONPATH") { - settings.path_list.extend( + additional_paths.extend( crate::settings::split_paths(paths) .map(|path| path.into_os_string().into_string().unwrap()), ) } else { #[cfg(feature = "rustpython-pylib")] - settings - .path_list - .push(rustpython_pylib::LIB_PATH.to_owned()) + additional_paths.push(rustpython_pylib::LIB_PATH.to_owned()) } - settings.path_list.extend(path_list); + // Add to both path_list (for compatibility) and module_search_paths (for sys.path) + // Insert at the beginning so stdlib comes before user paths + for path in additional_paths.into_iter().rev() { + state.config.paths.module_search_paths.insert(0, path); + } } } diff --git a/src/lib.rs b/src/lib.rs index 84a774ab02..8d27805893 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -169,7 +169,7 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { let scope = setup_main_module(vm)?; - if !vm.state.settings.safe_path { + if !vm.state.config.settings.safe_path { // TODO: The prepending path depends on running mode // See https://docs.python.org/3/using/cmdline.html#cmdoption-P vm.run_code_string( @@ -189,7 +189,7 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { // Enable faulthandler if -X faulthandler, PYTHONFAULTHANDLER or -X dev is set // _PyFaulthandler_Init() - if vm.state.settings.faulthandler { + if vm.state.config.settings.faulthandler { let _ = vm.run_code_string( vm.new_scope_with_builtins(), "import faulthandler; faulthandler.enable()", @@ -198,8 +198,8 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { } let is_repl = matches!(run_mode, RunMode::Repl); - if !vm.state.settings.quiet - && (vm.state.settings.verbose > 0 || (is_repl && std::io::stdin().is_terminal())) + if !vm.state.config.settings.quiet + && (vm.state.config.settings.verbose > 0 || (is_repl && std::io::stdin().is_terminal())) { eprintln!( "Welcome to the magnificent Rust Python {} interpreter \u{1f631} \u{1f596}", @@ -232,7 +232,7 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { } RunMode::Repl => Ok(()), }; - if is_repl || vm.state.settings.inspect { + if is_repl || vm.state.config.settings.inspect { shell::run_shell(vm, scope)?; } else { res?; @@ -241,7 +241,7 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { #[cfg(feature = "flame-it")] { main_guard.end(); - if let Err(e) = write_profile(&vm.state.as_ref().settings) { + if let Err(e) = write_profile(&vm.state.as_ref().config.settings) { error!("Error writing profile information: {}", e); } }