From be10fdd6ff0721fb4e74855638145ba11ed71682 Mon Sep 17 00:00:00 2001 From: Conner Nilsen Date: Fri, 3 Oct 2025 15:56:09 -0700 Subject: [PATCH 1/5] Move a bunch of build system files around Differential Revision: D83881917 --- crates/pyrefly_build/src/lib.rs | 14 +++++----- .../src/{buck/query.rs => query/buck.rs} | 0 .../pyrefly_build/src/{buck => query}/mod.rs | 4 +-- .../src/{buck => source_db}/buck_check.rs | 0 .../src/{ => source_db}/map_db.rs | 0 .../src/{source_db.rs => source_db/mod.rs} | 4 +++ .../bxl.rs => source_db/query_source_db.rs} | 26 +++++++++---------- pyrefly/lib/commands/buck_check.rs | 2 +- pyrefly/lib/test/incremental.rs | 2 +- pyrefly/lib/test/state.rs | 2 +- pyrefly/lib/test/util.rs | 2 +- 11 files changed, 28 insertions(+), 28 deletions(-) rename crates/pyrefly_build/src/{buck/query.rs => query/buck.rs} (100%) rename crates/pyrefly_build/src/{buck => query}/mod.rs (79%) rename crates/pyrefly_build/src/{buck => source_db}/buck_check.rs (100%) rename crates/pyrefly_build/src/{ => source_db}/map_db.rs (100%) rename crates/pyrefly_build/src/{source_db.rs => source_db/mod.rs} (97%) rename crates/pyrefly_build/src/{buck/bxl.rs => source_db/query_source_db.rs} (96%) diff --git a/crates/pyrefly_build/src/lib.rs b/crates/pyrefly_build/src/lib.rs index a1a303260f..d53d6af18a 100644 --- a/crates/pyrefly_build/src/lib.rs +++ b/crates/pyrefly_build/src/lib.rs @@ -31,12 +31,13 @@ use std::path::PathBuf; use serde::Deserialize; use serde::Serialize; -use crate::buck::query::BxlArgs; - -pub mod buck; pub mod handle; -pub mod map_db; pub mod source_db; +pub use source_db::SourceDatabase; +mod query; + +use crate::query::buck::BxlArgs; +use crate::source_db::query_source_db::QuerySourceDatabase; #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case", tag = "type")] @@ -50,10 +51,7 @@ impl BuildSystem { config_root: PathBuf, ) -> Box { match &self { - Self::Buck(args) => Box::new(buck::bxl::BuckSourceDatabase::new( - config_root, - args.clone(), - )), + Self::Buck(args) => Box::new(QuerySourceDatabase::new(config_root, args.clone())), } } } diff --git a/crates/pyrefly_build/src/buck/query.rs b/crates/pyrefly_build/src/query/buck.rs similarity index 100% rename from crates/pyrefly_build/src/buck/query.rs rename to crates/pyrefly_build/src/query/buck.rs diff --git a/crates/pyrefly_build/src/buck/mod.rs b/crates/pyrefly_build/src/query/mod.rs similarity index 79% rename from crates/pyrefly_build/src/buck/mod.rs rename to crates/pyrefly_build/src/query/mod.rs index f32d66b604..6131d1c00b 100644 --- a/crates/pyrefly_build/src/buck/mod.rs +++ b/crates/pyrefly_build/src/query/mod.rs @@ -5,6 +5,4 @@ * LICENSE file in the root directory of this source tree. */ -pub mod buck_check; -pub mod bxl; -pub mod query; +pub mod buck; diff --git a/crates/pyrefly_build/src/buck/buck_check.rs b/crates/pyrefly_build/src/source_db/buck_check.rs similarity index 100% rename from crates/pyrefly_build/src/buck/buck_check.rs rename to crates/pyrefly_build/src/source_db/buck_check.rs diff --git a/crates/pyrefly_build/src/map_db.rs b/crates/pyrefly_build/src/source_db/map_db.rs similarity index 100% rename from crates/pyrefly_build/src/map_db.rs rename to crates/pyrefly_build/src/source_db/map_db.rs diff --git a/crates/pyrefly_build/src/source_db.rs b/crates/pyrefly_build/src/source_db/mod.rs similarity index 97% rename from crates/pyrefly_build/src/source_db.rs rename to crates/pyrefly_build/src/source_db/mod.rs index b72f206968..a4f4b98a1c 100644 --- a/crates/pyrefly_build/src/source_db.rs +++ b/crates/pyrefly_build/src/source_db/mod.rs @@ -24,6 +24,10 @@ use static_interner::Interner; use crate::handle::Handle; +pub mod buck_check; +pub mod map_db; +pub(crate) mod query_source_db; + // We're interning `Target`s, since they'll be duplicated all over the place, // and it would be nice to have something that implements `Copy`. // We choose Interning over `Arc`, since we want to make sure all `Target`s diff --git a/crates/pyrefly_build/src/buck/bxl.rs b/crates/pyrefly_build/src/source_db/query_source_db.rs similarity index 96% rename from crates/pyrefly_build/src/buck/bxl.rs rename to crates/pyrefly_build/src/source_db/query_source_db.rs index 8f9992ded7..55694b878c 100644 --- a/crates/pyrefly_build/src/buck/bxl.rs +++ b/crates/pyrefly_build/src/source_db/query_source_db.rs @@ -20,12 +20,12 @@ use starlark_map::small_set::SmallSet; use tracing::debug; use tracing::info; -use crate::buck::query::BxlArgs; -use crate::buck::query::Include; -use crate::buck::query::PythonLibraryManifest; -use crate::buck::query::TargetManifestDatabase; -use crate::buck::query::query_source_db; use crate::handle::Handle; +use crate::query::buck::BxlArgs; +use crate::query::buck::Include; +use crate::query::buck::PythonLibraryManifest; +use crate::query::buck::TargetManifestDatabase; +use crate::query::buck::query_source_db; use crate::source_db::SourceDatabase; use crate::source_db::Target; @@ -52,7 +52,7 @@ impl Inner { } #[derive(Debug)] -pub struct BuckSourceDatabase { +pub struct QuerySourceDatabase { inner: RwLock, /// The set of items the sourcedb has been queried for. Not all of the targets /// or files listed here will necessarily appear in the sourcedb, for example, @@ -64,9 +64,9 @@ pub struct BuckSourceDatabase { bxl_args: BxlArgs, } -impl BuckSourceDatabase { +impl QuerySourceDatabase { pub fn new(cwd: PathBuf, bxl_args: BxlArgs) -> Self { - BuckSourceDatabase { + QuerySourceDatabase { cwd, inner: RwLock::new(Inner::new()), includes: Mutex::new(SmallSet::new()), @@ -100,7 +100,7 @@ impl BuckSourceDatabase { } } -impl SourceDatabase for BuckSourceDatabase { +impl SourceDatabase for QuerySourceDatabase { fn modules_to_check(&self) -> Vec { // TODO(connernilsen): implement modules_to_check vec![] @@ -206,9 +206,9 @@ mod tests { use starlark_map::smallset; use super::*; - use crate::buck::query::TargetManifest; + use crate::query::buck::TargetManifest; - impl BuckSourceDatabase { + impl QuerySourceDatabase { fn from_target_manifest_db( raw_db: TargetManifestDatabase, files: &SmallSet, @@ -229,7 +229,7 @@ mod tests { } } - fn get_db() -> (BuckSourceDatabase, PathBuf) { + fn get_db() -> (QuerySourceDatabase, PathBuf) { let raw_db = TargetManifestDatabase::get_test_database(); let root = raw_db.root.to_path_buf(); let files = smallset! { @@ -238,7 +238,7 @@ mod tests { }; ( - BuckSourceDatabase::from_target_manifest_db(raw_db, &files), + QuerySourceDatabase::from_target_manifest_db(raw_db, &files), root, ) } diff --git a/pyrefly/lib/commands/buck_check.rs b/pyrefly/lib/commands/buck_check.rs index edff53bb13..3681202f13 100644 --- a/pyrefly/lib/commands/buck_check.rs +++ b/pyrefly/lib/commands/buck_check.rs @@ -13,8 +13,8 @@ use std::sync::Arc; use anyhow::Context as _; use clap::Parser; use dupe::Dupe; -use pyrefly_build::buck::buck_check::BuckCheckSourceDatabase; use pyrefly_build::source_db::SourceDatabase; +use pyrefly_build::source_db::buck_check::BuckCheckSourceDatabase; use pyrefly_python::sys_info::PythonPlatform; use pyrefly_python::sys_info::PythonVersion; use pyrefly_python::sys_info::SysInfo; diff --git a/pyrefly/lib/test/incremental.rs b/pyrefly/lib/test/incremental.rs index b3f5c83674..31d94af5f5 100644 --- a/pyrefly/lib/test/incremental.rs +++ b/pyrefly/lib/test/incremental.rs @@ -13,7 +13,7 @@ use std::sync::Arc; use dupe::Dupe; use pyrefly_build::handle::Handle; -use pyrefly_build::map_db::MapDatabase; +use pyrefly_build::source_db::map_db::MapDatabase; use pyrefly_python::module_name::ModuleName; use pyrefly_python::module_path::ModulePath; use pyrefly_python::sys_info::SysInfo; diff --git a/pyrefly/lib/test/state.rs b/pyrefly/lib/test/state.rs index c6a4731010..2c1e78d048 100644 --- a/pyrefly/lib/test/state.rs +++ b/pyrefly/lib/test/state.rs @@ -14,7 +14,7 @@ use std::time::Duration; use dupe::Dupe; use pyrefly_build::handle::Handle; -use pyrefly_build::map_db::MapDatabase; +use pyrefly_build::source_db::map_db::MapDatabase; use pyrefly_python::module_name::ModuleName; use pyrefly_python::module_path::ModulePath; use pyrefly_python::sys_info::PythonPlatform; diff --git a/pyrefly/lib/test/util.rs b/pyrefly/lib/test/util.rs index b1b5fc17aa..78dec65a62 100644 --- a/pyrefly/lib/test/util.rs +++ b/pyrefly/lib/test/util.rs @@ -18,7 +18,7 @@ use anstream::ColorChoice; use anyhow::anyhow; use dupe::Dupe; use pyrefly_build::handle::Handle; -use pyrefly_build::map_db::MapDatabase; +use pyrefly_build::source_db::map_db::MapDatabase; use pyrefly_config::error::ErrorDisplayConfig; use pyrefly_config::error_kind::ErrorKind; use pyrefly_config::error_kind::Severity; From bb9a8850320141b02c85d8ad017a0f574e06e5b7 Mon Sep 17 00:00:00 2001 From: Conner Nilsen Date: Fri, 3 Oct 2025 15:56:09 -0700 Subject: [PATCH 2/5] Pull generic query stuff out into query mod Differential Revision: D83881915 --- crates/pyrefly_build/src/query/buck.rs | 419 +---------------- crates/pyrefly_build/src/query/mod.rs | 420 ++++++++++++++++++ .../src/source_db/query_source_db.rs | 8 +- 3 files changed, 427 insertions(+), 420 deletions(-) diff --git a/crates/pyrefly_build/src/query/buck.rs b/crates/pyrefly_build/src/query/buck.rs index 7e3b7357de..cf5e7307bd 100644 --- a/crates/pyrefly_build/src/query/buck.rs +++ b/crates/pyrefly_build/src/query/buck.rs @@ -5,48 +5,20 @@ * LICENSE file in the root directory of this source tree. */ -use std::ffi::OsStr; use std::ffi::OsString; use std::fmt::Debug; use std::io::Write; use std::path::Path; -use std::path::PathBuf; use std::process::Command; use anyhow::Context as _; -use dupe::Dupe as _; -use pyrefly_python::module_name::ModuleName; -use pyrefly_python::sys_info::SysInfo; use serde::Deserialize; use serde::Serialize; use starlark_map::small_map::SmallMap; -use starlark_map::small_set::SmallSet; use tempfile::NamedTempFile; -use vec1::Vec1; -use crate::source_db::Target; - -/// An enum representing something that has been included by the build system, and -/// which the build system should query for when building the sourcedb. -#[derive(Debug, PartialEq, Eq, Hash)] -pub(crate) enum Include { - #[allow(unused)] - Target(Target), - Path(PathBuf), -} - -impl Include { - pub fn path(path: PathBuf) -> Self { - Self::Path(path) - } - - fn to_bxl_args(&self) -> impl Iterator { - match self { - Include::Target(target) => [OsStr::new("--target"), target.to_os_str()].into_iter(), - Include::Path(path) => [OsStr::new("--file"), path.as_os_str()].into_iter(), - } - } -} +use crate::query::Include; +use crate::query::TargetManifestDatabase; #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, Default)] #[serde(rename_all = "kebab-case")] @@ -67,11 +39,10 @@ pub(crate) fn query_source_db<'a>( root: cwd.to_path_buf(), }); } - let mut argfile = NamedTempFile::with_prefix("pyrefly_buck_query_") .with_context(|| "Failed to create temporary argfile for querying Buck".to_owned())?; let mut argfile_args = OsString::from("--"); - files.flat_map(Include::to_bxl_args).for_each(|arg| { + files.flat_map(Include::to_cli_arg).for_each(|arg| { argfile_args.push("\n"); argfile_args.push(arg); }); @@ -111,387 +82,3 @@ pub(crate) fn query_source_db<'a>( "Failed to construct valid `TargetManifestDatabase` from BXL query result".to_owned() }) } - -#[derive(Debug, PartialEq, Eq, Deserialize, Clone)] -pub(crate) struct PythonLibraryManifest { - pub deps: SmallSet, - pub srcs: SmallMap>, - #[serde(flatten)] - pub sys_info: SysInfo, - pub buildfile_path: PathBuf, -} - -impl PythonLibraryManifest { - fn replace_alias_deps(&mut self, aliases: &SmallMap) { - self.deps = self - .deps - .iter() - .map(|t| { - if let Some(replace) = aliases.get(t) { - replace.dupe() - } else { - t.dupe() - } - }) - .collect(); - } - - fn rewrite_relative_to_root(&mut self, root: &Path) { - self.srcs - .iter_mut() - .for_each(|(_, paths)| paths.iter_mut().for_each(|p| *p = root.join(&**p))); - self.buildfile_path = root.join(&self.buildfile_path); - } -} - -#[derive(Debug, PartialEq, Eq, Deserialize, Clone)] -#[serde(untagged)] -pub(crate) enum TargetManifest { - Library(PythonLibraryManifest), - Alias { alias: Target }, -} - -#[derive(Debug, PartialEq, Eq, Deserialize, Clone)] -pub(crate) struct TargetManifestDatabase { - db: SmallMap, - pub root: PathBuf, -} - -impl TargetManifestDatabase { - pub fn produce_map(self) -> SmallMap { - let mut result = SmallMap::new(); - let aliases: SmallMap = self - .db - .iter() - .filter_map(|(t, manifest)| match manifest { - TargetManifest::Alias { alias } => Some((t.dupe(), alias.dupe())), - _ => None, - }) - .collect(); - for (target, manifest) in self.db { - match manifest { - TargetManifest::Alias { .. } => continue, - TargetManifest::Library(mut lib) => { - lib.replace_alias_deps(&aliases); - lib.rewrite_relative_to_root(&self.root); - result.insert(target, lib); - } - } - } - result - } -} - -#[cfg(test)] -mod tests { - use pretty_assertions::assert_eq; - use pyrefly_python::sys_info::PythonPlatform; - use pyrefly_python::sys_info::PythonVersion; - use starlark_map::smallmap; - - use super::*; - - impl TargetManifestDatabase { - pub fn new(db: SmallMap, root: PathBuf) -> Self { - TargetManifestDatabase { db, root } - } - - /// This is a simplified sourcedb taken from the BXL output run on pyre/client/log/log.py. - /// We also add a few extra entries to model some of the behavior around multiple entries - /// (i.e. multiple file paths corresponding to a module path, multiple module paths in - /// different targets). - pub fn get_test_database() -> Self { - TargetManifestDatabase::new( - smallmap! { - Target::from_string("//colorama:py".to_owned()) => TargetManifest::lib( - &[ - ( - "colorama", - &[ - "colorama/__init__.py", - "colorama/__init__.pyi", - ] - ), - ], - &[], - "colorama/BUCK", - ), - Target::from_string("//colorama:colorama".to_owned()) => TargetManifest::alias( - "//colorama:py" - ), - Target::from_string("//click:py".to_owned()) => TargetManifest::lib( - &[ - ( - "click", - &[ - "click/__init__.pyi", - "click/__init__.py", - ], - ) - ], - &[ - "//colorama:colorama" - ], - "click/BUCK", - ), - Target::from_string("//click:click".to_owned()) => TargetManifest::alias( - "//click:py" - ), - Target::from_string("//pyre/client/log:log".to_owned()) => TargetManifest::lib( - &[ - ( - "pyre.client.log", - &[ - "pyre/client/log/__init__.py" - ] - ), - ( - "pyre.client.log.log", - &[ - "pyre/client/log/log.py", - "pyre/client/log/log.pyi", - ] - ), - ], - &[ - "//click:click" - ], - "pyre/client/log/BUCK" - ), - Target::from_string("//pyre/client/log:log2".to_owned()) => TargetManifest::lib( - &[ - ( - "log", - &[ - "pyre/client/log/__init__.py" - ] - ), - ( - "log.log", - &[ - "pyre/client/log/log.py", - "pyre/client/log/log.pyi", - ] - ) - ], - &[ - "//click:click" - ], - "pyre/client/log/BUCK" - ) - }, - PathBuf::from("/path/to/this/repository"), - ) - } - } - - fn map_srcs( - srcs: &[(&str, &[&str])], - prefix_paths: Option<&str>, - ) -> SmallMap> { - let prefix = prefix_paths.map(Path::new); - let map_path = |p| prefix.map_or_else(|| PathBuf::from(p), |prefix| prefix.join(p)); - srcs.iter() - .map(|(n, paths)| { - ( - ModuleName::from_str(n), - Vec1::try_from_vec(paths.iter().map(map_path).collect()).unwrap(), - ) - }) - .collect() - } - - fn map_deps(deps: &[&str]) -> SmallSet { - deps.iter() - .map(|s| Target::from_string((*s).to_owned())) - .collect() - } - - impl TargetManifest { - fn alias(target: &str) -> Self { - TargetManifest::Alias { - alias: Target::from_string(target.to_owned()), - } - } - - pub fn lib(srcs: &[(&str, &[&str])], deps: &[&str], buildfile: &str) -> Self { - TargetManifest::Library(PythonLibraryManifest { - srcs: map_srcs(srcs, None), - deps: map_deps(deps), - sys_info: SysInfo::new(PythonVersion::new(3, 12, 0), PythonPlatform::linux()), - buildfile_path: PathBuf::from(buildfile), - }) - } - } - - impl PythonLibraryManifest { - fn new(srcs: &[(&str, &[&str])], deps: &[&str], buildfile: &str) -> Self { - let root = "/path/to/this/repository"; - Self { - srcs: map_srcs(srcs, Some(root)), - deps: map_deps(deps), - sys_info: SysInfo::new(PythonVersion::new(3, 12, 0), PythonPlatform::linux()), - buildfile_path: PathBuf::from(root).join(buildfile), - } - } - } - - #[test] - fn example_json_parses() { - const EXAMPLE_JSON: &str = r#" -{ - "db": { - "//colorama:py": { - "srcs": { - "colorama": [ - "colorama/__init__.py", - "colorama/__init__.pyi" - ] - }, - "deps": [], - "buildfile_path": "colorama/BUCK", - "python_version": "3.12", - "python_platform": "linux" - }, - "//colorama:colorama": { - "alias": "//colorama:py" - }, - "//click:py": { - "srcs": { - "click": [ - "click/__init__.pyi", - "click/__init__.py" - ] - }, - "deps": [ - "//colorama:colorama" - ], - "buildfile_path": "click/BUCK", - "python_version": "3.12", - "python_platform": "linux" - }, - "//click:click": { - "alias": "//click:py" - }, - "//pyre/client/log:log": { - "srcs": { - "pyre.client.log": [ - "pyre/client/log/__init__.py" - ], - "pyre.client.log.log": [ - "pyre/client/log/log.py", - "pyre/client/log/log.pyi" - ] - }, - "deps": [ - "//click:click" - ], - "buildfile_path": "pyre/client/log/BUCK", - "python_version": "3.12", - "python_platform": "linux" - }, - "//pyre/client/log:log2": { - "srcs": { - "log": [ - "pyre/client/log/__init__.py" - ], - "log.log": [ - "pyre/client/log/log.py", - "pyre/client/log/log.pyi" - ] - }, - "deps": [ - "//click:click" - ], - "buildfile_path": "pyre/client/log/BUCK", - "python_version": "3.12", - "python_platform": "linux" - } - }, - "root": "/path/to/this/repository" -} - "#; - let parsed: TargetManifestDatabase = serde_json::from_str(EXAMPLE_JSON).unwrap(); - assert_eq!(parsed, TargetManifestDatabase::get_test_database()); - } - - #[test] - fn test_produce_db() { - let expected = smallmap! { - Target::from_string("//colorama:py".to_owned()) => PythonLibraryManifest::new( - &[ - ( - "colorama", - &[ - "colorama/__init__.py", - "colorama/__init__.pyi", - ] - ), - ], - &[], - "colorama/BUCK", - ), - Target::from_string("//click:py".to_owned()) => PythonLibraryManifest::new( - &[ - ( - "click", - &[ - "click/__init__.pyi", - "click/__init__.py", - ], - ) - ], - &[ - "//colorama:py" - ], - "click/BUCK", - ), - Target::from_string("//pyre/client/log:log".to_owned()) => PythonLibraryManifest::new( - &[ - ( - "pyre.client.log", - &[ - "pyre/client/log/__init__.py" - ] - ), - ( - "pyre.client.log.log", - &[ - "pyre/client/log/log.py", - "pyre/client/log/log.pyi", - ] - ), - ], - &[ - "//click:py" - ], - "pyre/client/log/BUCK", - ), - Target::from_string("//pyre/client/log:log2".to_owned()) => PythonLibraryManifest::new( - &[ - ( - "log", - &[ - "pyre/client/log/__init__.py" - ] - ), - ( - "log.log", - &[ - "pyre/client/log/log.py", - "pyre/client/log/log.pyi", - ] - ) - ], - &[ - "//click:py" - ], - "pyre/client/log/BUCK", - ) - }; - assert_eq!( - TargetManifestDatabase::get_test_database().produce_map(), - expected - ); - } -} diff --git a/crates/pyrefly_build/src/query/mod.rs b/crates/pyrefly_build/src/query/mod.rs index 6131d1c00b..52dec7cc07 100644 --- a/crates/pyrefly_build/src/query/mod.rs +++ b/crates/pyrefly_build/src/query/mod.rs @@ -5,4 +5,424 @@ * LICENSE file in the root directory of this source tree. */ +use std::ffi::OsStr; +use std::path::Path; +use std::path::PathBuf; + +use dupe::Dupe as _; +use pyrefly_python::module_name::ModuleName; +use pyrefly_python::sys_info::SysInfo; +use serde::Deserialize; +use starlark_map::small_map::SmallMap; +use starlark_map::small_set::SmallSet; +use vec1::Vec1; + +use crate::source_db::Target; + pub mod buck; + +/// An enum representing something that has been included by the build system, and +/// which the build system should query for when building the sourcedb. +#[derive(Debug, PartialEq, Eq, Hash)] +pub(crate) enum Include { + #[allow(unused)] + Target(Target), + Path(PathBuf), +} + +impl Include { + pub fn path(path: PathBuf) -> Self { + Self::Path(path) + } + + fn to_cli_arg(&self) -> impl Iterator { + match self { + Include::Target(target) => [OsStr::new("--target"), target.to_os_str()].into_iter(), + Include::Path(path) => [OsStr::new("--file"), path.as_os_str()].into_iter(), + } + } +} + +#[derive(Debug, PartialEq, Eq, Deserialize, Clone)] +pub(crate) struct PythonLibraryManifest { + pub deps: SmallSet, + pub srcs: SmallMap>, + #[serde(flatten)] + pub sys_info: SysInfo, + pub buildfile_path: PathBuf, +} + +impl PythonLibraryManifest { + fn replace_alias_deps(&mut self, aliases: &SmallMap) { + self.deps = self + .deps + .iter() + .map(|t| { + if let Some(replace) = aliases.get(t) { + replace.dupe() + } else { + t.dupe() + } + }) + .collect(); + } + + fn rewrite_relative_to_root(&mut self, root: &Path) { + self.srcs + .iter_mut() + .for_each(|(_, paths)| paths.iter_mut().for_each(|p| *p = root.join(&**p))); + self.buildfile_path = root.join(&self.buildfile_path); + } +} + +#[derive(Debug, PartialEq, Eq, Deserialize, Clone)] +#[serde(untagged)] +pub(crate) enum TargetManifest { + Library(PythonLibraryManifest), + Alias { alias: Target }, +} + +#[derive(Debug, PartialEq, Eq, Deserialize, Clone)] +pub(crate) struct TargetManifestDatabase { + db: SmallMap, + pub root: PathBuf, +} + +impl TargetManifestDatabase { + pub fn produce_map(self) -> SmallMap { + let mut result = SmallMap::new(); + let aliases: SmallMap = self + .db + .iter() + .filter_map(|(t, manifest)| match manifest { + TargetManifest::Alias { alias } => Some((t.dupe(), alias.dupe())), + _ => None, + }) + .collect(); + for (target, manifest) in self.db { + match manifest { + TargetManifest::Alias { .. } => continue, + TargetManifest::Library(mut lib) => { + lib.replace_alias_deps(&aliases); + lib.rewrite_relative_to_root(&self.root); + result.insert(target, lib); + } + } + } + result + } +} + +#[cfg(test)] +mod tests { + use pretty_assertions::assert_eq; + use pyrefly_python::sys_info::PythonPlatform; + use pyrefly_python::sys_info::PythonVersion; + use starlark_map::smallmap; + + use super::*; + + impl TargetManifestDatabase { + pub fn new(db: SmallMap, root: PathBuf) -> Self { + TargetManifestDatabase { db, root } + } + + /// This is a simplified sourcedb taken from the BXL output run on pyre/client/log/log.py. + /// We also add a few extra entries to model some of the behavior around multiple entries + /// (i.e. multiple file paths corresponding to a module path, multiple module paths in + /// different targets). + pub fn get_test_database() -> Self { + TargetManifestDatabase::new( + smallmap! { + Target::from_string("//colorama:py".to_owned()) => TargetManifest::lib( + &[ + ( + "colorama", + &[ + "colorama/__init__.py", + "colorama/__init__.pyi", + ] + ), + ], + &[], + "colorama/BUCK", + ), + Target::from_string("//colorama:colorama".to_owned()) => TargetManifest::alias( + "//colorama:py" + ), + Target::from_string("//click:py".to_owned()) => TargetManifest::lib( + &[ + ( + "click", + &[ + "click/__init__.pyi", + "click/__init__.py", + ], + ) + ], + &[ + "//colorama:colorama" + ], + "click/BUCK", + ), + Target::from_string("//click:click".to_owned()) => TargetManifest::alias( + "//click:py" + ), + Target::from_string("//pyre/client/log:log".to_owned()) => TargetManifest::lib( + &[ + ( + "pyre.client.log", + &[ + "pyre/client/log/__init__.py" + ] + ), + ( + "pyre.client.log.log", + &[ + "pyre/client/log/log.py", + "pyre/client/log/log.pyi", + ] + ), + ], + &[ + "//click:click" + ], + "pyre/client/log/BUCK" + ), + Target::from_string("//pyre/client/log:log2".to_owned()) => TargetManifest::lib( + &[ + ( + "log", + &[ + "pyre/client/log/__init__.py" + ] + ), + ( + "log.log", + &[ + "pyre/client/log/log.py", + "pyre/client/log/log.pyi", + ] + ) + ], + &[ + "//click:click" + ], + "pyre/client/log/BUCK" + ) + }, + PathBuf::from("/path/to/this/repository"), + ) + } + } + + fn map_srcs( + srcs: &[(&str, &[&str])], + prefix_paths: Option<&str>, + ) -> SmallMap> { + let prefix = prefix_paths.map(Path::new); + let map_path = |p| prefix.map_or_else(|| PathBuf::from(p), |prefix| prefix.join(p)); + srcs.iter() + .map(|(n, paths)| { + ( + ModuleName::from_str(n), + Vec1::try_from_vec(paths.iter().map(map_path).collect()).unwrap(), + ) + }) + .collect() + } + + fn map_deps(deps: &[&str]) -> SmallSet { + deps.iter() + .map(|s| Target::from_string((*s).to_owned())) + .collect() + } + + impl TargetManifest { + fn alias(target: &str) -> Self { + TargetManifest::Alias { + alias: Target::from_string(target.to_owned()), + } + } + + pub fn lib(srcs: &[(&str, &[&str])], deps: &[&str], buildfile: &str) -> Self { + TargetManifest::Library(PythonLibraryManifest { + srcs: map_srcs(srcs, None), + deps: map_deps(deps), + sys_info: SysInfo::new(PythonVersion::new(3, 12, 0), PythonPlatform::linux()), + buildfile_path: PathBuf::from(buildfile), + }) + } + } + + impl PythonLibraryManifest { + fn new(srcs: &[(&str, &[&str])], deps: &[&str], buildfile: &str) -> Self { + let root = "/path/to/this/repository"; + Self { + srcs: map_srcs(srcs, Some(root)), + deps: map_deps(deps), + sys_info: SysInfo::new(PythonVersion::new(3, 12, 0), PythonPlatform::linux()), + buildfile_path: PathBuf::from(root).join(buildfile), + } + } + } + + #[test] + fn example_json_parses() { + const EXAMPLE_JSON: &str = r#" +{ + "db": { + "//colorama:py": { + "srcs": { + "colorama": [ + "colorama/__init__.py", + "colorama/__init__.pyi" + ] + }, + "deps": [], + "buildfile_path": "colorama/BUCK", + "python_version": "3.12", + "python_platform": "linux" + }, + "//colorama:colorama": { + "alias": "//colorama:py" + }, + "//click:py": { + "srcs": { + "click": [ + "click/__init__.pyi", + "click/__init__.py" + ] + }, + "deps": [ + "//colorama:colorama" + ], + "buildfile_path": "click/BUCK", + "python_version": "3.12", + "python_platform": "linux" + }, + "//click:click": { + "alias": "//click:py" + }, + "//pyre/client/log:log": { + "srcs": { + "pyre.client.log": [ + "pyre/client/log/__init__.py" + ], + "pyre.client.log.log": [ + "pyre/client/log/log.py", + "pyre/client/log/log.pyi" + ] + }, + "deps": [ + "//click:click" + ], + "buildfile_path": "pyre/client/log/BUCK", + "python_version": "3.12", + "python_platform": "linux" + }, + "//pyre/client/log:log2": { + "srcs": { + "log": [ + "pyre/client/log/__init__.py" + ], + "log.log": [ + "pyre/client/log/log.py", + "pyre/client/log/log.pyi" + ] + }, + "deps": [ + "//click:click" + ], + "buildfile_path": "pyre/client/log/BUCK", + "python_version": "3.12", + "python_platform": "linux" + } + }, + "root": "/path/to/this/repository" +} + "#; + let parsed: TargetManifestDatabase = serde_json::from_str(EXAMPLE_JSON).unwrap(); + assert_eq!(parsed, TargetManifestDatabase::get_test_database()); + } + + #[test] + fn test_produce_db() { + let expected = smallmap! { + Target::from_string("//colorama:py".to_owned()) => PythonLibraryManifest::new( + &[ + ( + "colorama", + &[ + "colorama/__init__.py", + "colorama/__init__.pyi", + ] + ), + ], + &[], + "colorama/BUCK", + ), + Target::from_string("//click:py".to_owned()) => PythonLibraryManifest::new( + &[ + ( + "click", + &[ + "click/__init__.pyi", + "click/__init__.py", + ], + ) + ], + &[ + "//colorama:py" + ], + "click/BUCK", + ), + Target::from_string("//pyre/client/log:log".to_owned()) => PythonLibraryManifest::new( + &[ + ( + "pyre.client.log", + &[ + "pyre/client/log/__init__.py" + ] + ), + ( + "pyre.client.log.log", + &[ + "pyre/client/log/log.py", + "pyre/client/log/log.pyi", + ] + ), + ], + &[ + "//click:py" + ], + "pyre/client/log/BUCK", + ), + Target::from_string("//pyre/client/log:log2".to_owned()) => PythonLibraryManifest::new( + &[ + ( + "log", + &[ + "pyre/client/log/__init__.py" + ] + ), + ( + "log.log", + &[ + "pyre/client/log/log.py", + "pyre/client/log/log.pyi", + ] + ) + ], + &[ + "//click:py" + ], + "pyre/client/log/BUCK", + ) + }; + assert_eq!( + TargetManifestDatabase::get_test_database().produce_map(), + expected + ); + } +} diff --git a/crates/pyrefly_build/src/source_db/query_source_db.rs b/crates/pyrefly_build/src/source_db/query_source_db.rs index 55694b878c..267b2bb0db 100644 --- a/crates/pyrefly_build/src/source_db/query_source_db.rs +++ b/crates/pyrefly_build/src/source_db/query_source_db.rs @@ -21,10 +21,10 @@ use tracing::debug; use tracing::info; use crate::handle::Handle; +use crate::query::Include; +use crate::query::PythonLibraryManifest; +use crate::query::TargetManifestDatabase; use crate::query::buck::BxlArgs; -use crate::query::buck::Include; -use crate::query::buck::PythonLibraryManifest; -use crate::query::buck::TargetManifestDatabase; use crate::query::buck::query_source_db; use crate::source_db::SourceDatabase; use crate::source_db::Target; @@ -206,7 +206,7 @@ mod tests { use starlark_map::smallset; use super::*; - use crate::query::buck::TargetManifest; + use crate::query::TargetManifest; impl QuerySourceDatabase { fn from_target_manifest_db( From 8588adc1bf0338c9d7c6f98ea29f20fdfc508d00 Mon Sep 17 00:00:00 2001 From: Conner Nilsen Date: Fri, 3 Oct 2025 15:56:09 -0700 Subject: [PATCH 3/5] Make SourceDbQuerier trait Differential Revision: D83881916 --- crates/pyrefly_build/src/lib.rs | 7 +- crates/pyrefly_build/src/query/buck.rs | 111 ++++++++++-------- crates/pyrefly_build/src/query/mod.rs | 9 ++ .../src/source_db/query_source_db.rs | 27 +++-- 4 files changed, 98 insertions(+), 56 deletions(-) diff --git a/crates/pyrefly_build/src/lib.rs b/crates/pyrefly_build/src/lib.rs index d53d6af18a..0f276d4e97 100644 --- a/crates/pyrefly_build/src/lib.rs +++ b/crates/pyrefly_build/src/lib.rs @@ -27,6 +27,7 @@ #![feature(if_let_guard)] use std::path::PathBuf; +use std::sync::Arc; use serde::Deserialize; use serde::Serialize; @@ -37,6 +38,7 @@ pub use source_db::SourceDatabase; mod query; use crate::query::buck::BxlArgs; +use crate::query::buck::BxlQuerier; use crate::source_db::query_source_db::QuerySourceDatabase; #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] @@ -51,7 +53,10 @@ impl BuildSystem { config_root: PathBuf, ) -> Box { match &self { - Self::Buck(args) => Box::new(QuerySourceDatabase::new(config_root, args.clone())), + Self::Buck(args) => Box::new(QuerySourceDatabase::new( + config_root, + Arc::new(BxlQuerier::new(args.clone())), + )), } } } diff --git a/crates/pyrefly_build/src/query/buck.rs b/crates/pyrefly_build/src/query/buck.rs index cf5e7307bd..83be5bec41 100644 --- a/crates/pyrefly_build/src/query/buck.rs +++ b/crates/pyrefly_build/src/query/buck.rs @@ -15,9 +15,11 @@ use anyhow::Context as _; use serde::Deserialize; use serde::Serialize; use starlark_map::small_map::SmallMap; +use starlark_map::small_set::SmallSet; use tempfile::NamedTempFile; use crate::query::Include; +use crate::query::SourceDbQuerier; use crate::query::TargetManifestDatabase; #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, Default)] @@ -27,58 +29,71 @@ pub struct BxlArgs { extras: Option>, } -pub(crate) fn query_source_db<'a>( - files: impl Iterator, - cwd: &Path, - bxl_args: &BxlArgs, -) -> anyhow::Result { - let mut files = files.peekable(); - if files.peek().is_none() { - return Ok(TargetManifestDatabase { - db: SmallMap::new(), - root: cwd.to_path_buf(), - }); +#[derive(Debug)] +pub struct BxlQuerier(BxlArgs); + +impl BxlQuerier { + pub fn new(args: BxlArgs) -> Self { + Self(args) } - let mut argfile = NamedTempFile::with_prefix("pyrefly_buck_query_") - .with_context(|| "Failed to create temporary argfile for querying Buck".to_owned())?; - let mut argfile_args = OsString::from("--"); - files.flat_map(Include::to_cli_arg).for_each(|arg| { - argfile_args.push("\n"); - argfile_args.push(arg); - }); +} - argfile - .as_file_mut() - .write_all(argfile_args.as_encoded_bytes()) - .with_context(|| "Could not write to argfile when querying Buck".to_owned())?; +impl SourceDbQuerier for BxlQuerier { + fn query_source_db( + &self, + files: &SmallSet, + cwd: &Path, + ) -> anyhow::Result { + if files.is_empty() { + return Ok(TargetManifestDatabase { + db: SmallMap::new(), + root: cwd.to_path_buf(), + }); + } - let mut cmd = Command::new("buck2"); - if let Some(isolation_dir) = &bxl_args.isolation_dir { - cmd.arg("--isolation-dir"); - cmd.arg(isolation_dir); - } - cmd.arg("bxl"); - cmd.arg("--reuse-current-config"); - if let Some(metadata) = &bxl_args.extras { - cmd.args(metadata); - } - cmd.arg("prelude//python/sourcedb/pyrefly.bxl:main"); - cmd.arg(format!("@{}", argfile.path().display())); - cmd.current_dir(cwd); + let mut argfile = NamedTempFile::with_prefix("pyrefly_buck_query_") + .with_context(|| "Failed to create temporary argfile for querying Buck".to_owned())?; + let mut argfile_args = OsString::from("--"); + files.iter().flat_map(Include::to_cli_arg).for_each(|arg| { + argfile_args.push("\n"); + argfile_args.push(arg); + }); - let result = cmd.output()?; - if !result.status.success() { - let stdout = String::from_utf8(result.stdout) - .unwrap_or_else(|_| "".to_owned()); - let stderr = String::from_utf8(result.stderr) - .unwrap_or_else(|_| "".to_owned()); + argfile + .as_file_mut() + .write_all(argfile_args.as_encoded_bytes()) + .with_context(|| "Could not write to argfile when querying Buck".to_owned())?; - return Err(anyhow::anyhow!( - "Buck source db query failed...\nSTDOUT: {stdout}\nSTDERR: {stderr}" - )); - } + let mut cmd = Command::new("buck2"); + if let Some(isolation_dir) = &self.0.isolation_dir { + cmd.arg("--isolation-dir"); + cmd.arg(isolation_dir); + } + cmd.arg("bxl"); + cmd.arg("--reuse-current-config"); + if let Some(metadata) = &self.0.extras { + cmd.args(metadata); + } + cmd.arg("prelude//python/sourcedb/pyrefly.bxl:main"); + cmd.arg(format!("@{}", argfile.path().display())); + cmd.current_dir(cwd); + + let result = cmd.output()?; + if !result.status.success() { + let stdout = String::from_utf8(result.stdout).unwrap_or_else(|_| { + "".to_owned() + }); + let stderr = String::from_utf8(result.stderr).unwrap_or_else(|_| { + "".to_owned() + }); - serde_json::from_slice(&result.stdout).with_context(|| { - "Failed to construct valid `TargetManifestDatabase` from BXL query result".to_owned() - }) + return Err(anyhow::anyhow!( + "Buck source db query failed...\nSTDOUT: {stdout}\nSTDERR: {stderr}" + )); + } + + serde_json::from_slice(&result.stdout).with_context(|| { + "Failed to construct valid `TargetManifestDatabase` from BXL query result".to_owned() + }) + } } diff --git a/crates/pyrefly_build/src/query/mod.rs b/crates/pyrefly_build/src/query/mod.rs index 52dec7cc07..214605db67 100644 --- a/crates/pyrefly_build/src/query/mod.rs +++ b/crates/pyrefly_build/src/query/mod.rs @@ -6,6 +6,7 @@ */ use std::ffi::OsStr; +use std::fmt; use std::path::Path; use std::path::PathBuf; @@ -43,6 +44,14 @@ impl Include { } } +pub(crate) trait SourceDbQuerier: Send + Sync + fmt::Debug { + fn query_source_db( + &self, + files: &SmallSet, + cwd: &Path, + ) -> anyhow::Result; +} + #[derive(Debug, PartialEq, Eq, Deserialize, Clone)] pub(crate) struct PythonLibraryManifest { pub deps: SmallSet, diff --git a/crates/pyrefly_build/src/source_db/query_source_db.rs b/crates/pyrefly_build/src/source_db/query_source_db.rs index 267b2bb0db..a26403adf7 100644 --- a/crates/pyrefly_build/src/source_db/query_source_db.rs +++ b/crates/pyrefly_build/src/source_db/query_source_db.rs @@ -8,6 +8,7 @@ use std::collections::VecDeque; use std::path::Path; use std::path::PathBuf; +use std::sync::Arc; use dupe::Dupe as _; use pyrefly_python::module_name::ModuleName; @@ -23,9 +24,8 @@ use tracing::info; use crate::handle::Handle; use crate::query::Include; use crate::query::PythonLibraryManifest; +use crate::query::SourceDbQuerier; use crate::query::TargetManifestDatabase; -use crate::query::buck::BxlArgs; -use crate::query::buck::query_source_db; use crate::source_db::SourceDatabase; use crate::source_db::Target; @@ -61,16 +61,16 @@ pub struct QuerySourceDatabase { /// The directory that will be passed into the sourcedb query shell-out. Should /// be the same as the directory containing the config this sourcedb is a part of. cwd: PathBuf, - bxl_args: BxlArgs, + querier: Arc, } impl QuerySourceDatabase { - pub fn new(cwd: PathBuf, bxl_args: BxlArgs) -> Self { + pub fn new(cwd: PathBuf, querier: Arc) -> Self { QuerySourceDatabase { cwd, inner: RwLock::new(Inner::new()), includes: Mutex::new(SmallSet::new()), - bxl_args, + querier, } } @@ -176,7 +176,7 @@ impl SourceDatabase for QuerySourceDatabase { } *includes = new_includes; info!("Querying Buck for source DB"); - let raw_db = query_source_db(includes.iter(), &self.cwd, &self.bxl_args)?; + let raw_db = self.querier.query_source_db(&includes, &self.cwd)?; info!("Finished querying Buck for source DB"); Ok(self.update_with_target_manifest(raw_db)) } @@ -208,6 +208,19 @@ mod tests { use super::*; use crate::query::TargetManifest; + #[derive(Debug)] + struct DummyQuerier {} + + impl SourceDbQuerier for DummyQuerier { + fn query_source_db( + &self, + _: &SmallSet, + _: &Path, + ) -> anyhow::Result { + Ok(TargetManifestDatabase::get_test_database()) + } + } + impl QuerySourceDatabase { fn from_target_manifest_db( raw_db: TargetManifestDatabase, @@ -222,7 +235,7 @@ mod tests { .collect(), ), cwd: PathBuf::new(), - bxl_args: Default::default(), + querier: Arc::new(DummyQuerier {}), }; new.update_with_target_manifest(raw_db); new From 39134fb2baee619807121f4ffebd1876d68daeb1 Mon Sep 17 00:00:00 2001 From: Conner Nilsen Date: Mon, 6 Oct 2025 09:07:49 -0700 Subject: [PATCH 4/5] Pull out common command logic for sourcedb querying Differential Revision: D83881919 --- crates/pyrefly_build/src/query/buck.rs | 56 +----------------- crates/pyrefly_build/src/query/mod.rs | 58 ++++++++++++++++++- .../src/source_db/query_source_db.rs | 4 ++ 3 files changed, 62 insertions(+), 56 deletions(-) diff --git a/crates/pyrefly_build/src/query/buck.rs b/crates/pyrefly_build/src/query/buck.rs index 83be5bec41..b38231e3ba 100644 --- a/crates/pyrefly_build/src/query/buck.rs +++ b/crates/pyrefly_build/src/query/buck.rs @@ -5,22 +5,13 @@ * LICENSE file in the root directory of this source tree. */ -use std::ffi::OsString; use std::fmt::Debug; -use std::io::Write; -use std::path::Path; use std::process::Command; -use anyhow::Context as _; use serde::Deserialize; use serde::Serialize; -use starlark_map::small_map::SmallMap; -use starlark_map::small_set::SmallSet; -use tempfile::NamedTempFile; -use crate::query::Include; use crate::query::SourceDbQuerier; -use crate::query::TargetManifestDatabase; #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, Default)] #[serde(rename_all = "kebab-case")] @@ -39,31 +30,7 @@ impl BxlQuerier { } impl SourceDbQuerier for BxlQuerier { - fn query_source_db( - &self, - files: &SmallSet, - cwd: &Path, - ) -> anyhow::Result { - if files.is_empty() { - return Ok(TargetManifestDatabase { - db: SmallMap::new(), - root: cwd.to_path_buf(), - }); - } - - let mut argfile = NamedTempFile::with_prefix("pyrefly_buck_query_") - .with_context(|| "Failed to create temporary argfile for querying Buck".to_owned())?; - let mut argfile_args = OsString::from("--"); - files.iter().flat_map(Include::to_cli_arg).for_each(|arg| { - argfile_args.push("\n"); - argfile_args.push(arg); - }); - - argfile - .as_file_mut() - .write_all(argfile_args.as_encoded_bytes()) - .with_context(|| "Could not write to argfile when querying Buck".to_owned())?; - + fn construct_command(&self) -> Command { let mut cmd = Command::new("buck2"); if let Some(isolation_dir) = &self.0.isolation_dir { cmd.arg("--isolation-dir"); @@ -75,25 +42,6 @@ impl SourceDbQuerier for BxlQuerier { cmd.args(metadata); } cmd.arg("prelude//python/sourcedb/pyrefly.bxl:main"); - cmd.arg(format!("@{}", argfile.path().display())); - cmd.current_dir(cwd); - - let result = cmd.output()?; - if !result.status.success() { - let stdout = String::from_utf8(result.stdout).unwrap_or_else(|_| { - "".to_owned() - }); - let stderr = String::from_utf8(result.stderr).unwrap_or_else(|_| { - "".to_owned() - }); - - return Err(anyhow::anyhow!( - "Buck source db query failed...\nSTDOUT: {stdout}\nSTDERR: {stderr}" - )); - } - - serde_json::from_slice(&result.stdout).with_context(|| { - "Failed to construct valid `TargetManifestDatabase` from BXL query result".to_owned() - }) + cmd } } diff --git a/crates/pyrefly_build/src/query/mod.rs b/crates/pyrefly_build/src/query/mod.rs index 214605db67..1d6a709eeb 100644 --- a/crates/pyrefly_build/src/query/mod.rs +++ b/crates/pyrefly_build/src/query/mod.rs @@ -6,21 +6,27 @@ */ use std::ffi::OsStr; +use std::ffi::OsString; use std::fmt; +use std::io::Write as _; use std::path::Path; use std::path::PathBuf; +use std::process::Command; +use anyhow::Context as _; use dupe::Dupe as _; use pyrefly_python::module_name::ModuleName; use pyrefly_python::sys_info::SysInfo; use serde::Deserialize; use starlark_map::small_map::SmallMap; use starlark_map::small_set::SmallSet; +use tempfile::NamedTempFile; use vec1::Vec1; use crate::source_db::Target; pub mod buck; +pub mod custom; /// An enum representing something that has been included by the build system, and /// which the build system should query for when building the sourcedb. @@ -44,12 +50,60 @@ impl Include { } } -pub(crate) trait SourceDbQuerier: Send + Sync + fmt::Debug { +pub trait SourceDbQuerier: Send + Sync + fmt::Debug { fn query_source_db( &self, files: &SmallSet, cwd: &Path, - ) -> anyhow::Result; + ) -> anyhow::Result { + if files.is_empty() { + return Ok(TargetManifestDatabase { + db: SmallMap::new(), + root: cwd.to_path_buf(), + }); + } + + let mut argfile = + NamedTempFile::with_prefix("pyrefly_build_query_").with_context(|| { + "Failed to create temporary argfile for querying source DB".to_owned() + })?; + let mut argfile_args = OsString::from("--"); + files.iter().flat_map(Include::to_cli_arg).for_each(|arg| { + argfile_args.push("\n"); + argfile_args.push(arg); + }); + + argfile + .as_file_mut() + .write_all(argfile_args.as_encoded_bytes()) + .with_context(|| "Could not write to argfile when querying source DB".to_owned())?; + + let mut cmd = self.construct_command(); + cmd.arg(format!("@{}", argfile.path().display())); + cmd.current_dir(cwd); + + let result = cmd.output()?; + if !result.status.success() { + let stdout = String::from_utf8(result.stdout) + .unwrap_or_else(|_| "".to_owned()); + let stderr = String::from_utf8(result.stderr).unwrap_or_else(|_| { + "".to_owned() + }); + + return Err(anyhow::anyhow!( + "Source DB query failed...\nSTDOUT: {stdout}\nSTDERR: {stderr}" + )); + } + + serde_json::from_slice(&result.stdout).with_context(|| { + format!( + "Failed to construct valid `TargetManifestDatabase` from querier result. Command run: `{}`", + cmd.get_program().display(), + ) + }) + } + + fn construct_command(&self) -> Command; } #[derive(Debug, PartialEq, Eq, Deserialize, Clone)] diff --git a/crates/pyrefly_build/src/source_db/query_source_db.rs b/crates/pyrefly_build/src/source_db/query_source_db.rs index a26403adf7..6cdfcdb4bf 100644 --- a/crates/pyrefly_build/src/source_db/query_source_db.rs +++ b/crates/pyrefly_build/src/source_db/query_source_db.rs @@ -219,6 +219,10 @@ mod tests { ) -> anyhow::Result { Ok(TargetManifestDatabase::get_test_database()) } + + fn construct_command(&self) -> std::process::Command { + panic!("We shouldn't be calling this..."); + } } impl QuerySourceDatabase { From 349eb34ba8c170fc14a0a04ebf0037f7b1f67f90 Mon Sep 17 00:00:00 2001 From: Conner Nilsen Date: Mon, 6 Oct 2025 09:16:35 -0700 Subject: [PATCH 5/5] Create custom querier for sourcedb (#1243) Summary: This diff creates a `CustomQuerier` `SourceDbQuerier` implementation, which allows arbitrary commands to be run when querying a sourcedb. This takes in different options from a config file than the BXL querier, but is a more general version of it. Pull Request resolved: https://github.com/facebook/pyrefly/pull/1243 Differential Revision: D83881918 --- crates/pyrefly_build/src/lib.rs | 15 ++++--- crates/pyrefly_build/src/query/custom.rs | 52 ++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 crates/pyrefly_build/src/query/custom.rs diff --git a/crates/pyrefly_build/src/lib.rs b/crates/pyrefly_build/src/lib.rs index 0f276d4e97..f4fea65114 100644 --- a/crates/pyrefly_build/src/lib.rs +++ b/crates/pyrefly_build/src/lib.rs @@ -37,14 +37,18 @@ pub mod source_db; pub use source_db::SourceDatabase; mod query; +use crate::query::SourceDbQuerier; use crate::query::buck::BxlArgs; use crate::query::buck::BxlQuerier; +use crate::query::custom::CustomQuerier; +use crate::query::custom::CustomQueryArgs; use crate::source_db::query_source_db::QuerySourceDatabase; #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case", tag = "type")] pub enum BuildSystem { Buck(BxlArgs), + Custom(CustomQueryArgs), } impl BuildSystem { @@ -52,11 +56,10 @@ impl BuildSystem { &self, config_root: PathBuf, ) -> Box { - match &self { - Self::Buck(args) => Box::new(QuerySourceDatabase::new( - config_root, - Arc::new(BxlQuerier::new(args.clone())), - )), - } + let querier: Arc = match &self { + Self::Buck(args) => Arc::new(BxlQuerier::new(args.clone())), + Self::Custom(args) => Arc::new(CustomQuerier::new(args.clone())), + }; + Box::new(QuerySourceDatabase::new(config_root, querier)) } } diff --git a/crates/pyrefly_build/src/query/custom.rs b/crates/pyrefly_build/src/query/custom.rs new file mode 100644 index 0000000000..3980d33933 --- /dev/null +++ b/crates/pyrefly_build/src/query/custom.rs @@ -0,0 +1,52 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use std::fmt::Debug; +use std::process::Command; + +use serde::Deserialize; +use serde::Serialize; +use vec1::Vec1; + +use crate::query::SourceDbQuerier; + +/// Args and settings for querying a custom source DB. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, Default)] +#[serde(rename_all = "kebab-case")] +pub struct CustomQueryArgs { + /// The command to run, requires at least the command to execute. + /// Pyrefly will call this in the form ` @`, + /// where `` has the format + /// ``` + /// -- + /// + /// + /// ... + /// ``` + /// and `` is either `--file` or `--target`, depeding on the type + /// of `` + /// and `` is an absolute path to a file or a build system's target. + command: Vec1, +} + +/// A querier allowing for a custom command when querying and constructing source DB. +#[derive(Debug)] +pub struct CustomQuerier(CustomQueryArgs); + +impl CustomQuerier { + pub fn new(args: CustomQueryArgs) -> Self { + Self(args) + } +} + +impl SourceDbQuerier for CustomQuerier { + fn construct_command(&self) -> Command { + let mut cmd = Command::new(self.0.command.first()); + cmd.args(self.0.command.iter().skip(1)); + cmd + } +}