Detect setuptools/hatch/flit and infer flat-layout source dirs (#69)

Author: andrew

detect/detect.go

@@ -165,7 +165,7 @@ func (e *Engine) Run() (*brief.Report, error) {
 	e.detectSelf(abs, report)
 
 	report.Style = e.detectStyle()
-	report.Layout = e.detectLayout()
+	report.Layout = e.detectLayout(report.Languages)
 	report.Platforms = e.detectPlatforms()
 
 	// Run slow detections concurrently.
@@ -973,7 +973,7 @@ func (e *Engine) inferStyle() *brief.StyleInfo {
 }
 
 // detectLayout checks for source and test directory patterns from the knowledge base.
-func (e *Engine) detectLayout() *brief.LayoutInfo {
+func (e *Engine) detectLayout(languages []brief.Detection) *brief.LayoutInfo {
 	if e.KB.Layouts == nil {
 		return nil
 	}
@@ -992,13 +992,98 @@ func (e *Engine) detectLayout() *brief.LayoutInfo {
 		}
 	}
 
+	if len(layout.SourceDirs) == 0 {
+		layout.SourceDirs = e.inferFlatLayout(languages, layout.TestDirs)
+	}
+
 	if len(layout.SourceDirs) == 0 && len(layout.TestDirs) == 0 {
 		return nil
 	}
 
 	return layout
 }
 
+// inferFlatLayout finds top-level directories that hold source for the primary
+// detected language when no conventional source directory (src/, lib/, etc.)
+// exists. This covers projects where the package directory is named after the
+// project rather than a generic name.
+func (e *Engine) inferFlatLayout(languages []brief.Detection, testDirs []string) []string {
+	if len(languages) == 0 {
+		return nil
+	}
+
+	exts := e.languageExtensions(languages[0].Name)
+	if len(exts) == 0 {
+		return nil
+	}
+
+	skip := make(map[string]bool)
+	for _, d := range e.KB.Layouts.Layout.ExcludeDirs {
+		skip[d] = true
+	}
+	for _, d := range e.KB.Layouts.Layout.TestDirs {
+		skip[d] = true
+	}
+	for _, d := range testDirs {
+		skip[d] = true
+	}
+
+	entries, err := os.ReadDir(e.Root)
+	if err != nil {
+		return nil
+	}
+
+	var found []string
+	for _, ent := range entries {
+		if !ent.IsDir() {
+			continue
+		}
+		name := ent.Name()
+		if e.shouldSkipDir(name) || skip[name] {
+			continue
+		}
+		if e.dirHasExtension(name, exts) {
+			found = append(found, name)
+		}
+	}
+	return found
+}
+
+// languageExtensions returns the file extensions a language tool definition
+// matches on, derived from its "*.ext" detection patterns.
+func (e *Engine) languageExtensions(name string) []string {
+	tool := e.KB.ByName[name]
+	if tool == nil {
+		return nil
+	}
+	seen := make(map[string]bool)
+	var exts []string
+	for _, pattern := range tool.Detect.Files {
+		if idx := strings.LastIndex(pattern, "*."); idx >= 0 {
+			ext := pattern[idx+1:]
+			if !seen[ext] {
+				seen[ext] = true
+				exts = append(exts, ext)
+			}
+		}
+	}
+	return exts
+}
+
+// dirHasExtension reports whether dir directly contains a file with one of the
+// given extensions.
+func (e *Engine) dirHasExtension(dir string, exts []string) bool {
+	for _, name := range e.dirFiles(dir) {
+		ext := filepath.Ext(name)
+		for _, want := range exts {
+			if ext == want {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // detectResources checks for project resource files defined in the knowledge base.
 func (e *Engine) detectResources() *brief.ResourceInfo {
 	if len(e.KB.Resources) == 0 {

detect/detect_test.go

@@ -374,6 +374,131 @@ func TestPythonProject(t *testing.T) {
 	}
 }
 
+func writeProjectFile(t *testing.T, dir, path, content string) {
+	t.Helper()
+	full := filepath.Join(dir, path)
+	if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(full, []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func runOn(t *testing.T, dir string) *brief.Report {
+	t.Helper()
+	r, err := New(loadKB(t), dir).Run()
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	return r
+}
+
+func packageManagerNames(r *brief.Report) []string {
+	names := make([]string, 0, len(r.PackageManagers))
+	for _, pm := range r.PackageManagers {
+		names = append(names, pm.Name)
+	}
+	return names
+}
+
+func TestPythonPackageManagerManifests(t *testing.T) {
+	cases := []struct {
+		name    string
+		file    string
+		content string
+		want    string
+	}{
+		{"setup.py", "setup.py", "from setuptools import setup\nsetup()\n", "setuptools"},
+		{"setup.cfg", "setup.cfg", "[metadata]\nname = x\n", "setuptools"},
+		{"pyproject setuptools", "pyproject.toml", "[build-system]\nrequires = [\"setuptools\"]\n", "setuptools"},
+		{"pyproject hatchling", "pyproject.toml", "[build-system]\nbuild-backend = \"hatchling.build\"\n", "Hatch"},
+		{"pyproject flit", "pyproject.toml", "[build-system]\nrequires = [\"flit_core\"]\n", "Flit"},
+		{"Pipfile", "Pipfile", "[packages]\n", "Pipenv"},
+		{"requirements.txt", "requirements.txt", "requests\n", "pip"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			dir := t.TempDir()
+			writeProjectFile(t, dir, "main.py", "")
+			writeProjectFile(t, dir, tc.file, tc.content)
+			r := runOn(t, dir)
+			got := packageManagerNames(r)
+			found := false
+			for _, n := range got {
+				if n == tc.want {
+					found = true
+				}
+			}
+			if !found {
+				t.Errorf("manifest %s: want %q in package managers, got %v", tc.file, tc.want, got)
+			}
+		})
+	}
+}
+
+func TestPythonFlatLayout(t *testing.T) {
+	dir := t.TempDir()
+	writeProjectFile(t, dir, "setup.py", "")
+	writeProjectFile(t, dir, "mypkg/__init__.py", "")
+	writeProjectFile(t, dir, "mypkg/core.py", "")
+	writeProjectFile(t, dir, "tests/test_core.py", "")
+	writeProjectFile(t, dir, "docs/index.md", "")
+	writeProjectFile(t, dir, "scripts/release.py", "")
+
+	r := runOn(t, dir)
+	if r.Layout == nil {
+		t.Fatal("expected layout info")
+	}
+	if got := r.Layout.SourceDirs; len(got) != 1 || got[0] != "mypkg" {
+		t.Errorf("source_dirs = %v, want [mypkg]", got)
+	}
+	for _, d := range r.Layout.SourceDirs {
+		if d == "docs" || d == "scripts" || d == "tests" {
+			t.Errorf("source_dirs should not include %q", d)
+		}
+	}
+	foundTests := false
+	for _, d := range r.Layout.TestDirs {
+		if d == "tests" {
+			foundTests = true
+		}
+	}
+	if !foundTests {
+		t.Error("expected tests/ in test_dirs")
+	}
+}
+
+func TestFlatLayoutSkippedWhenSrcExists(t *testing.T) {
+	dir := t.TempDir()
+	writeProjectFile(t, dir, "src/mypkg/__init__.py", "")
+	writeProjectFile(t, dir, "helper/tool.py", "")
+
+	r := runOn(t, dir)
+	if r.Layout == nil {
+		t.Fatal("expected layout info")
+	}
+	if got := r.Layout.SourceDirs; len(got) != 1 || got[0] != "src" {
+		t.Errorf("source_dirs = %v, want [src]", got)
+	}
+}
+
+func TestFlatLayoutNonPython(t *testing.T) {
+	dir := t.TempDir()
+	writeProjectFile(t, dir, "Gemfile", "source 'https://rubygems.org'\n")
+	writeProjectFile(t, dir, "mygem/version.rb", "VERSION = '1.0'\n")
+	writeProjectFile(t, dir, "examples/demo.rb", "")
+	writeProjectFile(t, dir, "spec/mygem_spec.rb", "")
+
+	r := runOn(t, dir)
+	if r.Layout == nil {
+		t.Fatal("expected layout info")
+	}
+	if got := r.Layout.SourceDirs; len(got) != 1 || got[0] != "mygem" {
+		t.Errorf("source_dirs = %v, want [mygem]", got)
+	}
+}
+
 func TestEmptyProject(t *testing.T) {
 	engine := New(loadKB(t), "../testdata/empty-project")
 	r, err := engine.Run()

kb/kb.go

@@ -161,8 +161,9 @@ type LayoutDef struct {
 
 // LayoutRules holds the layout detection rules.
 type LayoutRules struct {
-	SourceDirs []string `toml:"source_dirs"` // directories that indicate source
-	TestDirs   []string `toml:"test_dirs"`   // directories that indicate tests
+	SourceDirs  []string `toml:"source_dirs"`  // directories that indicate source
+	TestDirs    []string `toml:"test_dirs"`    // directories that indicate tests
+	ExcludeDirs []string `toml:"exclude_dirs"` // directories to skip during flat-layout inference
 }
 
 // StyleConfigDef defines style configuration files to check.

knowledge/_shared/_layout.toml

@@ -1,3 +1,13 @@
 [layout]
 source_dirs = ["src", "lib", "app", "pkg", "internal", "cmd"]
 test_dirs = ["test", "tests", "spec", "__tests__", "t"]
+exclude_dirs = [
+  "docs", "doc", "documentation",
+  "examples", "example", "samples", "sample", "demo", "demos",
+  "scripts", "script", "bin", "tools", "tool", "utils", "util",
+  "dev", "ci", "etc", "config", "configs", "conf",
+  "assets", "static", "public", "resources", "res",
+  "fixtures", "stubs", "mocks", "benchmark", "benchmarks", "bench",
+  "man", "share", "data", "migrations", "db",
+  "requires", "requirements", "deps",
+]

knowledge/python/flit.toml

@@ -0,0 +1,23 @@
+[tool]
+name = "Flit"
+category = "package_manager"
+homepage = "https://flit.pypa.io"
+docs = "https://flit.pypa.io/en/stable/"
+repo = "https://github.com/pypa/flit"
+description = "Simple Python build backend for pure-Python packages"
+
+[detect]
+[detect.file_contains]
+"pyproject.toml" = ["[tool.flit", "flit_core"]
+
+ecosystems = ["python"]
+
+[commands]
+run = "flit install"
+alternatives = ["pip install -e ."]
+
+[config]
+files = ["pyproject.toml"]
+
+[taxonomy]
+role = ["package-manager"]

knowledge/python/hatch.toml

@@ -0,0 +1,24 @@
+[tool]
+name = "Hatch"
+category = "package_manager"
+homepage = "https://hatch.pypa.io"
+docs = "https://hatch.pypa.io/latest/"
+repo = "https://github.com/pypa/hatch"
+description = "Modern Python project manager and build backend"
+
+[detect]
+files = ["hatch.toml"]
+[detect.file_contains]
+"pyproject.toml" = ["[tool.hatch", "hatchling"]
+
+ecosystems = ["python"]
+
+[commands]
+run = "hatch env create"
+alternatives = ["pip install -e ."]
+
+[config]
+files = ["pyproject.toml", "hatch.toml"]
+
+[taxonomy]
+role = ["package-manager"]

knowledge/python/setuptools.toml

@@ -0,0 +1,24 @@
+[tool]
+name = "setuptools"
+category = "package_manager"
+homepage = "https://setuptools.pypa.io"
+docs = "https://setuptools.pypa.io/en/latest/"
+repo = "https://github.com/pypa/setuptools"
+description = "Python build backend for setup.py/setup.cfg and PEP 517 projects"
+
+[detect]
+files = ["setup.py", "setup.cfg"]
+[detect.file_contains]
+"pyproject.toml" = ["setuptools"]
+
+ecosystems = ["python"]
+
+[commands]
+run = "pip install -e ."
+alternatives = ["pip install ."]
+
+[config]
+files = ["setup.py", "setup.cfg", "pyproject.toml"]
+
+[taxonomy]
+role = ["package-manager"]