fix: Paginate all API endpoints, remove Mobile testdata

Author: TsekNet

assets/screenshot.png

@@ -5,15 +5,24 @@ Single-binary Go CLI. Parses fleet-gitops YAML, fetches current state from Fleet
 ## Layout
 
 ```
-cmd/fleet-plan/        Cobra commands (main.go, diff.go, version.go)
+cmd/fleet-plan/
+  main.go               Cobra root command, flag wiring, runDiff entrypoint
+  version.go            Version subcommand (set via ldflags)
+  cmd_test.go           CLI flag and command tests
 internal/
-  api/client.go        Read-only Fleet REST client
-  config/config.go     Auth resolution (flags > env > config file)
-  parser/parser.go     YAML parser for fleet-gitops repos
-  diff/differ.go       Semantic diff engine
-  output/              Renderers: terminal, json, markdown
-  testutil/            Shared test helpers
-testdata/              Realistic fleet-gitops fixture for tests
+  api/client.go         Read-only Fleet REST client (GET only, HTTPS enforced)
+  config/config.go      Auth resolution: flags > env vars > config file
+  parser/parser.go      YAML parser for fleet-gitops repos (path traversal protected)
+  diff/differ.go        Semantic diff engine with per-field change tracking
+  output/
+    terminal.go         ANSI-colored terminal renderer (truncation, diff context)
+    json.go             JSON renderer
+    markdown.go         Markdown renderer
+    gen_screenshot.go   Screenshot generator (//go:build ignore)
+  testutil/             Shared test helpers (TestdataRoot)
+testdata/               Realistic fleet-gitops fixture repo for tests
+assets/                 Logo and screenshot images
+docs/                   Architecture and API endpoint docs
 ```
 
 ## Data flow
@@ -24,7 +33,7 @@ flowchart LR
     C[Fleet API] -->|api.FetchAll| D[FleetState]
     B --> E[diff.Diff]
     D --> E
-    E --> F[DiffResult]
+    E --> F["[]DiffResult"]
     F --> G{--format}
     G -->|terminal| H[terminal.go]
     G -->|json| I[json.go]
@@ -33,13 +42,30 @@ flowchart LR
 
 ## API client
 
-`FetchAll` parallelizes all GET requests via `errgroup`. When `default.yml` has global sections, it also fetches `/config`, global policies, and global queries.
+`FetchAll` parallelizes all GET requests via `errgroup`. When `default.yml` has global sections, it also fetches `/config`, global policies, and global queries. HTTPS is enforced by default (`FLEET_PLAN_INSECURE=1` to override for local dev).
 
 See [api-endpoints.md](api-endpoints.md) for the full list.
 
+## Auth resolution
+
+Priority order (highest wins):
+
+1. `--url` / `--token` flags
+2. `FLEET_PLAN_URL` / `FLEET_PLAN_TOKEN` env vars
+3. Config file: `~/.config/fleet-plan.json` or `<repo>/.config/fleet-plan.json`
+
+Config file supports multiple contexts:
+
+```json
+{
+  "contexts": { "dev": { "url": "...", "token": "..." } },
+  "default_context": "dev"
+}
+```
+
 ## Parser
 
-Walks `teams/*.yml`, resolves `path:` references, produces `ParsedRepo`. Also parses `default.yml` for labels, `org_settings`, `agent_options`, `controls`, and global policies/queries.
+Walks `teams/*.yml`, resolves `path:` references, produces `ParsedRepo`. Also parses `default.yml` for labels, `org_settings`, `agent_options`, `controls`, and global policies/queries. All path references are validated against the repo root to prevent traversal.
 
 ## Diff engine
 
@@ -56,12 +82,32 @@ Compares `FleetState` (API) vs `ParsedRepo` (YAML). Produces `[]DiffResult` per
 | Profiles | PayloadDisplayName | add/delete only |
 | Labels | `name` (cross-ref) | valid/missing with host counts |
 
-Whitespace is normalized before comparison to avoid false positives from YAML vs API newline differences.
+Whitespace is normalized before comparison to avoid false positives from YAML vs API newline differences. Per-field diffs are stored in `ResourceChange.Fields` for both added and modified resources.
+
+## Output modes
+
+| Mode | Flag | Description |
+|------|------|-------------|
+| Terminal (default) | `--format terminal` | ANSI-colored, smart truncation (80 chars), diff context around changes, capped at 3 fields per resource |
+| Terminal verbose | `--verbose` | Full untruncated old/new values for all changed fields |
+| JSON | `--format json` | Machine-readable, all fields |
+| Markdown | `--format markdown` | For CI comments / MR descriptions |
+
+## Screenshot generation
+
+`gen_screenshot.go` (`//go:build ignore`) renders representative output from testdata fixtures for the README screenshot. Regenerate with:
+
+```bash
+go run ./internal/output/gen_screenshot.go > /tmp/raw.txt
+termshot --raw-read /tmp/raw.txt -f assets/screenshot.png -C 100
+```
+
+Requires a PTY wrapper (e.g., `script -qec`) for ANSI color output when piping.
 
 ## Tests
 
 ```
 go test -race ./...
 ```
 
-All packages have `_test.go`. Tests use `testdata/` as a shared fleet-gitops fixture. Table-driven throughout.
+All packages have `_test.go`. Tests use `testdata/` as a shared fleet-gitops fixture. Table-driven throughout. Coverage target: >= 75% (current: ~81%).

docs/architecture.md

@@ -263,40 +263,86 @@ func (c *Client) GetConfig(ctx context.Context) (map[string]any, error) {
 	return result, nil
 }
 
-// GetTeams fetches all teams.
+// GetTeams fetches all teams with pagination.
 func (c *Client) GetTeams(ctx context.Context) ([]Team, error) {
-	var resp teamsResponse
-	q := url.Values{"per_page": {"250"}}
-	if err := c.get(ctx, "/api/v1/fleet/teams", q, &resp); err != nil {
-		return nil, fmt.Errorf("fetching teams: %w", err)
+	var all []Team
+	page := 0
+	for {
+		q := url.Values{
+			"per_page": {"250"},
+			"page":     {strconv.Itoa(page)},
+		}
+		var resp teamsResponse
+		if err := c.get(ctx, "/api/v1/fleet/teams", q, &resp); err != nil {
+			return nil, fmt.Errorf("fetching teams: %w", err)
+		}
+		all = append(all, resp.Teams...)
+		if len(resp.Teams) < 250 {
+			break
+		}
+		page++
+		if page > 100 { // safety: max 25k teams
+			break
+		}
 	}
-	return resp.Teams, nil
+	return all, nil
 }
 
-// GetPolicies fetches policies for a team (0 = global).
+// GetPolicies fetches policies for a team (0 = global) with pagination.
 func (c *Client) GetPolicies(ctx context.Context, teamID uint) ([]Policy, error) {
-	path := "/api/v1/fleet/policies"
+	apiPath := "/api/v1/fleet/policies"
 	if teamID > 0 {
-		path = fmt.Sprintf("/api/v1/fleet/teams/%d/policies", teamID)
+		apiPath = fmt.Sprintf("/api/v1/fleet/teams/%d/policies", teamID)
 	}
-	var resp policiesResponse
-	if err := c.get(ctx, path, nil, &resp); err != nil {
-		return nil, fmt.Errorf("fetching policies (team %d): %w", teamID, err)
+	var all []Policy
+	page := 0
+	for {
+		q := url.Values{
+			"per_page": {"250"},
+			"page":     {strconv.Itoa(page)},
+		}
+		var resp policiesResponse
+		if err := c.get(ctx, apiPath, q, &resp); err != nil {
+			return nil, fmt.Errorf("fetching policies (team %d): %w", teamID, err)
+		}
+		all = append(all, resp.Policies...)
+		if len(resp.Policies) < 250 {
+			break
+		}
+		page++
+		if page > 100 { // safety: max 25k policies
+			break
+		}
 	}
-	return resp.Policies, nil
+	return all, nil
 }
 
-// GetQueries fetches queries, optionally filtered by team.
+// GetQueries fetches queries, optionally filtered by team, with pagination.
 func (c *Client) GetQueries(ctx context.Context, teamID uint) ([]Query, error) {
-	q := url.Values{"per_page": {"250"}}
-	if teamID > 0 {
-		q.Set("team_id", strconv.FormatUint(uint64(teamID), 10))
-	}
-	var resp queriesResponse
-	if err := c.get(ctx, "/api/v1/fleet/queries", q, &resp); err != nil {
-		return nil, fmt.Errorf("fetching queries (team %d): %w", teamID, err)
+	var all []Query
+	page := 0
+	for {
+		q := url.Values{
+			"per_page": {"250"},
+			"page":     {strconv.Itoa(page)},
+		}
+		if teamID > 0 {
+			q.Set("team_id", strconv.FormatUint(uint64(teamID), 10))
+		}
+		var resp queriesResponse
+		if err := c.get(ctx, "/api/v1/fleet/queries", q, &resp); err != nil {
+			return nil, fmt.Errorf("fetching queries (team %d): %w", teamID, err)
+		}
+		all = append(all, resp.Queries...)
+		if len(resp.Queries) < 250 {
+			break
+		}
+		page++
+		if page > 100 { // safety: max 25k queries
+			break
+		}
 	}
-	return resp.Queries, nil
+	return all, nil
 }
 
 // GetSoftware fetches managed (available_for_install) software titles for a team.
@@ -356,14 +402,29 @@ func (c *Client) GetFleetMaintainedApps(ctx context.Context) ([]FleetMaintainedA
 	return all, nil
 }
 
-// GetLabels fetches all labels.
+// GetLabels fetches all labels with pagination.
 func (c *Client) GetLabels(ctx context.Context) ([]Label, error) {
-	var resp labelsResponse
-	q := url.Values{"per_page": {"250"}}
-	if err := c.get(ctx, "/api/v1/fleet/labels", q, &resp); err != nil {
-		return nil, fmt.Errorf("fetching labels: %w", err)
+	var all []Label
+	page := 0
+	for {
+		q := url.Values{
+			"per_page": {"250"},
+			"page":     {strconv.Itoa(page)},
+		}
+		var resp labelsResponse
+		if err := c.get(ctx, "/api/v1/fleet/labels", q, &resp); err != nil {
+			return nil, fmt.Errorf("fetching labels: %w", err)
+		}
+		all = append(all, resp.Labels...)
+		if len(resp.Labels) < 250 {
+			break
+		}
+		page++
+		if page > 100 { // safety: max 25k labels
+			break
+		}
 	}
-	return resp.Labels, nil
+	return all, nil
 }
 
 // GetProfiles fetches MDM profiles for a team.

internal/api/client.go

@@ -4,6 +4,7 @@
 package diff
 
 import (
+	"encoding/json"
 	"fmt"
 	"regexp"
 	"sort"
@@ -479,7 +480,7 @@ func diffSoftware(current api.TeamSoftware, proposed parser.ParsedSoftware) Reso
 }
 
 func normalizeSoftwarePath(s string) string {
-	s = strings.TrimSpace(strings.ToLower(s))
+	s = strings.TrimSpace(s)
 	if s == "" {
 		return ""
 	}
@@ -695,17 +696,6 @@ func validateLabels(team parser.ParsedTeam, labelMap map[string]api.Label, chang
 
 // ---------- Global config diffing ----------
 
-// serverOnlyKeys are keys returned by the Fleet API that don't exist in YAML.
-// Skip these during comparison to avoid false positives.
-var serverOnlyKeys = map[string]bool{
-	"license":          true,
-	"logging":          true,
-	"update_interval":  true,
-	"vulnerabilities":  true,
-	"sandbox_enabled":  true,
-	"server_settings":  true, // server_url is set by Fleet, not YAML
-}
-
 // diffConfig compares the current Fleet config (from API) against proposed
 // global config sections from default.yml. Returns a list of config changes.
 // Skips values containing "$" (env var placeholders that Fleet substitutes).
@@ -739,8 +729,12 @@ func diffConfig(apiConfig map[string]any, proposed *parser.ParsedGlobal) []Confi
 				}
 			}
 		case "controls":
-			// controls fields are spread across the top level (mdm, etc.)
-			apiSection = apiConfig
+			// controls fields map to the "mdm" section in the API response
+			if v, ok := apiConfig["mdm"]; ok {
+				if m, ok := v.(map[string]any); ok {
+					apiSection = m
+				}
+			}
 		}
 
 		if apiSection == nil {
@@ -783,16 +777,25 @@ func containsEnvVar(s string) bool {
 }
 
 // flattenMap recursively flattens a nested map into dot-separated key paths.
-// Calls fn(key, value) for each leaf value.
+// Calls fn(key, value) for each leaf value. Slices are serialized to JSON
+// for stable, order-independent comparison.
 func flattenMap(m map[string]any, prefix string, fn func(key, val string)) {
 	for k, v := range m {
 		fullKey := k
 		if prefix != "" {
 			fullKey = prefix + "." + k
 		}
-		if nested, ok := v.(map[string]any); ok {
-			flattenMap(nested, fullKey, fn)
-		} else {
+		switch val := v.(type) {
+		case map[string]any:
+			flattenMap(val, fullKey, fn)
+		case []any:
+			b, err := json.Marshal(val)
+			if err != nil {
+				fn(fullKey, fmt.Sprint(val))
+			} else {
+				fn(fullKey, string(b))
+			}
+		default:
 			fn(fullKey, fmt.Sprint(v))
 		}
 	}

internal/diff/differ.go

@@ -22,7 +22,6 @@ import (
 //   - Queries: Uptime (interval changed → modified)
 //   - No policies (SSH Root Login is new → added)
 //
-// Mobile team: not in API → new team info message.
 // Labels: macOS 14+ and Windows 11 exist. Ubuntu 24.04 does NOT → missing label error.
 func TestDiffTestdataAgainstMockAPI(t *testing.T) {
 	root := testutil.TestdataRoot(t)
@@ -84,10 +83,10 @@ func TestDiffTestdataAgainstMockAPI(t *testing.T) {
 	// --- Test all teams (unfiltered) ---
 	allResults := Diff(current, proposed, "")
 
-	// Should have 4 results: (global), Workstations, Servers, Mobile
+	// Should have 3 results: (global), Workstations, Servers
 	// The (global) result comes from default.yml parsing.
-	if len(allResults) != 4 {
-		t.Fatalf("expected 4 results, got %d", len(allResults))
+	if len(allResults) != 3 {
+		t.Fatalf("expected 3 results, got %d", len(allResults))
 	}
 
 	// Verify global result exists and is first
@@ -98,9 +97,9 @@ func TestDiffTestdataAgainstMockAPI(t *testing.T) {
 	// --- Workstations ---
 	ws := findTeam(t, allResults, "Workstations")
 
-	// Policies: Gatekeeper, Defender, SSH, Firewall are new (4 added)
-	if len(ws.Policies.Added) != 4 {
-		t.Errorf("Workstations: expected 4 added policies, got %d: %v", len(ws.Policies.Added), ws.Policies.Added)
+	// Policies: Defender, SSH, Firewall are new (3 added)
+	if len(ws.Policies.Added) != 3 {
+		t.Errorf("Workstations: expected 3 added policies, got %d: %v", len(ws.Policies.Added), ws.Policies.Added)
 	}
 	// FileVault modified (query changed)
 	if len(ws.Policies.Modified) != 1 {
@@ -184,28 +183,6 @@ func TestDiffTestdataAgainstMockAPI(t *testing.T) {
 		}
 	}
 
-	// --- Mobile (new team) ---
-	mob := findTeam(t, allResults, "Mobile")
-
-	// All resources should be "added" since team is new
-	if len(mob.Policies.Added) != 1 {
-		t.Errorf("Mobile: expected 1 added policy, got %d", len(mob.Policies.Added))
-	}
-	if len(mob.Queries.Added) != 1 {
-		t.Errorf("Mobile: expected 1 added query, got %d", len(mob.Queries.Added))
-	}
-
-	// Should have info message about new team
-	foundNewTeamInfo := false
-	for _, e := range mob.Errors {
-		if strings.Contains(e, "does not exist in Fleet yet") {
-			foundNewTeamInfo = true
-			break
-		}
-	}
-	if !foundNewTeamInfo {
-		t.Error("expected info message about new team for Mobile")
-	}
 }
 
 // TestDiffTestdataWorkstationsOnly verifies filtered diff for a single team.