feat(core): select affected tasks instead of affected projects

astro-docs/src/content/docs/reference/environment-variables.mdoc

@@ -168,6 +168,7 @@ The following environment variables are ones that you can set to change the beha
 
 | Property                      | Type           | Description                                                                                                                                                                                                      |
 | ----------------------------- | -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `NX_AFFECTED_GRANULARITY`     | string         | When set to `task`, `nx affected` selects individual tasks whose inputs a change reaches, not every task of every affected project. Defaults to `project`.                                                       |
 | `NX_BAIL`                     | boolean        | If set to `true`, Nx will stop command execution after the first failed task. Can be overridden on the command line with `--nxBail`.                                                                             |
 | `NX_BASE`                     | string         | The default base branch to use when calculating the affected projects. Can be overridden on the command line with `--base`.                                                                                      |
 | `NX_BATCH_MODE`               | boolean        | If set to `true`, Nx will run task(s) in batches for executors which support batches.                                                                                                                            |

packages/nx/src/command-line/affected/affected.ts

@@ -20,7 +20,9 @@ import { TargetDependencyConfig } from '../../config/workspace-json-project-json
 import { readNxJson } from '../../config/configuration';
 import { findMatchingProjects } from '../../utils/find-matching-projects';
 import { generateGraph } from '../graph/graph';
-import { allFileData } from '../../utils/all-file-data';
+import { computeAffectedTasks } from '../../project-graph/affected/affected-tasks';
+import { resolveAffectedGranularity } from '../../project-graph/affected/granularity';
+import type { TaskSelection } from '../../tasks-runner/run-command';
 
 export async function affected(
   command: 'graph' | 'print-affected' | 'affected',
@@ -56,7 +58,54 @@ export async function affected(
   const projectGraph = await createProjectGraphAsync({
     exitOnError: true,
   });
-  const projects = await getAffectedGraphNodes(nxArgs, projectGraph);
+  const granularity = resolveAffectedGranularity();
+  // Task granularity needs a target to select against, so `nx graph --affected`
+  // and the deprecated print-affected stay project-grained.
+  const useTasks =
+    granularity === 'task' &&
+    command === 'affected' &&
+    !!nxArgs.targets?.length;
+
+  let taskSelection: TaskSelection | undefined;
+  let projects: ProjectGraphProjectNode[];
+  if (useTasks) {
+    const affectedTasks = await computeAffectedTasks({
+      projectGraph,
+      nxJson,
+      targets: nxArgs.targets,
+      touchedFiles: calculateFileChanges(parseFiles(nxArgs).files, nxArgs),
+      configuration: nxArgs.configuration,
+      overrides,
+      extraTargetDependencies,
+      excludeTaskDependencies: extraOptions.excludeTaskDependencies,
+    });
+    taskSelection = {
+      taskIds: [...affectedTasks.affectedTaskIds],
+      planningContext: affectedTasks.planningContext,
+    };
+    // --exclude is honoured in getAffectedGraphNodes, which this branch does
+    // not call. Dropping it would restart a project someone deliberately took
+    // out of the pipeline.
+    if (nxArgs.exclude?.length) {
+      const excluded = new Set(
+        findMatchingProjects(nxArgs.exclude, projectGraph.nodes)
+      );
+      taskSelection.taskIds = taskSelection.taskIds.filter(
+        (id) => !excluded.has(affectedTasks.taskGraph.tasks[id].target.project)
+      );
+    }
+
+    // runCommand still seeds the graph from projects; the prune is what narrows
+    // it back down to the selected tasks and their dependencies.
+    const owning = new Set(
+      taskSelection.taskIds.map(
+        (id) => affectedTasks.taskGraph.tasks[id].target.project
+      )
+    );
+    projects = [...owning].map((name) => projectGraph.nodes[name]);
+  } else {
+    projects = await getAffectedGraphNodes(nxArgs, projectGraph);
+  }
 
   try {
     switch (command) {
@@ -89,7 +138,8 @@ export async function affected(
             overrides,
             null,
             extraTargetDependencies,
-            extraOptions
+            extraOptions,
+            taskSelection
           );
           await output.drain();
           process.exit(status);

packages/nx/src/hasher/create-task-hasher.ts

@@ -1,6 +1,7 @@
 import { NxJsonConfiguration } from '../config/nx-json';
 import { ProjectGraph } from '../config/project-graph';
 import { daemonClient } from '../daemon/client/client';
+import type { TaskPlanningContext } from './task-planning-context';
 import type { IoSnapshots } from '../native';
 import { getFileMap } from '../project-graph/build-project-graph';
 import {
@@ -18,7 +19,8 @@ export function createTaskHasher(
   projectGraph: ProjectGraph,
   nxJson: NxJsonConfiguration,
   runnerOptions?: any,
-  ioSnapshots?: IoSnapshots
+  ioSnapshots?: IoSnapshots,
+  planningContext?: TaskPlanningContext
 ): TaskHasher {
   if (daemonClient.enabled()) {
     return new DaemonBasedTaskHasher(
@@ -33,7 +35,8 @@ export function createTaskHasher(
       nxJson,
       rustReferences,
       runnerOptions,
-      ioSnapshots
+      ioSnapshots,
+      planningContext
     );
   }
 }

packages/nx/src/hasher/native-task-hasher-impl.ts

@@ -10,9 +10,11 @@ import {
   ProjectGraph as NativeProjectGraph,
   NxWorkspaceFilesExternals,
   TaskHasher,
+  subsetHashPlans,
   transferProjectGraph,
 } from '../native';
 import { transformProjectGraphForRust } from '../native/transform-objects';
+import type { TaskPlanningContext } from './task-planning-context';
 import { getRootTsConfigPath } from '../plugins/js/utils/typescript';
 import { getTaskIOService } from '../tasks-runner/task-io-service';
 import { readJsonFile } from '../utils/fileutils';
@@ -22,6 +24,7 @@ import { PartialHash, TaskHasherImpl } from './task-hasher';
 export class NativeTaskHasherImpl implements TaskHasherImpl {
   hasher: TaskHasher;
   planner: HashPlanner;
+  private readonly planningContext?: TaskPlanningContext;
   projectGraphRef: ExternalObject<NativeProjectGraph>;
   allWorkspaceFilesRef: ExternalObject<FileData[]>;
   projectFileMapRef: ExternalObject<Record<string, FileData[]>>;
@@ -32,11 +35,14 @@ export class NativeTaskHasherImpl implements TaskHasherImpl {
     private readonly nxJson: NxJsonConfiguration,
     private readonly projectGraph: ProjectGraph,
     externals: NxWorkspaceFilesExternals,
-    options: { selectivelyHashTsConfig: boolean }
+    options: { selectivelyHashTsConfig: boolean },
+    planningContext?: TaskPlanningContext
   ) {
-    this.projectGraphRef = transferProjectGraph(
-      transformProjectGraphForRust(projectGraph)
-    );
+    // Reuses the marshal and planner memo when affected already built them for
+    // this graph; otherwise this is the only phase that needs them.
+    this.projectGraphRef =
+      planningContext?.projectGraphRef ??
+      transferProjectGraph(transformProjectGraphForRust(projectGraph));
 
     this.allWorkspaceFilesRef = externals.allWorkspaceFiles;
     this.projectFileMapRef = externals.projectFiles;
@@ -53,7 +59,9 @@ export class NativeTaskHasherImpl implements TaskHasherImpl {
       }
     }
 
-    this.planner = new HashPlanner(nxJson, this.projectGraphRef);
+    this.planner =
+      planningContext?.planner ?? new HashPlanner(nxJson, this.projectGraphRef);
+    this.planningContext = planningContext;
     this.hasher = new TaskHasher(
       workspaceRoot,
       this.projectGraphRef,
@@ -93,14 +101,26 @@ export class NativeTaskHasherImpl implements TaskHasherImpl {
     collectInputs?: boolean,
     ioSnapshots?: IoSnapshots
   ): Promise<PartialHash[]> {
-    const plans = this.planner.getPlansReference(
-      tasks.map((t) => t.id),
-      taskGraph,
-      ioSnapshots,
-      ioSnapshots
-        ? customHasherTaskIds(this.projectGraph, taskGraph)
-        : undefined
-    );
+    const taskIds = tasks.map((t) => t.id);
+    // Affected already planned a superset of these tasks. Reusing that answer
+    // skips a second pass over the same planner, which costs about as much as
+    // the first even with the subtree memo warm.
+    //
+    // Only without a snapshot bundle: those plans were built before the bundle
+    // was fetched, so they describe a different hashing configuration and would
+    // produce keys that no snapshot-backed run can match.
+    const plans =
+      (!ioSnapshots &&
+        this.planningContext?.plans &&
+        subsetHashPlans(this.planningContext.plans, taskIds)) ||
+      this.planner.getPlansReference(
+        taskIds,
+        taskGraph,
+        ioSnapshots,
+        ioSnapshots
+          ? customHasherTaskIds(this.projectGraph, taskGraph)
+          : undefined
+      );
     const shouldCollectInputs =
       collectInputs ?? getTaskIOService().hasTaskInputSubscribers();
     const hashes = this.hasher.hashPlans(

packages/nx/src/hasher/task-hasher.ts

@@ -10,6 +10,7 @@ import { hashArray } from './file-hasher';
 import { InputDefinition } from '../config/workspace-json-project-json';
 import { minimatch } from 'minimatch';
 import { NativeTaskHasherImpl } from './native-task-hasher-impl';
+import type { TaskPlanningContext } from './task-planning-context';
 import { workspaceRoot } from '../utils/workspace-root';
 import { HashInputs, IoSnapshots, NxWorkspaceFilesExternals } from '../native';
 import { getTaskIOService } from '../tasks-runner/task-io-service';
@@ -203,7 +204,8 @@ export class InProcessTaskHasher implements TaskHasher {
     private readonly nxJson: NxJsonConfiguration,
     private readonly externalRustReferences: NxWorkspaceFilesExternals | null,
     private readonly options: any,
-    private readonly ioSnapshots?: IoSnapshots
+    private readonly ioSnapshots?: IoSnapshots,
+    private readonly planningContext?: TaskPlanningContext
   ) {
     this.taskHasher = new NativeTaskHasherImpl(
       workspaceRoot,
@@ -212,7 +214,8 @@ export class InProcessTaskHasher implements TaskHasher {
       this.externalRustReferences,
       {
         selectivelyHashTsConfig: this.options?.selectivelyHashTsConfig ?? false,
-      }
+      },
+      this.planningContext
     );
   }
 

packages/nx/src/hasher/task-planning-context.ts

@@ -0,0 +1,45 @@
+import { NxJsonConfiguration } from '../config/nx-json';
+import { ProjectGraph } from '../config/project-graph';
+import {
+  ExternalObject,
+  HashPlanner,
+  HashInstruction,
+  ProjectGraph as NativeProjectGraph,
+} from '../native';
+import { marshalGraph } from '../project-graph/affected/marshal-graph';
+
+/**
+ * A marshalled graph and the planner built over it, passed between the phases
+ * of one command.
+ *
+ * Task-grained affected plans the candidate tasks to decide what is affected,
+ * and the hasher then plans the survivors. A planner carries `subtree_memo`,
+ * `instruction_pool` and `external_deps_mapped` across `getPlans` calls, so
+ * handing the same instance to both makes the second pass mostly memo hits.
+ *
+ * Threaded as an argument rather than held in a module-level cache, because
+ * `plans` below is per-command rather than per-graph: two runs over the same
+ * graph select different tasks. The marshalled graph is cached, in
+ * ../project-graph/affected/marshal-graph.
+ */
+export interface TaskPlanningContext {
+  projectGraphRef: ExternalObject<NativeProjectGraph>;
+  planner: HashPlanner;
+  /**
+   * Plans for the task set affected already walked. The hasher narrows these to
+   * the tasks it was given instead of planning them again; it falls back when
+   * they cannot answer, so this is an optimisation and never a contract.
+   */
+  plans?: ExternalObject<Record<string, Array<HashInstruction>>>;
+}
+
+export function createTaskPlanningContext(
+  projectGraph: ProjectGraph,
+  nxJson: NxJsonConfiguration
+): TaskPlanningContext {
+  const projectGraphRef = marshalGraph(projectGraph);
+  return {
+    projectGraphRef,
+    planner: new HashPlanner(nxJson as any, projectGraphRef),
+  };
+}