Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,25 @@ class DAGRunStateCountsResponse(BaseModel):
state_counts: dict[DagRunState, int]


class DAGLatestRunTaskInstanceStateCountsResponse(BaseModel):
"""
Task-instance state counts for a Dag's latest run.

``state_counts`` only carries states present in the run; task instances without a
state yet are keyed as ``no_status``.
"""

dag_id: str
run_id: str
state_counts: dict[str, int]


class DAGsLatestRunTaskInstanceStateCountsCollectionResponse(BaseModel):
"""Collection of per-Dag latest-run task-instance state counts for the Dag list page."""

dags: list[DAGLatestRunTaskInstanceStateCountsResponse]


class DAGsRunStateCountsCollectionResponse(BaseModel):
"""Collection of per-Dag DagRun-state counts for the Dag list page."""

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -880,6 +880,52 @@ paths:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/ui/dags/latest_run_task_instance_state_counts:
get:
tags:
- DAG
summary: Get Latest Run Task Instance State Counts
description: 'Return task-instance state counts for the given Dag runs, for
the Dag list page.


The Dag list response already carries the latest run of each Dag, so the caller
passes

those run ids straight in. Deriving the latest run again here would mean an

``ORDER BY run_after DESC LIMIT 1`` per Dag, which has no supporting index
and degrades

badly once a Dag has many runs. Runs the caller may not read are dropped.'
operationId: get_latest_run_task_instance_state_counts_ui
security:
- OAuth2PasswordBearer: []
- HTTPBearer: []
parameters:
- name: dag_run_ids
in: query
required: true
schema:
type: array
items:
type: integer
minItems: 1
maxItems: 100
title: Dag Run Ids
responses:
'200':
description: Successful Response
content:
application/json:
schema:
$ref: '#/components/schemas/DAGsLatestRunTaskInstanceStateCountsCollectionResponse'
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/ui/dependencies:
get:
tags:
Expand Down Expand Up @@ -2750,6 +2796,32 @@ components:
that

the API server/Web UI can use this data to render connection form UI.'
DAGLatestRunTaskInstanceStateCountsResponse:
properties:
dag_id:
type: string
title: Dag Id
run_id:
type: string
title: Run Id
state_counts:
additionalProperties:
type: integer
type: object
title: State Counts
type: object
required:
- dag_id
- run_id
- state_counts
title: DAGLatestRunTaskInstanceStateCountsResponse
description: 'Task-instance state counts for a Dag''s latest run.


``state_counts`` only carries states present in the run; task instances without
a

state yet are keyed as ``no_status``.'
DAGRunLightResponse:
properties:
id:
Expand Down Expand Up @@ -3066,6 +3138,19 @@ components:
- file_token
title: DAGWithLatestDagRunsResponse
description: DAG with latest dag runs response serializer.
DAGsLatestRunTaskInstanceStateCountsCollectionResponse:
properties:
dags:
items:
$ref: '#/components/schemas/DAGLatestRunTaskInstanceStateCountsResponse'
type: array
title: Dags
type: object
required:
- dags
title: DAGsLatestRunTaskInstanceStateCountsCollectionResponse
description: Collection of per-Dag latest-run task-instance state counts for
the Dag list page.
DAGsRunStateCountsCollectionResponse:
properties:
dags:
Expand Down
72 changes: 72 additions & 0 deletions airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,9 @@
from airflow.api_fastapi.core_api.datamodels.dags import DAG_ALIAS_MAPPING, DAGResponse
from airflow.api_fastapi.core_api.datamodels.ui.dag_runs import DAGRunLightResponse
from airflow.api_fastapi.core_api.datamodels.ui.dags import (
DAGLatestRunTaskInstanceStateCountsResponse,
DAGRunStateCountsResponse,
DAGsLatestRunTaskInstanceStateCountsCollectionResponse,
DAGsRunStateCountsCollectionResponse,
DagTimetableTypeCollectionResponse,
DAGWithLatestDagRunsCollectionResponse,
Expand Down Expand Up @@ -402,3 +404,73 @@ def get_dag_run_state_counts(
],
state_count_limit=STATE_COUNT_CAP,
)


@dags_router.get(
"/latest_run_task_instance_state_counts",
dependencies=[
Depends(requires_access_dag(method="GET")),
Depends(requires_access_dag(method="GET", access_entity=DagAccessEntity.TASK_INSTANCE)),
],
operation_id="get_latest_run_task_instance_state_counts_ui",
)
def get_latest_run_task_instance_state_counts(
session: SessionDep,
readable_dags_filter: ReadableDagsFilterDep,
dag_run_ids: Annotated[
list[int], Query(min_length=1, max_length=conf.getint("api", "maximum_page_limit"))
],
) -> DAGsLatestRunTaskInstanceStateCountsCollectionResponse:
"""
Return task-instance state counts for the given Dag runs, for the Dag list page.

The Dag list response already carries the latest run of each Dag, so the caller passes
those run ids straight in. Deriving the latest run again here would mean an
``ORDER BY run_after DESC LIMIT 1`` per Dag, which has no supporting index and degrades
badly once a Dag has many runs. Runs the caller may not read are dropped.
"""
permitted_dag_ids = readable_dags_filter.value or set()

dags: list[DAGLatestRunTaskInstanceStateCountsResponse] = []
if not permitted_dag_ids:
return DAGsLatestRunTaskInstanceStateCountsCollectionResponse(dags=dags)

# Ascending run_after: if two runs of one Dag are passed, the newer one wins below.
requested_runs = session.execute(
select(DagRun.dag_id, DagRun.run_id)
.where(DagRun.id.in_(set(dag_run_ids)), DagRun.dag_id.in_(permitted_dag_ids))
.order_by(DagRun.run_after)
).all()
latest_run_id_by_dag: dict[str, str] = {row.dag_id: row.run_id for row in requested_runs}

if latest_run_id_by_dag:
# Each branch filters on (dag_id, run_id) equality, which the ti_dag_run index
# covers. A run's task instances are bounded by the Dag's task structure, so the
# per-state counts are exact (no cap needed here, unlike the cross-run counts in
# get_dag_run_state_counts).
ti_branches = [
select(literal(dag_id).label("dag_id"), TaskInstance.state.label("state"))
.where(TaskInstance.dag_id == dag_id, TaskInstance.run_id == run_id)
.subquery()
for dag_id, run_id in latest_run_id_by_dag.items()
]
tis_union = union_all(*(select(branch) for branch in ti_branches)).subquery()
counts_by_dag: dict[str, dict[str, int]] = {dag_id: {} for dag_id in latest_run_id_by_dag}
for row in session.execute(
select(tis_union.c.dag_id, tis_union.c.state, func.count().label("cnt")).group_by(
tis_union.c.dag_id, tis_union.c.state
)
):
state_key = row.state if row.state is not None else "no_status"
counts_by_dag[row.dag_id][state_key] = row.cnt

dags = [
DAGLatestRunTaskInstanceStateCountsResponse(
dag_id=dag_id,
run_id=run_id,
state_counts=counts_by_dag[dag_id],
)
for dag_id, run_id in sorted(latest_run_id_by_dag.items())
]

return DAGsLatestRunTaskInstanceStateCountsCollectionResponse(dags=dags)
6 changes: 6 additions & 0 deletions airflow-core/src/airflow/ui/openapi-gen/queries/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -410,6 +410,12 @@ export const useDagServiceGetDagRunStateCountsUiKey = "DagServiceGetDagRunStateC
export const UseDagServiceGetDagRunStateCountsUiKeyFn = ({ dagIds }: {
dagIds: string[];
}, queryKey?: Array<unknown>) => [useDagServiceGetDagRunStateCountsUiKey, ...(queryKey ?? [{ dagIds }])];
export type DagServiceGetLatestRunTaskInstanceStateCountsUiDefaultResponse = Awaited<ReturnType<typeof DagService.getLatestRunTaskInstanceStateCountsUi>>;
export type DagServiceGetLatestRunTaskInstanceStateCountsUiQueryResult<TData = DagServiceGetLatestRunTaskInstanceStateCountsUiDefaultResponse, TError = unknown> = UseQueryResult<TData, TError>;
export const useDagServiceGetLatestRunTaskInstanceStateCountsUiKey = "DagServiceGetLatestRunTaskInstanceStateCountsUi";
export const UseDagServiceGetLatestRunTaskInstanceStateCountsUiKeyFn = ({ dagRunIds }: {
dagRunIds: number[];
}, queryKey?: Array<unknown>) => [useDagServiceGetLatestRunTaskInstanceStateCountsUiKey, ...(queryKey ?? [{ dagRunIds }])];
export type EventLogServiceGetEventLogDefaultResponse = Awaited<ReturnType<typeof EventLogService.getEventLog>>;
export type EventLogServiceGetEventLogQueryResult<TData = EventLogServiceGetEventLogDefaultResponse, TError = unknown> = UseQueryResult<TData, TError>;
export const useEventLogServiceGetEventLogKey = "EventLogServiceGetEventLog";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -785,6 +785,22 @@ export const ensureUseDagServiceGetDagRunStateCountsUiData = (queryClient: Query
dagIds: string[];
}) => queryClient.ensureQueryData({ queryKey: Common.UseDagServiceGetDagRunStateCountsUiKeyFn({ dagIds }), queryFn: () => DagService.getDagRunStateCountsUi({ dagIds }) });
/**
* Get Latest Run Task Instance State Counts
* Return task-instance state counts for the given Dag runs, for the Dag list page.
*
* The Dag list response already carries the latest run of each Dag, so the caller passes
* those run ids straight in. Deriving the latest run again here would mean an
* ``ORDER BY run_after DESC LIMIT 1`` per Dag, which has no supporting index and degrades
* badly once a Dag has many runs. Runs the caller may not read are dropped.
* @param data The data for the request.
* @param data.dagRunIds
* @returns DAGsLatestRunTaskInstanceStateCountsCollectionResponse Successful Response
* @throws ApiError
*/
export const ensureUseDagServiceGetLatestRunTaskInstanceStateCountsUiData = (queryClient: QueryClient, { dagRunIds }: {
dagRunIds: number[];
}) => queryClient.ensureQueryData({ queryKey: Common.UseDagServiceGetLatestRunTaskInstanceStateCountsUiKeyFn({ dagRunIds }), queryFn: () => DagService.getLatestRunTaskInstanceStateCountsUi({ dagRunIds }) });
/**
* Get Event Log
* @param data The data for the request.
* @param data.eventLogId
Expand Down
16 changes: 16 additions & 0 deletions airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -785,6 +785,22 @@ export const prefetchUseDagServiceGetDagRunStateCountsUi = (queryClient: QueryCl
dagIds: string[];
}) => queryClient.prefetchQuery({ queryKey: Common.UseDagServiceGetDagRunStateCountsUiKeyFn({ dagIds }), queryFn: () => DagService.getDagRunStateCountsUi({ dagIds }) });
/**
* Get Latest Run Task Instance State Counts
* Return task-instance state counts for the given Dag runs, for the Dag list page.
*
* The Dag list response already carries the latest run of each Dag, so the caller passes
* those run ids straight in. Deriving the latest run again here would mean an
* ``ORDER BY run_after DESC LIMIT 1`` per Dag, which has no supporting index and degrades
* badly once a Dag has many runs. Runs the caller may not read are dropped.
* @param data The data for the request.
* @param data.dagRunIds
* @returns DAGsLatestRunTaskInstanceStateCountsCollectionResponse Successful Response
* @throws ApiError
*/
export const prefetchUseDagServiceGetLatestRunTaskInstanceStateCountsUi = (queryClient: QueryClient, { dagRunIds }: {
dagRunIds: number[];
}) => queryClient.prefetchQuery({ queryKey: Common.UseDagServiceGetLatestRunTaskInstanceStateCountsUiKeyFn({ dagRunIds }), queryFn: () => DagService.getLatestRunTaskInstanceStateCountsUi({ dagRunIds }) });
/**
* Get Event Log
* @param data The data for the request.
* @param data.eventLogId
Expand Down
16 changes: 16 additions & 0 deletions airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -785,6 +785,22 @@ export const useDagServiceGetDagRunStateCountsUi = <TData = Common.DagServiceGet
dagIds: string[];
}, queryKey?: TQueryKey, options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">) => useQuery<TData, TError>({ queryKey: Common.UseDagServiceGetDagRunStateCountsUiKeyFn({ dagIds }, queryKey), queryFn: () => DagService.getDagRunStateCountsUi({ dagIds }) as TData, ...options });
/**
* Get Latest Run Task Instance State Counts
* Return task-instance state counts for the given Dag runs, for the Dag list page.
*
* The Dag list response already carries the latest run of each Dag, so the caller passes
* those run ids straight in. Deriving the latest run again here would mean an
* ``ORDER BY run_after DESC LIMIT 1`` per Dag, which has no supporting index and degrades
* badly once a Dag has many runs. Runs the caller may not read are dropped.
* @param data The data for the request.
* @param data.dagRunIds
* @returns DAGsLatestRunTaskInstanceStateCountsCollectionResponse Successful Response
* @throws ApiError
*/
export const useDagServiceGetLatestRunTaskInstanceStateCountsUi = <TData = Common.DagServiceGetLatestRunTaskInstanceStateCountsUiDefaultResponse, TError = unknown, TQueryKey extends Array<unknown> = unknown[]>({ dagRunIds }: {
dagRunIds: number[];
}, queryKey?: TQueryKey, options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">) => useQuery<TData, TError>({ queryKey: Common.UseDagServiceGetLatestRunTaskInstanceStateCountsUiKeyFn({ dagRunIds }, queryKey), queryFn: () => DagService.getLatestRunTaskInstanceStateCountsUi({ dagRunIds }) as TData, ...options });
/**
* Get Event Log
* @param data The data for the request.
* @param data.eventLogId
Expand Down
16 changes: 16 additions & 0 deletions airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts
Original file line number Diff line number Diff line change
Expand Up @@ -785,6 +785,22 @@ export const useDagServiceGetDagRunStateCountsUiSuspense = <TData = Common.DagSe
dagIds: string[];
}, queryKey?: TQueryKey, options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">) => useSuspenseQuery<TData, TError>({ queryKey: Common.UseDagServiceGetDagRunStateCountsUiKeyFn({ dagIds }, queryKey), queryFn: () => DagService.getDagRunStateCountsUi({ dagIds }) as TData, ...options });
/**
* Get Latest Run Task Instance State Counts
* Return task-instance state counts for the given Dag runs, for the Dag list page.
*
* The Dag list response already carries the latest run of each Dag, so the caller passes
* those run ids straight in. Deriving the latest run again here would mean an
* ``ORDER BY run_after DESC LIMIT 1`` per Dag, which has no supporting index and degrades
* badly once a Dag has many runs. Runs the caller may not read are dropped.
* @param data The data for the request.
* @param data.dagRunIds
* @returns DAGsLatestRunTaskInstanceStateCountsCollectionResponse Successful Response
* @throws ApiError
*/
export const useDagServiceGetLatestRunTaskInstanceStateCountsUiSuspense = <TData = Common.DagServiceGetLatestRunTaskInstanceStateCountsUiDefaultResponse, TError = unknown, TQueryKey extends Array<unknown> = unknown[]>({ dagRunIds }: {
dagRunIds: number[];
}, queryKey?: TQueryKey, options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">) => useSuspenseQuery<TData, TError>({ queryKey: Common.UseDagServiceGetLatestRunTaskInstanceStateCountsUiKeyFn({ dagRunIds }, queryKey), queryFn: () => DagService.getLatestRunTaskInstanceStateCountsUi({ dagRunIds }) as TData, ...options });
/**
* Get Event Log
* @param data The data for the request.
* @param data.eventLogId
Expand Down
43 changes: 43 additions & 0 deletions airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9045,6 +9045,33 @@ It is used to transfer providers information loaded by providers_manager such th
the API server/Web UI can use this data to render connection form UI.`
} as const;

export const $DAGLatestRunTaskInstanceStateCountsResponse = {
properties: {
dag_id: {
type: 'string',
title: 'Dag Id'
},
run_id: {
type: 'string',
title: 'Run Id'
},
state_counts: {
additionalProperties: {
type: 'integer'
},
type: 'object',
title: 'State Counts'
}
},
type: 'object',
required: ['dag_id', 'run_id', 'state_counts'],
title: 'DAGLatestRunTaskInstanceStateCountsResponse',
description: `Task-instance state counts for a Dag's latest run.

\`\`state_counts\`\` only carries states present in the run; task instances without a
state yet are keyed as \`\`no_status\`\`.`
} as const;

export const $DAGRunLightResponse = {
properties: {
id: {
Expand Down Expand Up @@ -9506,6 +9533,22 @@ export const $DAGWithLatestDagRunsResponse = {
description: 'DAG with latest dag runs response serializer.'
} as const;

export const $DAGsLatestRunTaskInstanceStateCountsCollectionResponse = {
properties: {
dags: {
items: {
'$ref': '#/components/schemas/DAGLatestRunTaskInstanceStateCountsResponse'
},
type: 'array',
title: 'Dags'
}
},
type: 'object',
required: ['dags'],
title: 'DAGsLatestRunTaskInstanceStateCountsCollectionResponse',
description: 'Collection of per-Dag latest-run task-instance state counts for the Dag list page.'
} as const;

export const $DAGsRunStateCountsCollectionResponse = {
properties: {
dags: {
Expand Down
Loading
Loading