add support for None in forecast values

lsabor · lsabor · commit c734f33ee32f · 2025-11-30T12:15:16.000-08:00
support null values on frontend

fix flake8 complaints
diff --git a/front_end/src/app/(main)/aggregation-explorer/components/continuous_aggregations_chart.tsx b/front_end/src/app/(main)/aggregation-explorer/components/continuous_aggregations_chart.tsx
@@ -84,7 +84,12 @@ const ContinuousAggregationChart: FC<Props> = ({
       if (historyItem) {
         charts.push({
           pmf: cdfToPmf(historyItem.forecast_values),
-          cdf: historyItem.forecast_values,
+          cdf: historyItem.forecast_values.map((v) => {
+            if (v === null) {
+              throw new Error("Forecast values contain null values");
+            }
+            return v;
+          }),
           type: "community",
         });
       }
diff --git a/front_end/src/app/(main)/questions/[id]/components/question_view/consumer_question_view/prediction/single_question_prediction/binary_question_prediction.tsx b/front_end/src/app/(main)/questions/[id]/components/question_view/consumer_question_view/prediction/single_question_prediction/binary_question_prediction.tsx
@@ -81,7 +81,7 @@ const QuestionInfo: React.FC<{
 
   const getYesProbability = (
     q: QuestionWithNumericForecasts
-  ): number | undefined => {
+  ): number | null | undefined => {
     if (q.type !== QuestionType.Binary) return undefined;
     const values =
       q.aggregations?.[q.default_aggregation_method]?.latest?.forecast_values;
diff --git a/front_end/src/components/charts/continuous_area_chart.tsx b/front_end/src/components/charts/continuous_area_chart.tsx
@@ -1127,7 +1127,12 @@ export function getContinuousAreaChartData({
   if (latest && isForecastActive(latest)) {
     chartData.push({
       pmf: cdfToPmf(latest.forecast_values),
-      cdf: latest.forecast_values,
+      cdf: latest.forecast_values.map((v) => {
+        if (v === null) {
+          throw new Error("Forecast values contain null values");
+        }
+        return v;
+      }),
       type: (isClosed ? "community_closed" : "community") as ContinuousAreaType,
     });
   }
@@ -1141,7 +1146,12 @@ export function getContinuousAreaChartData({
   } else if (!!userForecast && isForecastActive(userForecast)) {
     chartData.push({
       pmf: cdfToPmf(userForecast.forecast_values),
-      cdf: userForecast.forecast_values,
+      cdf: userForecast.forecast_values.map((v) => {
+        if (v === null) {
+          throw new Error("Forecast values contain null values");
+        }
+        return v;
+      }),
       type: "user" as ContinuousAreaType,
     });
   }
diff --git a/front_end/src/components/charts/histogram.tsx b/front_end/src/components/charts/histogram.tsx
@@ -21,8 +21,8 @@ import ChartContainer from "./primitives/chart_container";
 
 type HistogramProps = {
   histogramData: { x: number; y: number }[];
-  median: number | undefined;
-  mean: number | undefined;
+  median: number | null | undefined;
+  mean: number | null | undefined;
   color: "blue" | "gray";
   width?: number;
 };
diff --git a/front_end/src/components/charts/minified_continuous_area_chart.tsx b/front_end/src/components/charts/minified_continuous_area_chart.tsx
@@ -607,7 +607,12 @@ export function getContinuousAreaChartData({
   if (latest && isForecastActive(latest)) {
     chartData.push({
       pmf: cdfToPmf(latest.forecast_values),
-      cdf: latest.forecast_values,
+      cdf: latest.forecast_values.map((v) => {
+        if (v === null) {
+          throw new Error("Forecast values contain null values");
+        }
+        return v;
+      }),
       type: (isClosed ? "community_closed" : "community") as ContinuousAreaType,
     });
   }
diff --git a/front_end/src/components/conditional_tile/conditional_chart.tsx b/front_end/src/components/conditional_tile/conditional_chart.tsx
@@ -146,7 +146,12 @@ const ConditionalChart: FC<Props> = ({
           ? [
               {
                 pmf: cdfToPmf(aggregateLatest.forecast_values),
-                cdf: aggregateLatest.forecast_values,
+                cdf: aggregateLatest.forecast_values.map((v) => {
+                  if (v === null) {
+                    throw new Error("Forecast values contain null values");
+                  }
+                  return v;
+                }),
                 type: "community" as ContinuousAreaType,
               },
             ]
diff --git a/front_end/src/components/forecast_maker/continuous_input/continuous_prediction_chart.tsx b/front_end/src/components/forecast_maker/continuous_input/continuous_prediction_chart.tsx
@@ -133,7 +133,12 @@ const ContinuousPredictionChart: FC<Props> = ({
     if (showCP && latestAggLatest && isForecastActive(latestAggLatest)) {
       charts.push({
         pmf: cdfToPmf(latestAggLatest.forecast_values),
-        cdf: latestAggLatest.forecast_values,
+        cdf: latestAggLatest.forecast_values.map((v) => {
+          if (v === null) {
+            throw new Error("Forecast values contain null values");
+          }
+          return v;
+        }),
         type:
           question.status === QuestionStatus.CLOSED
             ? "community_closed"
diff --git a/front_end/src/components/forecast_maker/forecast_maker_conditional/forecast_maker_conditional_continuous.tsx b/front_end/src/components/forecast_maker/forecast_maker_conditional/forecast_maker_conditional_continuous.tsx
@@ -676,15 +676,25 @@ const ForecastMakerConditionalContinuous: FC<Props> = ({
     ).cdf;
   const userPreviousCdf: number[] | undefined =
     overlayPreviousForecast && previousForecast
-      ? previousForecast.forecast_values
+      ? previousForecast.forecast_values.map((v) => {
+          if (v === null) {
+            throw new Error("Forecast values contain null values");
+          }
+          return v;
+        })
       : undefined;
   const aggregateLatest =
     activeOptionData?.question.aggregations[
       activeOptionData.question.default_aggregation_method
     ].latest;
   const communityCdf: number[] | undefined =
     aggregateLatest && isForecastActive(aggregateLatest)
-      ? aggregateLatest.forecast_values
+      ? aggregateLatest.forecast_values.map((v) => {
+          if (v === null) {
+            throw new Error("Forecast values contain null values");
+          }
+          return v;
+        })
       : undefined;
 
   const predictButtonIsDisabled =
diff --git a/front_end/src/components/forecast_maker/forecast_maker_group/continuous_input_wrapper.tsx b/front_end/src/components/forecast_maker/forecast_maker_group/continuous_input_wrapper.tsx
@@ -194,7 +194,12 @@ const ContinuousInputWrapper: FC<PropsWithChildren<Props>> = ({
     [option]
   );
 
-  const rawPreviousCdf = previousForecast?.forecast_values;
+  const rawPreviousCdf = previousForecast?.forecast_values.map((v) => {
+    if (v === null) {
+      throw new Error("Forecast values contain null values");
+    }
+    return v;
+  });
   const showWithdrawnRow = option.wasWithdrawn && !option.isDirty;
   const showPreviousRowByCheckbox =
     !showWithdrawnRow && overlayPreviousForecast;
@@ -238,9 +243,14 @@ const ContinuousInputWrapper: FC<PropsWithChildren<Props>> = ({
 
   const withdraw = () => onWithdraw();
 
-  const communityCdf: number[] | undefined =
-    option.question.aggregations[option.question.default_aggregation_method]
-      .latest?.forecast_values;
+  const communityCdf: number[] | undefined = option.question.aggregations[
+    option.question.default_aggregation_method
+  ].latest?.forecast_values.map((v) => {
+    if (v === null) {
+      throw new Error("Forecast values contain null values");
+    }
+    return v;
+  });
 
   const questionDuration =
     new Date(option.question.scheduled_close_time).getTime() -
diff --git a/front_end/src/components/forecast_maker/forecast_maker_group/forecast_maker_group_continuous.tsx b/front_end/src/components/forecast_maker/forecast_maker_group/forecast_maker_group_continuous.tsx
@@ -598,7 +598,12 @@ const ForecastMakerGroupContinuous: FC<Props> = ({
           questionId: id,
           forecastEndTime: forecastExpirationToDate(forecastExpiration),
           forecastData: {
-            continuousCdf: latest.forecast_values,
+            continuousCdf: latest.forecast_values.map((v) => {
+              if (v === null) {
+                throw new Error("Forecast values contain null values");
+              }
+              return v;
+            }),
             probabilityYesPerCategory: null,
             probabilityYes: null,
           },
diff --git a/front_end/src/components/forecast_maker/forecast_maker_question/forecast_maker_continuous.tsx b/front_end/src/components/forecast_maker/forecast_maker_question/forecast_maker_continuous.tsx
@@ -131,7 +131,12 @@ const ForecastMakerContinuous: FC<Props> = ({
 
   const overlayPreviousCdf =
     overlayPreviousForecast && previousForecast?.forecast_values
-      ? previousForecast.forecast_values
+      ? previousForecast.forecast_values.map((v) => {
+          if (v === null) {
+            throw new Error("Forecast values contain null values");
+          }
+          return v;
+        })
       : undefined;
 
   // Update states of forecast maker after new forecast is made
@@ -210,12 +215,24 @@ const ForecastMakerContinuous: FC<Props> = ({
   const userCdf: number[] = dataset.cdf;
   const userPreviousCdf: number[] | undefined =
     overlayPreviousForecast && previousForecast
-      ? previousForecast.forecast_values
+      ? previousForecast.forecast_values.map((v) => {
+          if (v === null) {
+            throw new Error("Forecast values contain null values");
+          }
+          return v;
+        })
       : undefined;
   const latest =
     question.aggregations[question.default_aggregation_method].latest;
   const communityCdf: number[] | undefined =
-    latest && isForecastActive(latest) ? latest?.forecast_values : undefined;
+    latest && isForecastActive(latest)
+      ? latest?.forecast_values.map((v) => {
+          if (v === null) {
+            throw new Error("Forecast values contain null values");
+          }
+          return v;
+        })
+      : undefined;
 
   const handleAddComponent = () => {
     setSliderDistributionComponents([
diff --git a/front_end/src/components/post_card/multiple_choice_tile/index.tsx b/front_end/src/components/post_card/multiple_choice_tile/index.tsx
@@ -363,7 +363,12 @@ function generateReaffirmData({
         const hasActivePrediction = latest && isForecastActive(latest);
 
         if (hasActivePrediction) {
-          forecastValues = latest.forecast_values;
+          forecastValues = latest.forecast_values.map((v) => {
+            if (v === null) {
+              throw new Error("Forecast values contain null values");
+            }
+            return v;
+          });
         }
 
         return {
diff --git a/front_end/src/components/post_card/question_tile/question_continuous_tile.tsx b/front_end/src/components/post_card/question_tile/question_continuous_tile.tsx
@@ -107,7 +107,12 @@ const QuestionContinuousTile: FC<Props> = ({
             {
               questionId: question.id,
               forecastData: {
-                continuousCdf: activeForecast.forecast_values,
+                continuousCdf: activeForecast.forecast_values.map((v) => {
+                  if (v === null) {
+                    throw new Error("Forecast values contain null values");
+                  }
+                  return v;
+                }),
                 probabilityYes: null,
                 probabilityYesPerCategory: null,
               },
diff --git a/front_end/src/components/prediction_chip.tsx b/front_end/src/components/prediction_chip.tsx
@@ -27,7 +27,7 @@ type Size = "compact" | "large";
 type Props = {
   question: QuestionWithForecasts;
   status: PostStatus;
-  predictionOverride?: number; // override displayed CP (e.g. for graph cursor), otherwise the latest CP is used
+  predictionOverride?: number | null; // override displayed CP (e.g. for graph cursor), otherwise the latest CP is used
   size?: Size;
   className?: string;
   chipClassName?: string;
diff --git a/front_end/src/types/question.ts b/front_end/src/types/question.ts
@@ -87,10 +87,10 @@ export type Forecast = {
   question_id: number;
   start_time: number;
   end_time: number | null;
-  forecast_values: number[];
-  interval_lower_bounds: number[] | null;
-  centers: number[] | null;
-  interval_upper_bounds: number[] | null;
+  forecast_values: (number | null)[];
+  interval_lower_bounds: (number | null)[] | null;
+  centers: (number | null)[] | null;
+  interval_upper_bounds: (number | null)[] | null;
 };
 
 export type ScoreData = {
@@ -155,9 +155,9 @@ export type UserForecastHistory = {
 export type AggregateForecast = Forecast & {
   method: AggregationMethod;
   forecaster_count: number;
-  means: number[] | null;
+  means: (number | null)[] | null;
   histogram: number[][] | null;
-  forecast_values: number[] | null;
+  forecast_values: (number | null)[] | null;
 };
 
 export type AggregateForecastHistory = {
diff --git a/front_end/src/utils/formatters/prediction.ts b/front_end/src/utils/formatters/prediction.ts
@@ -401,9 +401,9 @@ export function getUserPredictionDisplayValue({
     return "...";
   }
 
-  let center: number | undefined;
-  let lower: number | undefined = undefined;
-  let upper: number | undefined = undefined;
+  let center: number | null | undefined;
+  let lower: number | null | undefined = undefined;
+  let upper: number | null | undefined = undefined;
   if (questionType === QuestionType.Binary) {
     center = closestUserForecast.forecast_values[1];
   } else {
diff --git a/front_end/src/utils/math.ts b/front_end/src/utils/math.ts
@@ -64,14 +64,18 @@ function logisticCDF(
   );
 }
 
-export function cdfToPmf(cdf: number[]) {
+export function cdfToPmf(cdf: (number | null)[]) {
   const pdf = [];
   /* eslint-disable @typescript-eslint/no-non-null-assertion */
   for (let i = 0; i < cdf.length; i++) {
+    const value = cdf[i]!;
+    if (value === null) {
+      throw new Error("CDF contains null values");
+    }
     if (i === 0) {
-      pdf.push(cdf[i]!);
+      pdf.push(value);
     } else {
-      pdf.push(cdf[i]! - cdf[i - 1]!);
+      pdf.push(value - cdf[i - 1]!);
     }
   }
   pdf.push(1 - cdf[cdf.length - 1]!);
@@ -111,36 +115,40 @@ export function cdfFromSliders(
 }
 
 export function computeQuartilesFromCDF(
-  cdf: number[],
+  cdf: (number | null)[],
   extendedQuartiles: true,
   discrete?: boolean
 ): ExtendedQuartiles;
 export function computeQuartilesFromCDF(
-  cdf: number[],
+  cdf: (number | null)[],
   extendedQuartiles?: false,
   discrete?: boolean
 ): Quartiles;
 export function computeQuartilesFromCDF(cdf: number[]): Quartiles;
 export function computeQuartilesFromCDF(
-  cdf: number[],
+  cdf: (number | null)[],
   extendedQuartiles?: boolean,
   discrete?: boolean
 ): Quartiles | ExtendedQuartiles {
-  function findPercentile(cdf: number[], percentile: number) {
+  function findPercentile(cdf: (number | null)[], percentile: number) {
     if (cdf === null) {
       cdf = [];
     }
     const target = percentile / 100;
     for (let i = 0; i < cdf.length; i++) {
       /* eslint-disable @typescript-eslint/no-non-null-assertion */
-      if (cdf[i]! >= target) {
+      const value = cdf[i]!;
+      if (value === null) {
+        throw new Error("CDF contains null values");
+      }
+      if (value >= target) {
         if (i === 0) return 0;
 
         if (discrete) {
           return (i - 0.5) / (cdf.length - 1);
         }
 
-        const diff = cdf[i]! - cdf[i - 1]!;
+        const diff = value - cdf[i - 1]!;
         const adjustedPercentile = (target - cdf[i - 1]!) / diff;
         return (i - 1 + adjustedPercentile) / (cdf.length - 1);
       }
@@ -172,7 +180,9 @@ export function computeQuartilesFromCDF(
   }
 }
 
-export function getCdfBounds(cdf: number[] | undefined): Bounds | undefined {
+export function getCdfBounds(
+  cdf: (number | null)[] | undefined
+): Bounds | undefined {
   if (!cdf) {
     return;
   }
diff --git a/questions/migrations/0032_alter_aggregateforecast_forecast_values_and_more.py b/questions/migrations/0032_alter_aggregateforecast_forecast_values_and_more.py
diff --git a/questions/models.py b/questions/models.py
diff --git a/scoring/score_math.py b/scoring/score_math.py
diff --git a/utils/the_math/aggregations.py b/utils/the_math/aggregations.py
diff --git a/utils/the_math/measures.py b/utils/the_math/measures.py
diff --git a/utils/typing.py b/utils/typing.py

Original file line number	Diff line number	Diff line change
`@@ -146,7 +146,12 @@ const ConditionalChart: FC<Props> = ({`
`146`	`146`	`? [`
`147`	`147`	`{`
`148`	`148`	`pmf: cdfToPmf(aggregateLatest.forecast_values),`
`149`		`- cdf: aggregateLatest.forecast_values,`
	`149`	`+ cdf: aggregateLatest.forecast_values.map((v) => {`
	`150`	`+ if (v === null) {`
	`151`	`+ throw new Error("Forecast values contain null values");`
	`152`	`+ }`
	`153`	`+ return v;`
	`154`	`+ }),`
`150`	`155`	`type: "community" as ContinuousAreaType,`
`151`	`156`	`},`
`152`	`157`	`]`