diff --git a/front_end/src/app/(main)/aggregation-explorer/components/continuous_aggregations_chart.tsx b/front_end/src/app/(main)/aggregation-explorer/components/continuous_aggregations_chart.tsx index 22940967bd..b0a983a2d2 100644 --- a/front_end/src/app/(main)/aggregation-explorer/components/continuous_aggregations_chart.tsx +++ b/front_end/src/app/(main)/aggregation-explorer/components/continuous_aggregations_chart.tsx @@ -84,7 +84,12 @@ const ContinuousAggregationChart: FC = ({ if (historyItem) { charts.push({ pmf: cdfToPmf(historyItem.forecast_values), - cdf: historyItem.forecast_values, + cdf: historyItem.forecast_values.map((v) => { + if (v === null) { + throw new Error("Forecast values contain null values"); + } + return v; + }), type: "community", }); } diff --git a/front_end/src/app/(main)/questions/[id]/components/question_view/consumer_question_view/prediction/single_question_prediction/binary_question_prediction.tsx b/front_end/src/app/(main)/questions/[id]/components/question_view/consumer_question_view/prediction/single_question_prediction/binary_question_prediction.tsx index 1c971ee26e..47982eeab4 100644 --- a/front_end/src/app/(main)/questions/[id]/components/question_view/consumer_question_view/prediction/single_question_prediction/binary_question_prediction.tsx +++ b/front_end/src/app/(main)/questions/[id]/components/question_view/consumer_question_view/prediction/single_question_prediction/binary_question_prediction.tsx @@ -81,7 +81,7 @@ const QuestionInfo: React.FC<{ const getYesProbability = ( q: QuestionWithNumericForecasts - ): number | undefined => { + ): number | null | undefined => { if (q.type !== QuestionType.Binary) return undefined; const values = q.aggregations?.[q.default_aggregation_method]?.latest?.forecast_values; diff --git a/front_end/src/components/charts/continuous_area_chart.tsx b/front_end/src/components/charts/continuous_area_chart.tsx index c671eb2094..d0614beab1 100644 --- a/front_end/src/components/charts/continuous_area_chart.tsx +++ b/front_end/src/components/charts/continuous_area_chart.tsx @@ -1127,7 +1127,12 @@ export function getContinuousAreaChartData({ if (latest && isForecastActive(latest)) { chartData.push({ pmf: cdfToPmf(latest.forecast_values), - cdf: latest.forecast_values, + cdf: latest.forecast_values.map((v) => { + if (v === null) { + throw new Error("Forecast values contain null values"); + } + return v; + }), type: (isClosed ? "community_closed" : "community") as ContinuousAreaType, }); } @@ -1141,7 +1146,12 @@ export function getContinuousAreaChartData({ } else if (!!userForecast && isForecastActive(userForecast)) { chartData.push({ pmf: cdfToPmf(userForecast.forecast_values), - cdf: userForecast.forecast_values, + cdf: userForecast.forecast_values.map((v) => { + if (v === null) { + throw new Error("Forecast values contain null values"); + } + return v; + }), type: "user" as ContinuousAreaType, }); } diff --git a/front_end/src/components/charts/histogram.tsx b/front_end/src/components/charts/histogram.tsx index 6580cc56a4..ae1443be89 100644 --- a/front_end/src/components/charts/histogram.tsx +++ b/front_end/src/components/charts/histogram.tsx @@ -21,8 +21,8 @@ import ChartContainer from "./primitives/chart_container"; type HistogramProps = { histogramData: { x: number; y: number }[]; - median: number | undefined; - mean: number | undefined; + median: number | null | undefined; + mean: number | null | undefined; color: "blue" | "gray"; width?: number; }; diff --git a/front_end/src/components/charts/minified_continuous_area_chart.tsx b/front_end/src/components/charts/minified_continuous_area_chart.tsx index ccdc7a3bfe..ea80085b1a 100644 --- a/front_end/src/components/charts/minified_continuous_area_chart.tsx +++ b/front_end/src/components/charts/minified_continuous_area_chart.tsx @@ -607,7 +607,12 @@ export function getContinuousAreaChartData({ if (latest && isForecastActive(latest)) { chartData.push({ pmf: cdfToPmf(latest.forecast_values), - cdf: latest.forecast_values, + cdf: latest.forecast_values.map((v) => { + if (v === null) { + throw new Error("Forecast values contain null values"); + } + return v; + }), type: (isClosed ? "community_closed" : "community") as ContinuousAreaType, }); } diff --git a/front_end/src/components/conditional_tile/conditional_chart.tsx b/front_end/src/components/conditional_tile/conditional_chart.tsx index d208b80280..d5263f1d7d 100644 --- a/front_end/src/components/conditional_tile/conditional_chart.tsx +++ b/front_end/src/components/conditional_tile/conditional_chart.tsx @@ -146,7 +146,12 @@ const ConditionalChart: FC = ({ ? [ { pmf: cdfToPmf(aggregateLatest.forecast_values), - cdf: aggregateLatest.forecast_values, + cdf: aggregateLatest.forecast_values.map((v) => { + if (v === null) { + throw new Error("Forecast values contain null values"); + } + return v; + }), type: "community" as ContinuousAreaType, }, ] diff --git a/front_end/src/components/forecast_maker/continuous_input/continuous_prediction_chart.tsx b/front_end/src/components/forecast_maker/continuous_input/continuous_prediction_chart.tsx index 08bdb7d5af..c3d5b2c158 100644 --- a/front_end/src/components/forecast_maker/continuous_input/continuous_prediction_chart.tsx +++ b/front_end/src/components/forecast_maker/continuous_input/continuous_prediction_chart.tsx @@ -133,7 +133,12 @@ const ContinuousPredictionChart: FC = ({ if (showCP && latestAggLatest && isForecastActive(latestAggLatest)) { charts.push({ pmf: cdfToPmf(latestAggLatest.forecast_values), - cdf: latestAggLatest.forecast_values, + cdf: latestAggLatest.forecast_values.map((v) => { + if (v === null) { + throw new Error("Forecast values contain null values"); + } + return v; + }), type: question.status === QuestionStatus.CLOSED ? "community_closed" diff --git a/front_end/src/components/forecast_maker/forecast_maker_conditional/forecast_maker_conditional_continuous.tsx b/front_end/src/components/forecast_maker/forecast_maker_conditional/forecast_maker_conditional_continuous.tsx index 5231e9b03d..823418092b 100644 --- a/front_end/src/components/forecast_maker/forecast_maker_conditional/forecast_maker_conditional_continuous.tsx +++ b/front_end/src/components/forecast_maker/forecast_maker_conditional/forecast_maker_conditional_continuous.tsx @@ -676,7 +676,12 @@ const ForecastMakerConditionalContinuous: FC = ({ ).cdf; const userPreviousCdf: number[] | undefined = overlayPreviousForecast && previousForecast - ? previousForecast.forecast_values + ? previousForecast.forecast_values.map((v) => { + if (v === null) { + throw new Error("Forecast values contain null values"); + } + return v; + }) : undefined; const aggregateLatest = activeOptionData?.question.aggregations[ @@ -684,7 +689,12 @@ const ForecastMakerConditionalContinuous: FC = ({ ].latest; const communityCdf: number[] | undefined = aggregateLatest && isForecastActive(aggregateLatest) - ? aggregateLatest.forecast_values + ? aggregateLatest.forecast_values.map((v) => { + if (v === null) { + throw new Error("Forecast values contain null values"); + } + return v; + }) : undefined; const predictButtonIsDisabled = diff --git a/front_end/src/components/forecast_maker/forecast_maker_group/continuous_input_wrapper.tsx b/front_end/src/components/forecast_maker/forecast_maker_group/continuous_input_wrapper.tsx index 278314fd69..7af9cf8af5 100644 --- a/front_end/src/components/forecast_maker/forecast_maker_group/continuous_input_wrapper.tsx +++ b/front_end/src/components/forecast_maker/forecast_maker_group/continuous_input_wrapper.tsx @@ -194,7 +194,12 @@ const ContinuousInputWrapper: FC> = ({ [option] ); - const rawPreviousCdf = previousForecast?.forecast_values; + const rawPreviousCdf = previousForecast?.forecast_values.map((v) => { + if (v === null) { + throw new Error("Forecast values contain null values"); + } + return v; + }); const showWithdrawnRow = option.wasWithdrawn && !option.isDirty; const showPreviousRowByCheckbox = !showWithdrawnRow && overlayPreviousForecast; @@ -238,9 +243,14 @@ const ContinuousInputWrapper: FC> = ({ const withdraw = () => onWithdraw(); - const communityCdf: number[] | undefined = - option.question.aggregations[option.question.default_aggregation_method] - .latest?.forecast_values; + const communityCdf: number[] | undefined = option.question.aggregations[ + option.question.default_aggregation_method + ].latest?.forecast_values.map((v) => { + if (v === null) { + throw new Error("Forecast values contain null values"); + } + return v; + }); const questionDuration = new Date(option.question.scheduled_close_time).getTime() - diff --git a/front_end/src/components/forecast_maker/forecast_maker_group/forecast_maker_group_continuous.tsx b/front_end/src/components/forecast_maker/forecast_maker_group/forecast_maker_group_continuous.tsx index 6c1c9b2da5..dd7fb3cbd8 100644 --- a/front_end/src/components/forecast_maker/forecast_maker_group/forecast_maker_group_continuous.tsx +++ b/front_end/src/components/forecast_maker/forecast_maker_group/forecast_maker_group_continuous.tsx @@ -598,7 +598,12 @@ const ForecastMakerGroupContinuous: FC = ({ questionId: id, forecastEndTime: forecastExpirationToDate(forecastExpiration), forecastData: { - continuousCdf: latest.forecast_values, + continuousCdf: latest.forecast_values.map((v) => { + if (v === null) { + throw new Error("Forecast values contain null values"); + } + return v; + }), probabilityYesPerCategory: null, probabilityYes: null, }, diff --git a/front_end/src/components/forecast_maker/forecast_maker_question/forecast_maker_continuous.tsx b/front_end/src/components/forecast_maker/forecast_maker_question/forecast_maker_continuous.tsx index 1f5db4ed1b..5d2e302269 100644 --- a/front_end/src/components/forecast_maker/forecast_maker_question/forecast_maker_continuous.tsx +++ b/front_end/src/components/forecast_maker/forecast_maker_question/forecast_maker_continuous.tsx @@ -131,7 +131,12 @@ const ForecastMakerContinuous: FC = ({ const overlayPreviousCdf = overlayPreviousForecast && previousForecast?.forecast_values - ? previousForecast.forecast_values + ? previousForecast.forecast_values.map((v) => { + if (v === null) { + throw new Error("Forecast values contain null values"); + } + return v; + }) : undefined; // Update states of forecast maker after new forecast is made @@ -210,12 +215,24 @@ const ForecastMakerContinuous: FC = ({ const userCdf: number[] = dataset.cdf; const userPreviousCdf: number[] | undefined = overlayPreviousForecast && previousForecast - ? previousForecast.forecast_values + ? previousForecast.forecast_values.map((v) => { + if (v === null) { + throw new Error("Forecast values contain null values"); + } + return v; + }) : undefined; const latest = question.aggregations[question.default_aggregation_method].latest; const communityCdf: number[] | undefined = - latest && isForecastActive(latest) ? latest?.forecast_values : undefined; + latest && isForecastActive(latest) + ? latest?.forecast_values.map((v) => { + if (v === null) { + throw new Error("Forecast values contain null values"); + } + return v; + }) + : undefined; const handleAddComponent = () => { setSliderDistributionComponents([ diff --git a/front_end/src/components/post_card/multiple_choice_tile/index.tsx b/front_end/src/components/post_card/multiple_choice_tile/index.tsx index f979a66c82..6260a3e2a1 100644 --- a/front_end/src/components/post_card/multiple_choice_tile/index.tsx +++ b/front_end/src/components/post_card/multiple_choice_tile/index.tsx @@ -363,7 +363,12 @@ function generateReaffirmData({ const hasActivePrediction = latest && isForecastActive(latest); if (hasActivePrediction) { - forecastValues = latest.forecast_values; + forecastValues = latest.forecast_values.map((v) => { + if (v === null) { + throw new Error("Forecast values contain null values"); + } + return v; + }); } return { diff --git a/front_end/src/components/post_card/question_tile/question_continuous_tile.tsx b/front_end/src/components/post_card/question_tile/question_continuous_tile.tsx index 04f8ced4ed..cb00a25355 100644 --- a/front_end/src/components/post_card/question_tile/question_continuous_tile.tsx +++ b/front_end/src/components/post_card/question_tile/question_continuous_tile.tsx @@ -107,7 +107,12 @@ const QuestionContinuousTile: FC = ({ { questionId: question.id, forecastData: { - continuousCdf: activeForecast.forecast_values, + continuousCdf: activeForecast.forecast_values.map((v) => { + if (v === null) { + throw new Error("Forecast values contain null values"); + } + return v; + }), probabilityYes: null, probabilityYesPerCategory: null, }, diff --git a/front_end/src/components/prediction_chip.tsx b/front_end/src/components/prediction_chip.tsx index cf18f45a47..ff6ceef892 100644 --- a/front_end/src/components/prediction_chip.tsx +++ b/front_end/src/components/prediction_chip.tsx @@ -27,7 +27,7 @@ type Size = "compact" | "large"; type Props = { question: QuestionWithForecasts; status: PostStatus; - predictionOverride?: number; // override displayed CP (e.g. for graph cursor), otherwise the latest CP is used + predictionOverride?: number | null; // override displayed CP (e.g. for graph cursor), otherwise the latest CP is used size?: Size; className?: string; chipClassName?: string; diff --git a/front_end/src/types/question.ts b/front_end/src/types/question.ts index d236451516..0caae0eb6b 100644 --- a/front_end/src/types/question.ts +++ b/front_end/src/types/question.ts @@ -87,10 +87,10 @@ export type Forecast = { question_id: number; start_time: number; end_time: number | null; - forecast_values: number[]; - interval_lower_bounds: number[] | null; - centers: number[] | null; - interval_upper_bounds: number[] | null; + forecast_values: (number | null)[]; + interval_lower_bounds: (number | null)[] | null; + centers: (number | null)[] | null; + interval_upper_bounds: (number | null)[] | null; }; export type ScoreData = { @@ -155,9 +155,9 @@ export type UserForecastHistory = { export type AggregateForecast = Forecast & { method: AggregationMethod; forecaster_count: number; - means: number[] | null; + means: (number | null)[] | null; histogram: number[][] | null; - forecast_values: number[] | null; + forecast_values: (number | null)[] | null; }; export type AggregateForecastHistory = { diff --git a/front_end/src/utils/formatters/prediction.ts b/front_end/src/utils/formatters/prediction.ts index 0fff5aafd8..7765010330 100644 --- a/front_end/src/utils/formatters/prediction.ts +++ b/front_end/src/utils/formatters/prediction.ts @@ -401,9 +401,9 @@ export function getUserPredictionDisplayValue({ return "..."; } - let center: number | undefined; - let lower: number | undefined = undefined; - let upper: number | undefined = undefined; + let center: number | null | undefined; + let lower: number | null | undefined = undefined; + let upper: number | null | undefined = undefined; if (questionType === QuestionType.Binary) { center = closestUserForecast.forecast_values[1]; } else { diff --git a/front_end/src/utils/math.ts b/front_end/src/utils/math.ts index c0065166a0..53eb3d004b 100644 --- a/front_end/src/utils/math.ts +++ b/front_end/src/utils/math.ts @@ -64,14 +64,18 @@ function logisticCDF( ); } -export function cdfToPmf(cdf: number[]) { +export function cdfToPmf(cdf: (number | null)[]) { const pdf = []; /* eslint-disable @typescript-eslint/no-non-null-assertion */ for (let i = 0; i < cdf.length; i++) { + const value = cdf[i]!; + if (value === null) { + throw new Error("CDF contains null values"); + } if (i === 0) { - pdf.push(cdf[i]!); + pdf.push(value); } else { - pdf.push(cdf[i]! - cdf[i - 1]!); + pdf.push(value - cdf[i - 1]!); } } pdf.push(1 - cdf[cdf.length - 1]!); @@ -111,36 +115,40 @@ export function cdfFromSliders( } export function computeQuartilesFromCDF( - cdf: number[], + cdf: (number | null)[], extendedQuartiles: true, discrete?: boolean ): ExtendedQuartiles; export function computeQuartilesFromCDF( - cdf: number[], + cdf: (number | null)[], extendedQuartiles?: false, discrete?: boolean ): Quartiles; export function computeQuartilesFromCDF(cdf: number[]): Quartiles; export function computeQuartilesFromCDF( - cdf: number[], + cdf: (number | null)[], extendedQuartiles?: boolean, discrete?: boolean ): Quartiles | ExtendedQuartiles { - function findPercentile(cdf: number[], percentile: number) { + function findPercentile(cdf: (number | null)[], percentile: number) { if (cdf === null) { cdf = []; } const target = percentile / 100; for (let i = 0; i < cdf.length; i++) { /* eslint-disable @typescript-eslint/no-non-null-assertion */ - if (cdf[i]! >= target) { + const value = cdf[i]!; + if (value === null) { + throw new Error("CDF contains null values"); + } + if (value >= target) { if (i === 0) return 0; if (discrete) { return (i - 0.5) / (cdf.length - 1); } - const diff = cdf[i]! - cdf[i - 1]!; + const diff = value - cdf[i - 1]!; const adjustedPercentile = (target - cdf[i - 1]!) / diff; return (i - 1 + adjustedPercentile) / (cdf.length - 1); } @@ -172,7 +180,9 @@ export function computeQuartilesFromCDF( } } -export function getCdfBounds(cdf: number[] | undefined): Bounds | undefined { +export function getCdfBounds( + cdf: (number | null)[] | undefined +): Bounds | undefined { if (!cdf) { return; } diff --git a/questions/migrations/0032_alter_aggregateforecast_forecast_values_and_more.py b/questions/migrations/0032_alter_aggregateforecast_forecast_values_and_more.py new file mode 100644 index 0000000000..35f19f7b8a --- /dev/null +++ b/questions/migrations/0032_alter_aggregateforecast_forecast_values_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 5.1.13 on 2025-11-28 18:17 + +import django_better_admin_arrayfield.models.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("questions", "0031_forecast_questions_f_author__d4ea27_idx"), + ] + + operations = [ + migrations.AlterField( + model_name="aggregateforecast", + name="forecast_values", + field=django_better_admin_arrayfield.models.fields.ArrayField( + base_field=models.FloatField(null=True), max_length=201, size=None + ), + ), + migrations.AlterField( + model_name="forecast", + name="probability_yes_per_category", + field=django_better_admin_arrayfield.models.fields.ArrayField( + base_field=models.FloatField(null=True), + blank=True, + null=True, + size=None, + ), + ), + ] diff --git a/questions/models.py b/questions/models.py index 5111bd1563..3849b0f8e4 100644 --- a/questions/models.py +++ b/questions/models.py @@ -512,20 +512,20 @@ class Forecast(models.Model): # CDF of a continuous forecast # evaluated at [0.0, 0.005, 0.010, ..., 0.995, 1.0] (internal representation) - continuous_cdf = ArrayField( + continuous_cdf: list[float] = ArrayField( models.FloatField(), null=True, max_length=DEFAULT_INBOUND_OUTCOME_COUNT + 1, blank=True, ) # binary prediction - probability_yes = models.FloatField( + probability_yes: float = models.FloatField( null=True, blank=True, ) # multiple choice prediction - probability_yes_per_category = ArrayField( - models.FloatField(), + probability_yes_per_category: list[float | None] = ArrayField( + models.FloatField(null=True), null=True, blank=True, ) @@ -589,7 +589,7 @@ def __str__(self): f"by {self.author.username} on {self.question.id}: {pvs}" ) - def get_prediction_values(self) -> list[float]: + def get_prediction_values(self) -> list[float | None]: if self.probability_yes: return [1 - self.probability_yes, self.probability_yes] if self.probability_yes_per_category: @@ -597,10 +597,16 @@ def get_prediction_values(self) -> list[float]: return self.continuous_cdf def get_pmf(self) -> list[float]: + """ + gets the PMF for this forecast, replacing None values with 0.0 + Not for serialization use (keep None values in that case) + """ if self.probability_yes: return [1 - self.probability_yes, self.probability_yes] if self.probability_yes_per_category: - return self.probability_yes_per_category + return [ + v or 0.0 for v in self.probability_yes_per_category + ] # replace None with 0.0 cdf = self.continuous_cdf pmf = [cdf[0]] for i in range(1, len(cdf)): @@ -633,10 +639,10 @@ class AggregateForecast(models.Model): method = models.CharField(max_length=200, choices=AggregationMethod.choices) start_time = models.DateTimeField(db_index=True) end_time = models.DateTimeField(null=True, db_index=True) - forecast_values = ArrayField( - models.FloatField(), max_length=DEFAULT_INBOUND_OUTCOME_COUNT + 1 + forecast_values: list[float | None] = ArrayField( + models.FloatField(null=True), max_length=DEFAULT_INBOUND_OUTCOME_COUNT + 1 ) - forecaster_count = models.IntegerField(null=True) + forecaster_count: int | None = models.IntegerField(null=True) interval_lower_bounds = ArrayField(models.FloatField(), null=True) centers = ArrayField(models.FloatField(), null=True) interval_upper_bounds = ArrayField(models.FloatField(), null=True) @@ -665,25 +671,33 @@ def __str__(self): f"by {self.method} on {self.question_id}: {pvs}>" ) - def get_cdf(self) -> list[float] | None: + def get_cdf(self) -> list[float | None] | None: # grab annotation if it exists for efficiency question_type = getattr(self, "question_type", self.question.type) if question_type in QUESTION_CONTINUOUS_TYPES: return self.forecast_values + return None def get_pmf(self) -> list[float]: + """ + gets the PMF for this forecast, replacing None values with 0.0 + Not for serialization use (keep None values in that case) + """ # grab annotation if it exists for efficiency question_type = getattr(self, "question_type", self.question.type) + forecast_values = [ + v or 0.0 for v in self.forecast_values + ] # replace None with 0.0 if question_type in QUESTION_CONTINUOUS_TYPES: - cdf = self.forecast_values + cdf: list[float] = forecast_values pmf = [cdf[0]] for i in range(1, len(cdf)): pmf.append(cdf[i] - cdf[i - 1]) pmf.append(1 - cdf[-1]) return pmf - return self.forecast_values + return forecast_values - def get_prediction_values(self) -> list[float]: + def get_prediction_values(self) -> list[float | None]: return self.forecast_values diff --git a/scoring/score_math.py b/scoring/score_math.py index 585807dc97..fada04f0d1 100644 --- a/scoring/score_math.py +++ b/scoring/score_math.py @@ -1,6 +1,7 @@ from dataclasses import dataclass - from datetime import datetime +from typing import Sequence + import numpy as np from scipy.stats.mstats import gmean @@ -25,7 +26,7 @@ class AggregationEntry: def get_geometric_means( - forecasts: list[Forecast | AggregateForecast], + forecasts: Sequence[Forecast | AggregateForecast], ) -> list[AggregationEntry]: geometric_means = [] timesteps: set[float] = set() @@ -52,32 +53,6 @@ def get_geometric_means( return geometric_means -def get_medians( - forecasts: list[Forecast | AggregateForecast], -) -> list[AggregationEntry]: - medians = [] - timesteps: set[float] = set() - for forecast in forecasts: - timesteps.add(forecast.start_time.timestamp()) - if forecast.end_time: - timesteps.add(forecast.end_time.timestamp()) - for timestep in sorted(timesteps): - prediction_values = [ - f.get_pmf() - for f in forecasts - if f.start_time.timestamp() <= timestep - and (f.end_time is None or f.end_time.timestamp() > timestep) - ] - if not prediction_values: - continue # TODO: doesn't account for going from 1 active forecast to 0 - median = np.median(prediction_values, axis=0) - predictors = len(prediction_values) - medians.append( - AggregationEntry(median, predictors if predictors > 1 else 0, timestep) - ) - return medians - - @dataclass class ForecastScore: score: float @@ -85,7 +60,7 @@ class ForecastScore: def evaluate_forecasts_baseline_accuracy( - forecasts: list[Forecast | AggregateForecast], + forecasts: Sequence[Forecast | AggregateForecast], resolution_bucket: int, forecast_horizon_start: float, actual_close_time: float, @@ -126,7 +101,7 @@ def evaluate_forecasts_baseline_accuracy( def evaluate_forecasts_baseline_spot_forecast( - forecasts: list[Forecast | AggregateForecast], + forecasts: Sequence[Forecast | AggregateForecast], resolution_bucket: int, spot_forecast_timestamp: float, question_type: str, @@ -157,7 +132,7 @@ def evaluate_forecasts_baseline_spot_forecast( def evaluate_forecasts_peer_accuracy( - forecasts: list[Forecast | AggregateForecast], + forecasts: Sequence[Forecast | AggregateForecast], base_forecasts: list[Forecast | AggregateForecast] | None, resolution_bucket: int, forecast_horizon_start: float, @@ -206,10 +181,11 @@ def evaluate_forecasts_peer_accuracy( if gm.timestamp < actual_close_time ] + [actual_close_time] for i in range(len(times) - 1): - if interval_scores[i] is None: + interval_score = interval_scores[i] + if interval_score is None: continue interval_duration = times[i + 1] - times[i] - forecast_score += interval_scores[i] * interval_duration / total_duration + forecast_score += interval_score * interval_duration / total_duration forecast_coverage += interval_duration / total_duration forecast_scores.append(ForecastScore(forecast_score, forecast_coverage)) @@ -217,7 +193,7 @@ def evaluate_forecasts_peer_accuracy( def evaluate_forecasts_peer_spot_forecast( - forecasts: list[Forecast | AggregateForecast], + forecasts: Sequence[Forecast | AggregateForecast], base_forecasts: list[Forecast | AggregateForecast] | None, resolution_bucket: int, spot_forecast_timestamp: float, @@ -256,8 +232,8 @@ def evaluate_forecasts_peer_spot_forecast( def evaluate_forecasts_legacy_relative( - forecasts: list[Forecast | AggregateForecast], - base_forecasts: list[Forecast | AggregateForecast], + forecasts: Sequence[Forecast | AggregateForecast], + base_forecasts: Sequence[Forecast | AggregateForecast], resolution_bucket: int, forecast_horizon_start: float, actual_close_time: float, @@ -300,10 +276,11 @@ def evaluate_forecasts_legacy_relative( if bf.timestamp < actual_close_time ] + [actual_close_time] for i in range(len(times) - 1): - if interval_scores[i] is None: + interval_score = interval_scores[i] + if interval_score is None: continue interval_duration = times[i + 1] - times[i] - forecast_score += interval_scores[i] * interval_duration / total_duration + forecast_score += interval_score * interval_duration / total_duration forecast_coverage += interval_duration / total_duration forecast_scores.append(ForecastScore(forecast_score, forecast_coverage)) diff --git a/utils/the_math/aggregations.py b/utils/the_math/aggregations.py index 1ff6a98c01..40c0193de3 100644 --- a/utils/the_math/aggregations.py +++ b/utils/the_math/aggregations.py @@ -78,6 +78,8 @@ def get_histogram( weights = np.ones(len(values)) transposed_values = values.T if question_type == Question.QuestionType.BINARY: + if np.any(np.equal(values, None)): + raise ValueError("Forecast values contain None values") histogram = np.zeros(100) for p, w in zip(transposed_values[1], weights): histogram[int(p * 100)] += w @@ -85,7 +87,8 @@ def get_histogram( histogram = np.zeros((len(values[0]), 100)) for forecast_values, w in zip(values, weights): for i, p in enumerate(forecast_values): - histogram[i, int(p * 100)] += w + if p is not None: + histogram[i, int(p * 100)] += w return histogram @@ -100,13 +103,13 @@ def compute_discrete_forecast_values( if forecasts_values.shape[1] == 2: return weighted_percentile_2d( forecasts_values, weights=weights, percentiles=percentile - ).tolist() + ) # TODO: this needs to be normalized for MC, but special care needs to be taken # if the percentile isn't 50 (namely it needs to be normalized based off the values # at the median) return weighted_percentile_2d( forecasts_values, weights=weights, percentiles=percentile - ).tolist() + ) def compute_weighted_semi_standard_deviations( @@ -115,6 +118,7 @@ def compute_weighted_semi_standard_deviations( ) -> tuple[ForecastValues, ForecastValues]: """returns the upper and lower standard_deviations""" forecasts_values = np.array(forecasts_values) + forecasts_values[np.equal(forecasts_values, None)] = np.nan if weights is None: weights = np.ones(forecasts_values.shape[0]) average = np.average(forecasts_values, axis=0, weights=weights) @@ -439,46 +443,71 @@ def calculate_forecast_values( ) -> np.ndarray: # Default Aggregation method uses weighted medians for binary and MC questions # and weighted average for continuous + forecasts_values = np.array(forecast_set.forecasts_values) if self.question.type == Question.QuestionType.BINARY: + if np.any(np.equal(forecasts_values, None)): + raise ValueError("Forecast values contain None values") return np.array( - compute_discrete_forecast_values( - forecast_set.forecasts_values, weights, 50.0 - )[0] + compute_discrete_forecast_values(forecasts_values, weights, 50.0)[0] ) elif self.question.type == Question.QuestionType.MULTIPLE_CHOICE: - medians = np.array( - compute_discrete_forecast_values( - forecast_set.forecasts_values, weights, 50.0 - )[0] + arr = np.array( + compute_discrete_forecast_values(forecasts_values, weights, 50.0)[0] ) - floored_medians = medians - 0.001 - normalized_floored_medians = floored_medians / sum(floored_medians) - return normalized_floored_medians * (1 - len(medians) * 0.001) + 0.001 + non_nones = np.logical_not(np.equal(arr, None)) + arr[non_nones] -= 0.001 # remove minimum forecastable value + arr[non_nones] = arr[non_nones] / sum(arr[non_nones]) # renormalize + # squeeze into forecastable value range + arr[non_nones] = arr[non_nones] * (1 - len(arr[non_nones]) * 0.001) + 0.001 + return arr else: # continuous - return np.average(forecast_set.forecasts_values, axis=0, weights=weights) + if np.any(np.equal(forecasts_values, None)): + raise ValueError("Forecast values contain None values") + return np.average(forecasts_values, axis=0, weights=weights) def get_range_values( self, forecast_set: ForecastSet, aggregation_forecast_values: ForecastValues, weights: np.ndarray | None = None, - ): + ) -> tuple[list[float | None], list[float | None], list[float | None]]: if self.question.type == Question.QuestionType.BINARY: + forecasts_values = np.array(forecast_set.forecasts_values) + if np.any(np.equal(forecasts_values, None)): + raise ValueError("Forecast values contain None values") lowers, centers, uppers = compute_discrete_forecast_values( - forecast_set.forecasts_values, weights, [25.0, 50.0, 75.0] + forecasts_values, weights, [25.0, 50.0, 75.0] ) elif self.question.type == Question.QuestionType.MULTIPLE_CHOICE: + forecasts_values = np.array(forecast_set.forecasts_values) + non_nones = ( + np.logical_not(np.equal(forecasts_values[0], None)) + if forecasts_values.size + else [] + ) lowers, centers, uppers = compute_discrete_forecast_values( - forecast_set.forecasts_values, weights, [25.0, 50.0, 75.0] + forecasts_values, weights, [25.0, 50.0, 75.0] ) centers_array = np.array(centers) normalized_centers = np.array(aggregation_forecast_values) - normalized_lowers = np.array(lowers) * normalized_centers / centers_array - normalized_uppers = np.array(uppers) * normalized_centers / centers_array + normalized_lowers = np.array(lowers) + normalized_lowers[non_nones] = ( + normalized_lowers[non_nones] + * normalized_centers[non_nones] + / centers_array[non_nones] + ) + normalized_uppers = np.array(uppers) + normalized_uppers[non_nones] = ( + normalized_lowers[non_nones] + * normalized_centers[non_nones] + / centers_array[non_nones] + ) centers = normalized_centers.tolist() lowers = normalized_lowers.tolist() uppers = normalized_uppers.tolist() else: # continuous + if np.any(np.equal(aggregation_forecast_values, None)): + raise ValueError("Forecast values contain None values") lowers, centers, uppers = percent_point_function( aggregation_forecast_values, [25.0, 50.0, 75.0] ) @@ -496,7 +525,13 @@ class MeanAggregatorMixin: def calculate_forecast_values( self, forecast_set: ForecastSet, weights: np.ndarray | None = None ) -> np.ndarray: - return np.average(forecast_set.forecasts_values, axis=0, weights=weights) + forecasts_values = np.array(forecast_set.forecasts_values) + forecast_values = forecasts_values[0] if forecasts_values.size else np.array([]) + non_nones = np.logical_not(np.equal(forecast_values, None)) + forecast_values[non_nones] = np.average( + forecasts_values[:, non_nones], axis=0, weights=weights + ) + return forecast_values def get_range_values( self, @@ -505,6 +540,8 @@ def get_range_values( weights: np.ndarray | None = None, ): if self.question.type in QUESTION_CONTINUOUS_TYPES: + if np.any(np.equal(aggregation_forecast_values, None)): + raise ValueError("Forecast values contain None values") lowers, centers, uppers = percent_point_function( aggregation_forecast_values, [25.0, 50.0, 75.0] ) @@ -512,12 +549,18 @@ def get_range_values( centers = [centers] uppers = [uppers] else: - centers = aggregation_forecast_values + centers = np.array(aggregation_forecast_values) lowers_sd, uppers_sd = compute_weighted_semi_standard_deviations( forecast_set.forecasts_values, weights ) - lowers = (np.array(centers) - lowers_sd).tolist() - uppers = (np.array(centers) + uppers_sd).tolist() + non_nones = np.logical_not(np.equal(centers, None)) + lowers = centers.copy() + uppers = centers.copy() + lowers[non_nones] = centers[non_nones] - lowers_sd[non_nones] + uppers[non_nones] = centers[non_nones] + uppers_sd[non_nones] + lowers = lowers.tolist() + centers = centers.tolist() + uppers = uppers.tolist() return lowers, centers, uppers @@ -703,7 +746,12 @@ def get_aggregations_at_time( if len(forecasts) == 0: return dict() forecast_set = ForecastSet( - forecasts_values=[forecast.get_prediction_values() for forecast in forecasts], + forecasts_values=[ + [ + v or 0.0 for v in forecast.get_prediction_values() + ] # replace Nones with 0.0 for calculation purposes + for forecast in forecasts + ], timestep=time, forecaster_ids=[forecast.author_id for forecast in forecasts], timesteps=[forecast.start_time for forecast in forecasts], @@ -856,7 +904,7 @@ def minimize_history( def get_user_forecast_history( - forecasts: list[Forecast], + forecasts: Sequence[Forecast], minimize: bool = False, cutoff: datetime | None = None, ) -> list[ForecastSet]: @@ -929,11 +977,7 @@ def get_aggregation_history( cutoff = question.actual_close_time else: cutoff = min(timezone.now(), question.actual_close_time or timezone.now()) - forecast_history = get_user_forecast_history( - forecasts, - minimize, - cutoff=cutoff, - ) + forecast_history = get_user_forecast_history(forecasts, minimize, cutoff=cutoff) forecaster_ids = set(forecast.author_id for forecast in forecasts) for method in aggregation_methods: diff --git a/utils/the_math/measures.py b/utils/the_math/measures.py index e1f2289a50..e20bd381be 100644 --- a/utils/the_math/measures.py +++ b/utils/the_math/measures.py @@ -1,6 +1,6 @@ import numpy as np -from questions.models import Question, AggregateForecast +from questions.models import AggregateForecast, Question from questions.types import Direction from utils.the_math.formulas import unscaled_location_to_scaled_location from utils.typing import ( @@ -25,6 +25,8 @@ def weighted_percentile_2d( percentiles = np.array(percentiles or [50.0]) sorted_values = values.copy() # avoid side effects + # replace None with -1.0 for calculations (return to None at the end) + sorted_values[np.equal(sorted_values, None)] = -1.0 sorted_values.sort(axis=0) # get the normalized cumulative weights @@ -50,11 +52,16 @@ def weighted_percentile_2d( + sorted_values[right_indexes, column_indicies] ) ) - return np.array(weighted_percentiles) + # replace -1.0 back to None + weighted_percentiles = np.array(weighted_percentiles) + weighted_percentiles = np.where( + weighted_percentiles == -1.0, None, weighted_percentiles + ) + return weighted_percentiles.tolist() def percent_point_function( - cdf: ForecastValues, percentiles: Percentiles | float | int + cdf: ForecastValues, percentiles: Percentiles | list[float] | float | int ) -> Percentiles: """returns the x-location in the cdf where it crosses the given percentiles, treating the cdf as starting at x=0 and ending at x=1 @@ -66,6 +73,8 @@ def percent_point_function( if return_float := isinstance(percentiles, float | int): percentiles = np.array([percentiles]) ppf_values = [] + if any(v is None for v in cdf): + raise ValueError("cdf contains None values") for percent in percentiles: # percent is a float between 0 and 100 if percent < cdf[0] * 100: @@ -95,6 +104,8 @@ def prediction_difference_for_sorting( """for binary and multiple choice, takes pmfs for continuous takes cdfs""" p1, p2 = np.array(p1), np.array(p2) + p1[np.equal(p1, None)] = -1.0 # replace None with -1.0 for calculations + p2[np.equal(p2, None)] = -1.0 # replace None with -1.0 for calculations # Uses Jeffrey's Divergence if question_type in ["binary", "multiple_choice"]: return sum([(p - q) * np.log2(p / q) for p, q in zip(p1, p2)]) @@ -111,6 +122,9 @@ def prediction_difference_for_display( ) -> list[tuple[float, float]]: """for binary and multiple choice, takes pmfs for continuous takes cdfs""" + p1, p2 = np.array(p1), np.array(p2) + p1[np.equal(p1, None)] = -1.0 # replace None with -1.0 for calculations + p2[np.equal(p2, None)] = -1.0 # replace None with -1.0 for calculations if question.type == "binary": # single-item list of (pred diff, ratio of odds) return [(p2[1] - p1[1], (p2[1] / (1 - p2[1])) / (p1[1] / (1 - p1[1])))] @@ -168,7 +182,7 @@ def to_direction_and_magnitude(diff: float) -> tuple[Direction, float]: return [to_direction_and_magnitude(asymmetry)] # Otherwise, compare spread of prediction intervals - def get_scaled_interval(forecast): + def get_scaled_interval(forecast: AggregateForecast): # Try to use precomputed bounds lower = (forecast.interval_lower_bounds or [None])[0] upper = (forecast.interval_upper_bounds or [None])[0] diff --git a/utils/typing.py b/utils/typing.py index ffc7404400..eef43b620c 100644 --- a/utils/typing.py +++ b/utils/typing.py @@ -1,7 +1,7 @@ import numpy as np from typing import TypeAlias -ForecastValues = np.ndarray | list[float] # values from a single forecast -ForecastsValues = np.ndarray | list[list[float]] # values from multiple forecasts +ForecastValues = np.ndarray | list[float | None] # values from a single forecast +ForecastsValues = np.ndarray | list[ForecastValues] # values from multiple forecasts Weights = np.ndarray | None Percentiles: TypeAlias = np.ndarray