make tides resilient to flaky NOAA upstream

NOAA's API intermittently fails, which surfaced as a dead-end "Failed to load
NOAA data" screen. harden it:
- noaa: retry transient failures, and serve last-known-good extrema when NOAA is
  unavailable (stale-if-error) — tide predictions change very slowly
- bootstrap: gather stations with return_exceptions and drop any that fail, so
  one bad station no longer sinks the whole dashboard
- frontend: auto-retry the bootstrap a few times, then show a manual retry button
  instead of a terminal error

Author: Sean-Michael

app/tides/noaa.py

@@ -28,37 +28,61 @@ class Extremum:
     kind: str  # "H" | "L"
 
 
-_CACHE: dict[tuple[str, str, str], tuple[float, list[Extremum]]] = {}
+# Keyed by station id (the begin/end window is always ~now±20d, so the last
+# successful fetch stays usable). Holds the last-known-good extrema indefinitely
+# so we can serve stale data when NOAA is flaky (predictions change very slowly).
+_CACHE: dict[str, tuple[float, list[Extremum]]] = {}
+
+_RETRIES = 2  # extra attempts on transient NOAA failures
 
 
 def _fetch_blocking(station_id: str, begin: str, end: str) -> list[Extremum]:
-    station = CoopsStation(id=station_id)
-    df = station.get_data(
-        begin_date=begin,
-        end_date=end,
-        product="predictions",
-        datum="MLLW",
-        units="english",
-        time_zone="lst_ldt",
-        interval="hilo",
-    )
-    out: list[Extremum] = []
-    for ts, row in df.iterrows():
-        out.append(Extremum(t=ts.to_pydatetime(), height=float(row["v"]), kind=str(row["type"])))
-    return out
+    last_err: Exception | None = None
+    for attempt in range(_RETRIES + 1):
+        try:
+            station = CoopsStation(id=station_id)
+            df = station.get_data(
+                begin_date=begin,
+                end_date=end,
+                product="predictions",
+                datum="MLLW",
+                units="english",
+                time_zone="lst_ldt",
+                interval="hilo",
+            )
+            out: list[Extremum] = []
+            for ts, row in df.iterrows():
+                out.append(
+                    Extremum(t=ts.to_pydatetime(), height=float(row["v"]), kind=str(row["type"]))
+                )
+            return out
+        except Exception as err:  # noqa: BLE001 — retry any upstream hiccup
+            last_err = err
+            if attempt < _RETRIES:
+                time.sleep(0.6 * (attempt + 1))
+    assert last_err is not None
+    raise last_err
 
 
 async def fetch_extrema(station_id: str, begin: str, end: str) -> list[Extremum]:
-    """Return real NOAA hi/lo predictions for ``[begin, end]`` (YYYYMMDD)."""
-    key = (station_id, begin, end)
+    """Return real NOAA hi/lo predictions for ``[begin, end]`` (YYYYMMDD).
+
+    Serves fresh data within the TTL, refetches after, and falls back to the
+    last-known-good extrema if NOAA is unavailable (stale-if-error).
+    """
     now = time.monotonic()
-    cached = _CACHE.get(key)
+    cached = _CACHE.get(station_id)
     if cached is not None and now - cached[0] < CACHE_TTL:
         return cached[1]
 
-    extrema = await asyncio.to_thread(_fetch_blocking, station_id, begin, end)
-    _CACHE[key] = (now, extrema)
-    return extrema
+    try:
+        extrema = await asyncio.to_thread(_fetch_blocking, station_id, begin, end)
+        _CACHE[station_id] = (now, extrema)
+        return extrema
+    except Exception:
+        if cached is not None:
+            return cached[1]  # stale-if-error
+        raise
 
 
 def clear_cache() -> None:

app/tides/router.py

@@ -62,34 +62,39 @@ async def _station_payload(s: stations.Station, begin: str, end: str) -> dict[st
 @router.get("/bootstrap")
 async def bootstrap() -> dict[str, Any]:
     """Everything the dashboard loads on startup, in one round trip."""
-    try:
-        now = datetime.now(_TZ)
-        begin, end = _date_range(now)
-        payloads = await asyncio.gather(
-            *(_station_payload(s, begin, end) for s in stations.STATIONS)
-        )
-        sm = astro.sun_moon(now.date())
-        return {
-            "now": _epoch_ms(now),
-            "tz": "America/Los_Angeles",
-            "datum": "MLLW",
-            "units": "english",
-            "date": {
-                "iso": now.strftime("%Y-%m-%d"),
-                "pretty": now.strftime("%a · %b %-d, %Y"),
-            },
-            "sun_moon": {
-                "sunrise": sm.sunrise,
-                "sunset": sm.sunset,
-                "noon": sm.noon,
-                "moon_phase": sm.moon_phase,
-                "moon_illum": sm.moon_illum,
-                "moon_glyph": sm.moon_glyph,
-            },
-            "stations": list(payloads),
-        }
-    except Exception as exc:  # noqa: BLE001 — surface upstream NOAA failures cleanly
-        raise HTTPException(status_code=502, detail=f"NOAA upstream error: {exc}") from exc
+    now = datetime.now(_TZ)
+    begin, end = _date_range(now)
+    # Don't let one flaky station sink the whole dashboard — gather, then drop
+    # any that failed (each station already falls back to last-known-good data).
+    results = await asyncio.gather(
+        *(_station_payload(s, begin, end) for s in stations.STATIONS),
+        return_exceptions=True,
+    )
+    good = [r for r in results if not isinstance(r, BaseException)]
+    if not good:
+        first = next((r for r in results if isinstance(r, BaseException)), None)
+        raise HTTPException(status_code=502, detail=f"NOAA upstream error: {first}")
+
+    sm = astro.sun_moon(now.date())
+    return {
+        "now": _epoch_ms(now),
+        "tz": "America/Los_Angeles",
+        "datum": "MLLW",
+        "units": "english",
+        "date": {
+            "iso": now.strftime("%Y-%m-%d"),
+            "pretty": now.strftime("%a · %b %-d, %Y"),
+        },
+        "sun_moon": {
+            "sunrise": sm.sunrise,
+            "sunset": sm.sunset,
+            "noon": sm.noon,
+            "moon_phase": sm.moon_phase,
+            "moon_illum": sm.moon_illum,
+            "moon_glyph": sm.moon_glyph,
+        },
+        "stations": good,
+    }
 
 
 @router.get("/widget")

frontend/src/tides/App.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useState } from 'react'
+import { useCallback, useEffect, useRef, useState } from 'react'
 import { fetchBootstrap } from './api'
 import type { Bootstrap } from './types'
 import { Dashboard } from './Dashboard'
@@ -18,27 +18,58 @@ function useIsMobile(): boolean {
   return isMobile
 }
 
+// NOAA's API is occasionally flaky; auto-retry a few times before giving up.
+const MAX_AUTO_RETRIES = 3
+
 export default function App() {
   const [data, setData] = useState<Bootstrap | null>(null)
   const [error, setError] = useState<string | null>(null)
   const isMobile = useIsMobile()
+  const attempt = useRef(0)
 
-  useEffect(() => {
-    const ctrl = new AbortController()
-    fetchBootstrap(ctrl.signal)
-      .then(setData)
+  const load = useCallback((signal?: AbortSignal) => {
+    setError(null)
+    fetchBootstrap(signal)
+      .then((d) => {
+        attempt.current = 0
+        setData(d)
+      })
       .catch((e) => {
-        if (e.name !== 'AbortError') setError(String(e.message ?? e))
+        if (e.name === 'AbortError') return
+        if (attempt.current < MAX_AUTO_RETRIES) {
+          attempt.current += 1
+          setTimeout(() => load(signal), 1200 * attempt.current)
+        } else {
+          setError(String(e.message ?? e))
+        }
       })
-    return () => ctrl.abort()
   }, [])
 
+  useEffect(() => {
+    const ctrl = new AbortController()
+    load(ctrl.signal)
+    return () => ctrl.abort()
+  }, [load])
+
   if (error) {
     return (
       <div className="bto-splash">
         <div className="bto-splash-mark">◆</div>
         <div className="bto-splash-title">Bellingham Tidal Observatory</div>
-        <div className="bto-splash-err">Failed to load NOAA data — {error}</div>
+        <div className="bto-splash-err">Couldn’t reach NOAA — {error}</div>
+        <button className="bto-splash-retry" onClick={() => { attempt.current = 0; load() }}>
+          retry
+        </button>
+      </div>
+    )
+  }
+
+  if (!data) {
+    return (
+      <div className="bto-splash">
+        <div className="bto-splash-mark">◆</div>
+        <div className="bto-splash-title">Bellingham Tidal Observatory</div>
+        <div className="bto-splash-sub">fetching noaa predictions…</div>
       </div>
     )
   }

frontend/src/tides/styles/tides.css

@@ -900,3 +900,18 @@
   cursor: pointer;
 }
 .tic-reset:hover { border-color: rgba(150, 195, 235, 0.6); }
+
+.bto-splash-retry {
+  margin-top: 16px;
+  appearance: none;
+  background: rgba(8, 16, 30, 0.6);
+  border: 1px solid rgba(150, 195, 235, 0.4);
+  color: var(--ice-100, #eaf4ff);
+  border-radius: 6px;
+  padding: 8px 18px;
+  font-family: var(--mono);
+  font-size: 12px;
+  letter-spacing: 0.04em;
+  cursor: pointer;
+}
+.bto-splash-retry:hover { border-color: rgba(150, 195, 235, 0.7); }

tests/test_tides.py

@@ -47,6 +47,34 @@ def test_widget_payload(client: TestClient, fake_noaa: None) -> None:
     assert {"t", "v"} <= body["samples"][0].keys()
 
 
+def test_fetch_extrema_serves_stale_on_error(monkeypatch: pytest.MonkeyPatch) -> None:
+    import asyncio
+
+    from app.tides import noaa as noaa_mod
+
+    noaa_mod.clear_cache()
+    calls = {"n": 0}
+    good = [noaa.Extremum(t=datetime(2026, 5, 27, 1, 0), height=8.0, kind="H")]
+
+    def fake_blocking(station_id: str, begin: str, end: str) -> list[noaa.Extremum]:
+        calls["n"] += 1
+        if calls["n"] == 1:
+            return good
+        raise RuntimeError("NOAA down")
+
+    monkeypatch.setattr(noaa_mod, "_fetch_blocking", fake_blocking)
+    monkeypatch.setattr(noaa_mod, "CACHE_TTL", 0)  # force a refetch on the 2nd call
+
+    async def run() -> tuple[list[noaa.Extremum], list[noaa.Extremum]]:
+        first = await noaa_mod.fetch_extrema("9449211", "20260501", "20260601")
+        second = await noaa_mod.fetch_extrema("9449211", "20260501", "20260601")
+        return first, second
+
+    first, second = asyncio.run(run())
+    assert first == good
+    assert second == good  # stale-if-error: returns last-known-good despite failure
+
+
 def test_bootstrap_shape(client: TestClient, fake_noaa: None) -> None:
     res = client.get("/api/tides/bootstrap")
     assert res.status_code == 200