Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion python/packages/isce3/focus/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@
from .sar_duration import (get_sar_duration, get_radar_velocities,
predict_azimuth_envelope)
from .valid_regions import (RadarPoint, RadarBoundingBox,
get_focused_sub_swaths, fill_gaps)
get_focused_sub_swaths, fill_gaps, find_bad_rangline_slices)
from .calibration_luts import make_los_luts, make_cal_luts
from .notch import Notch, FrequencyDomain
57 changes: 57 additions & 0 deletions python/packages/isce3/focus/valid_regions.py
Original file line number Diff line number Diff line change
Expand Up @@ -549,6 +549,10 @@ def get_gap_slices(pulse_swaths: np.ndarray) -> Iterator[slice]:
pulse_swaths = np.asarray(
[swath for swath in pulse_swaths if swath[1] > swath[0]]
)
# If there's no valid data then the whole rangeline is a "gap".
if len(pulse_swaths) == 0:
yield slice(None)
return
Comment on lines +552 to +555

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Even though in reality and during warmup the entire pulse is not invalid but first 2/3rd and then 1/3rd is missing, looks like we have decided to mark all invalid in L0B. Anyways that is an upstream discussion which we already had. So that is ok to give up on the warm up part in RSLC.
Out of curiosity what happens if the entire L0B has a mask that marks all pulses invalid? Does focus detect and complain, or with this minor change we are letting that go through?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess that's true about the first 1/3rd of the swath. We'd want to test that EAP does the right thing in that case.

If all pulses are marked completely invalid then yes processing would continue and the result would be an image of all zeros. Similarly, a block of 5 seconds of missing data would no longer be a fatal error. That's why I added the extra logging so there's at least some visibility into those sorts of edge cases.

The L0B .met file contains information about missing rangelines, so PCM could implement a rule to avoid RLSC processing stuff with missing data.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree that it is the responsibility of job manager (PCM) to check if we want a threshold for not submitting job. I think it is actually a good thing to let data be processed even if there is 5 sec or whatever gap in the data. Again somebody else's job (e.g, QA) to accept the partial SLC or not!

num_swaths = pulse_swaths.shape[0]
# Gap leading up to first swath.
yield slice(None, pulse_swaths[0, 0])
Expand All @@ -570,3 +574,56 @@ def get_gap_slices(pulse_swaths: np.ndarray) -> Iterator[slice]:
for ipulse in range(num_pulses):
for gap in get_gap_slices(swaths[:, ipulse, :]):
data[..., ipulse, gap] = value


def false_runs(arr: np.ndarray) -> list[slice]:
"""
Return a list of slices selecting continuous runs of False values.

Parameters
----------
arr : np.ndarray
A 1D boolean NumPy array.

Returns
-------
list[slice]
A list of slice objects, each selecting a contiguous run of False values.
"""
if arr.ndim != 1:
raise ValueError("Array must be 1D")

# Pad with True on both ends so edges are detected as transitions
padded = np.concatenate(([True], arr, [True]))

# Find where transitions occur
diff = np.diff(padded.view(np.int8))

# A False run starts where True→False (diff == -1), ends where False→True (diff == 1)
starts = np.where(diff == -1)[0] # index in original array
ends = np.where(diff == 1)[0] # exclusive end in original array

return [slice(s, e) for s, e in zip(starts, ends)]


def find_bad_rangline_slices(swaths):
"""
Get slices corresponding to completely invalid rangelines.

Parameters
----------
swaths : numpy.ndarray[int]
Array of [start, stop) valid data regions, shape = (nswath, npulse, 2)
where nswath is the number of valid sub-swaths and npulse is the length
of the raw image grid.

Returns
-------
num_invalid : int
Number of completely invalid ranglines.
slices : list[slice]
Azimuth index slices where all samples in the rangelines are invalid.
"""
has_some_valid_data = np.any(swaths[..., 0] < swaths[..., 1], axis=0)
num_invalid = swaths.shape[1] - np.sum(has_some_valid_data)
return num_invalid, false_runs(has_some_valid_data)
14 changes: 13 additions & 1 deletion python/packages/nisar/workflows/focus.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@
import numpy as np
import isce3
from isce3.core import DateTime, TimeDelta, LUT2d, Attitude, Orbit
from isce3.focus import make_los_luts, fill_gaps, make_cal_luts, Notch
from isce3.focus import (make_los_luts, fill_gaps, make_cal_luts, Notch,
find_bad_rangline_slices)
from isce3.geometry import los2doppler
from isce3.io.gdal import Raster, GDT_CFloat32
from isce3.product import (RadarGridParameters,
Expand Down Expand Up @@ -1750,6 +1751,16 @@ def get_caltone_algorithm(cfg, fc, fs, n, is_dithered):

return algorithm, wavelets


def log_bad_pulses(swaths, max_slices=10):

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume the purpose of the max_slices is to avoid crowding the log/screen with too many bad range lines.

n_bad_pulses, bad_slices = find_bad_rangline_slices(swaths)
log.info(f"Number of pulses with no valid samples = {n_bad_pulses}")
if n_bad_pulses > 0:
log.warning(f"Bad pulses appear in {len(bad_slices)} unique blocks")
for s in bad_slices[:max_slices]:
log.warning(f"Bad ranglines in pulse {s}")


def focus(runconfig, runconfig_path=""):
# Strip off two leading namespaces.
cfg = runconfig.runconfig.groups
Expand Down Expand Up @@ -2050,6 +2061,7 @@ def temp(suffix):
swaths = raw.getSubSwaths(channel_in.freq_id, tx=pol[0])
swaths = swaths[:, pulse_begin:pulse_end, :]
log.info(f"Number of sub-swaths = {swaths.shape[0]}")
log_bad_pulses(swaths)

rawfd = temp(f"_{frequency}{pol}_raw.c8")
log.info(f"Decoding raw data to memory map {rawfd.name}.")
Expand Down
Loading