[MSD-350][feat] Ensure pyramidal tiff on import

src/odemis/dataio/test/tiff_test.py

@@ -25,11 +25,11 @@
 import logging
 import os
 import re
+import tempfile
 import time
 import unittest
 import xml.etree.ElementTree as ET
 from datetime import datetime
-from unittest.case import skip
 
 import libtiff
 import libtiff.libtiff_ctypes as T  # for the constant names
@@ -47,6 +47,43 @@
 logging.getLogger().setLevel(logging.DEBUG)
 
 FILENAME = "test" + tiff.EXTENSIONS[0]
+
+
+class TestTiffPyramidalIO(unittest.TestCase):
+
+    def setUp(self) -> None:
+        self.temp_dir = tempfile.mkdtemp()
+
+    def tearDown(self) -> None:
+        try:
+            for filename in os.listdir(self.temp_dir):
+                file_path = os.path.join(self.temp_dir, filename)
+                if os.path.isfile(file_path):
+                    os.remove(file_path)
+            os.rmdir(self.temp_dir)
+        except Exception:
+            pass
+
+    def _create_test_tiff(self, filename: str, pyramid: bool = False) -> str:
+        array = numpy.random.randint(0, 255, (256, 256), dtype=numpy.uint8)
+        data = model.DataArray(array)
+        full_path = os.path.join(self.temp_dir, filename)
+        tiff.export(full_path, data, pyramid=pyramid, imagej=True)
+        return full_path
+
+    def test_is_pyramidal_tiff(self) -> None:
+        pyramidal_file = self._create_test_tiff("pyramidal.tif", pyramid=True)
+        self.assertTrue(tiff.is_pyramidal(pyramidal_file))
+
+    def test_is_pyramidal_non_pyramidal_tiff(self) -> None:
+        non_pyramidal_file = self._create_test_tiff("non_pyramidal.tif", pyramid=False)
+        self.assertFalse(tiff.is_pyramidal(non_pyramidal_file))
+
+    def test_is_pyramidal_invalid_file(self) -> None:
+        with self.assertRaises(IOError):
+            tiff.is_pyramidal("/nonexistent/file.tif")
+
+
 class TestTiffIO(unittest.TestCase):
 
     def tearDown(self):

src/odemis/dataio/tiff.py

@@ -2401,6 +2401,25 @@ def read_data(filename):
     return [acd.content[n].getData() for n in range(len(acd.content))]
 
 
+def is_pyramidal(filename: str) -> bool:
+    """
+    Check whether a TIFF file has SubIFD pyramid levels.
+
+    :param filename: Path to TIFF file
+    :return: True when TIFFTAG_SUBIFD contains one or more entries
+    :raises IOError: If the file cannot be opened as TIFF
+    """
+    try:
+        tiff_file = TIFF.open(filename, mode="r")
+        try:
+            sub_ifds = tiff_file.GetField(T.TIFFTAG_SUBIFD)
+            return sub_ifds is not None and len(sub_ifds) > 0
+        finally:
+            tiff_file.close()
+    except Exception as exc:
+        raise IOError("Failed to inspect TIFF pyramid status for %s: %s" % (filename, exc))
+
+
 def read_thumbnail(filename):
     """
     Read the thumbnail data of a given TIFF file.

src/odemis/gui/cont/tabs/_constants.py

@@ -34,3 +34,5 @@
 MIRROR_PARKED = 1
 MIRROR_BAD = 2  # not parked, but not fully engaged either
 MIRROR_ENGAGED = 3
+
+PYRAMIDAL_CONVERSION_SUFFIX = "_pyramidal"

src/odemis/gui/cont/tabs/analysis_tab.py

@@ -27,6 +27,8 @@
 import gc
 import logging
 import os.path
+from pathlib import Path
+from typing import Union
 import wx
 # IMPORTANT: wx.html needs to be imported for the HTMLWindow defined in the XRC
 # file to be correctly identified. See: http://trac.wxwidgets.org/ticket/3626
@@ -55,11 +57,19 @@
     AngularSpectrumViewport, ThetaViewport
 from odemis.gui.conf import get_acqui_conf
 from odemis.gui.cont import settings
+from odemis.gui.cont.tabs import PYRAMIDAL_CONVERSION_SUFFIX
 from odemis.gui.cont.tabs.tab import Tab
 from odemis.gui.model import TOOL_POINT, TOOL_LINE, TOOL_ACT_ZOOM_FIT, TOOL_RULER, TOOL_LABEL, \
     TOOL_NONE
 from odemis.gui.util import call_in_wx_main
-from odemis.util.dataio import data_to_static_streams, open_acquisition, open_files_and_stitch
+from odemis.util.dataio import (
+    data_to_static_streams,
+    export_data_as_pyramidal,
+    open_acquisition,
+    open_files_and_stitch,
+    is_pyramidal,
+    convert_file_to_pyramidal,
+)
 
 
 class AnalysisTab(Tab):
@@ -93,6 +103,9 @@ def __init__(self, name, button, panel, main_frame, main_data):
         # displayed
         tab_data = guimod.AnalysisGUIData(main_data)
         super(AnalysisTab, self).__init__(name, button, panel, main_frame, tab_data)
+
+        self.main_data = main_data
+
         if main_data.role in ("sparc-simplex", "sparc", "sparc2"):
             # Different name on the SPARC to reflect the slightly different usage
             self.set_label("ANALYSIS")
@@ -318,13 +331,52 @@ def _on_add_file(self):
     def _on_add_tileset(self):
         self.select_acq_file(extend=True, tileset=True)
 
+    def _get_pyramidal_path(self, filename: Union[str, Path]):
+        """
+        Returns the output path for a given input file, used in the pyramidal conversion process.
+
+        :param filename: The input filename of the file that is to be converted
+        :return: The adjusted output path
+        """
+        filename = Path(filename)
+        name_converted = f"{filename.stem}{PYRAMIDAL_CONVERSION_SUFFIX}.tif"
+
+        project_folder = (
+            None
+            if getattr(self.main_data, "is_viewer", False)
+            else self.main_data.project_path.value
+        )
+
+        if project_folder:
+            return Path(project_folder) / name_converted
+        else:
+            # For simplicity, always convert to tiff format
+            return filename.with_name(name_converted)
+
     def load_tileset(self, filenames, extend=False):
-        data = open_files_and_stitch(filenames) # TODO: allow user defined registration / weave methods
-        self.display_new_data(filenames[0], data, extend=extend)
+        data = open_files_and_stitch(filenames)  # TODO: allow user defined registration / weave methods
+        # For the lack of a better name, use the name of the first tile
+        source_filename = filenames[0]
+        filename_converted = self._get_pyramidal_path(source_filename)
+        try:
+            export_data_as_pyramidal(data, filename_converted, compressed=True)
+            self.display_new_data(filename_converted, open_acquisition(filename_converted), extend=extend)
+        except Exception:
+            logging.exception("Failed to export/reopen stitched pyramidal tileset")
+            self.display_new_data(filenames[0], data, extend=extend)
 
     def load_data(self, filename, fmt=None, extend=False):
-        data = open_acquisition(filename, fmt)
-        self.display_new_data(filename, data, extend=extend)
+        filename = Path(filename)
+        if is_pyramidal(filename):
+            self.display_new_data(filename, open_acquisition(filename, fmt), extend=extend)
+            return
+        filename_converted = self._get_pyramidal_path(filename)
+        try:
+            convert_file_to_pyramidal(filename, filename_converted, compressed=True)
+            self.display_new_data(filename_converted, open_acquisition(filename_converted, fmt), extend=extend)
+        except Exception as ex:
+            logging.exception("Failed to process TIFF import, loading original. Error: %s", ex)
+            self.display_new_data(filename, open_acquisition(filename, fmt), extend=extend)
 
     def _get_time_spectrum_streams(self, spec_streams):
         """
@@ -358,6 +410,7 @@ def display_new_data(self, filename, data, extend=False):
         extend (bool): if False, will ensure that the previous streams are closed.
           If True, will add the new file to the current streams opened.
         """
+        filename = str(filename)
         if not extend:
             # Remove all the previous streams
             self._stream_bar_controller.clear()

src/odemis/gui/cont/tabs/correlation_tab.py

@@ -25,26 +25,33 @@
 
 import collections
 import logging
-import os.path
-from typing import List, Optional
+import os
+from pathlib import Path
+from typing import List, Optional, Union
 import wx
-
 from odemis.gui import conf
-
 from odemis import dataio
 from odemis import model
 import odemis.gui
 import odemis.gui.cont.export as exportcont
-from odemis.gui.cont.stream_bar import StreamBarController
 import odemis.gui.cont.views as viewcont
 import odemis.gui.model as guimod
 import odemis.gui.util as guiutil
 from odemis.acq.stream import OpticalStream, EMStream, StaticStream
 from odemis.gui.cont.correlation import CorrelationController
+from odemis.gui.cont.stream_bar import StreamBarController
+from odemis.gui.cont.tabs import PYRAMIDAL_CONVERSION_SUFFIX
+from odemis.gui.cont.tabs.tab import Tab
 from odemis.gui.model import TOOL_ACT_ZOOM_FIT
 from odemis.gui.util import call_in_wx_main
-from odemis.gui.cont.tabs.tab import Tab
-from odemis.util.dataio import data_to_static_streams, open_acquisition, open_files_and_stitch
+from odemis.util.dataio import (
+    convert_file_to_pyramidal,
+    data_to_static_streams,
+    export_data_as_pyramidal,
+    is_pyramidal,
+    open_acquisition,
+    open_files_and_stitch,
+)
 
 
 class CorrelationTab(Tab):
@@ -166,13 +173,53 @@ def _on_add_file(self) -> None:
     def _on_add_tileset(self) -> None:
         self.select_acq_file(extend=True, tileset=True)
 
-    def load_tileset(self, filenames: List[str], extend: bool = False) -> None:
-        data = open_files_and_stitch(filenames) # TODO: allow user defined registration / weave methods
-        self.load_streams(data)
+    def _get_pyramidal_path(self, filename: Union[str, Path]):
+        """
+        Returns the output path for a given input file, used in the pyramidal conversion process.
 
-    def load_data(self, filename: str, fmt: str = None, extend: bool = False) -> None:
-        data = open_acquisition(filename, fmt)
-        self.load_streams(data)
+        :param filename: The input filename of the file that is to be converted
+        :return: The adjusted output path
+        """
+        filename = Path(filename)
+        name_converted = f"{filename.stem}{PYRAMIDAL_CONVERSION_SUFFIX}.tif"
+
+        project_folder = (
+            None
+            if getattr(self.main_data, "is_viewer", False)
+            else self.main_data.project_path.value
+        )
+
+        if project_folder:
+            return Path(project_folder) / name_converted
+        else:
+            # For simplicity, always convert to tiff format
+            return filename.with_name(name_converted)
+
+    def load_tileset(self, filenames: List[str], extend: bool = False) -> None:
+        data = open_files_and_stitch(filenames)  # TODO: allow user defined registration / weave methods
+        # For the lack of a better name, use the name of the first tile
+        source_filename = filenames[0]
+        filename_converted = self._get_pyramidal_path(source_filename)
+        try:
+            export_data_as_pyramidal(data, filename_converted, compressed=True)
+            self.load_streams(open_acquisition(filename_converted))
+        except Exception:
+            logging.exception("Failed to export/reopen stitched pyramidal tileset")
+            self.load_streams(data)
+
+
+    def load_data(self, filename: Union[str, Path], fmt: Optional[str] = None, extend: bool = False) -> None:
+        filename = Path(filename)
+        if is_pyramidal(filename):
+            self.load_streams(open_acquisition(filename, fmt))
+            return
+        filename_converted = self._get_pyramidal_path(filename)
+        try:
+            convert_file_to_pyramidal(filename, filename_converted, compressed=True)
+            self.load_streams(open_acquisition(filename_converted, fmt))
+        except Exception as ex:
+            logging.exception("Failed to process TIFF import, loading original. Error: %s", ex)
+            self.load_streams(open_acquisition(filename, fmt))
 
     def select_acq_file(self, extend: bool = False, tileset: bool = False):
         """ Open an image file using a file dialog box

src/odemis/util/dataio.py

@@ -21,8 +21,9 @@
 """
 import json
 import logging
+from pathlib import Path
 import os
-from typing import Any, Optional
+from typing import Any, List, Optional, Union
 
 import numpy
 
@@ -199,15 +200,16 @@ def _split_planes(data):
     return data
 
 
-def open_acquisition(filename, fmt=None):
+def open_acquisition(filename, fmt=None) -> List[Union[model.DataArray, model.DataArrayShadow]]:
     """
     Opens the data according to the type of file, and returns the opened data.
     If it's a pyramidal image, do not fetch the whole data from the image. If the image
     is not pyramidal, it reads the entire image and returns it
     filename (string): Name of the file where the image is
     fmt (string): The format of the file
-    return (list of DataArrays or DataArrayShadows): The opened acquisition source
+    return: The opened acquisition source
     """
+    filename = str(filename)
     if fmt:
         converter = dataio.get_converter(fmt)
     else:
@@ -225,6 +227,60 @@ def open_acquisition(filename, fmt=None):
     return data
 
 
+def is_pyramidal(filename: Union[str, Path]) -> bool:
+    """
+    Check whether a file is in pyramidal format.
+
+    :param filename: Path to file
+    :return: True if the file is pyramidal, False otherwise
+    :raises IOError: If the file cannot be read or converter doesn't support pyramid detection
+    """
+    reader = dataio.find_fittest_converter(str(filename), mode=os.O_RDONLY)
+    if not hasattr(reader, "is_pyramidal"):
+        return False
+
+    return reader.is_pyramidal(str(filename))
+
+
+def export_data_as_pyramidal(
+    data: Union[model.DataArray, List[model.DataArray]],
+    dst_filename: Union[str, Path],
+    compressed: bool = True,
+) -> None:
+    """
+    Export provided data to a pyramidal file format selected by destination extension.
+
+    :param data: Array that holds image data
+    :param dst_filename: Destination file path
+    :param compressed: Whether to request compressed output
+    """
+    dst_filename = str(dst_filename)
+    exporter = dataio.find_fittest_converter(dst_filename, mode=os.O_WRONLY)
+    if not getattr(exporter, "CAN_SAVE_PYRAMID", False):
+        raise IOError(
+            "Destination format %s does not support pyramidal export"
+            % (getattr(exporter, "FORMAT", type(exporter).__name__),)
+        )
+    exporter.export(dst_filename, data, compressed=compressed, pyramid=True)
+
+
+def convert_file_to_pyramidal(src_filename: Union[str, Path], dst_filename: Union[str, Path], compressed: bool = True) -> None:
+    """
+    Convert any readable acquisition file into a pyramidal file format.
+
+    The source is read via the source converter's read_data API. The destination
+    converter is selected from dst_filename and must support pyramidal export.
+
+    :param src_filename: Source file path
+    :param dst_filename: Destination file path
+    :param compressed: Whether to request compressed output
+    :raises IOError: If reading/exporting fails or destination doesn't support pyramids
+    """
+    reader = dataio.find_fittest_converter(str(src_filename), mode=os.O_RDONLY)
+    data = reader.read_data(str(src_filename))
+    export_data_as_pyramidal(data, dst_filename, compressed)
+
+
 def splitext(path):
     """
     Split a pathname into basename + ext (.XXX).