diff --git a/src/odemis/dataio/test/tiff_test.py b/src/odemis/dataio/test/tiff_test.py index 12a40eda9b..b80444f4f1 100644 --- a/src/odemis/dataio/test/tiff_test.py +++ b/src/odemis/dataio/test/tiff_test.py @@ -25,11 +25,11 @@ import logging import os import re +import tempfile import time import unittest import xml.etree.ElementTree as ET from datetime import datetime -from unittest.case import skip import libtiff import libtiff.libtiff_ctypes as T # for the constant names @@ -47,6 +47,43 @@ logging.getLogger().setLevel(logging.DEBUG) FILENAME = "test" + tiff.EXTENSIONS[0] + + +class TestTiffPyramidalIO(unittest.TestCase): + + def setUp(self) -> None: + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self) -> None: + try: + for filename in os.listdir(self.temp_dir): + file_path = os.path.join(self.temp_dir, filename) + if os.path.isfile(file_path): + os.remove(file_path) + os.rmdir(self.temp_dir) + except Exception: + pass + + def _create_test_tiff(self, filename: str, pyramid: bool = False) -> str: + array = numpy.random.randint(0, 255, (256, 256), dtype=numpy.uint8) + data = model.DataArray(array) + full_path = os.path.join(self.temp_dir, filename) + tiff.export(full_path, data, pyramid=pyramid, imagej=True) + return full_path + + def test_is_pyramidal_tiff(self) -> None: + pyramidal_file = self._create_test_tiff("pyramidal.tif", pyramid=True) + self.assertTrue(tiff.is_pyramidal(pyramidal_file)) + + def test_is_pyramidal_non_pyramidal_tiff(self) -> None: + non_pyramidal_file = self._create_test_tiff("non_pyramidal.tif", pyramid=False) + self.assertFalse(tiff.is_pyramidal(non_pyramidal_file)) + + def test_is_pyramidal_invalid_file(self) -> None: + with self.assertRaises(IOError): + tiff.is_pyramidal("/nonexistent/file.tif") + + class TestTiffIO(unittest.TestCase): def tearDown(self): diff --git a/src/odemis/dataio/tiff.py b/src/odemis/dataio/tiff.py index 66865d7970..087c411876 100644 --- a/src/odemis/dataio/tiff.py +++ b/src/odemis/dataio/tiff.py @@ -2401,6 +2401,25 @@ def read_data(filename): return [acd.content[n].getData() for n in range(len(acd.content))] +def is_pyramidal(filename: str) -> bool: + """ + Check whether a TIFF file has SubIFD pyramid levels. + + :param filename: Path to TIFF file + :return: True when TIFFTAG_SUBIFD contains one or more entries + :raises IOError: If the file cannot be opened as TIFF + """ + try: + tiff_file = TIFF.open(filename, mode="r") + try: + sub_ifds = tiff_file.GetField(T.TIFFTAG_SUBIFD) + return sub_ifds is not None and len(sub_ifds) > 0 + finally: + tiff_file.close() + except Exception as exc: + raise IOError("Failed to inspect TIFF pyramid status for %s: %s" % (filename, exc)) + + def read_thumbnail(filename): """ Read the thumbnail data of a given TIFF file. diff --git a/src/odemis/gui/cont/tabs/_constants.py b/src/odemis/gui/cont/tabs/_constants.py index 5bb4b61f7c..6e42c58e60 100644 --- a/src/odemis/gui/cont/tabs/_constants.py +++ b/src/odemis/gui/cont/tabs/_constants.py @@ -34,3 +34,5 @@ MIRROR_PARKED = 1 MIRROR_BAD = 2 # not parked, but not fully engaged either MIRROR_ENGAGED = 3 + +PYRAMIDAL_CONVERSION_SUFFIX = "_pyramidal" diff --git a/src/odemis/gui/cont/tabs/analysis_tab.py b/src/odemis/gui/cont/tabs/analysis_tab.py index 2c79971541..676f0ba61d 100644 --- a/src/odemis/gui/cont/tabs/analysis_tab.py +++ b/src/odemis/gui/cont/tabs/analysis_tab.py @@ -27,6 +27,8 @@ import gc import logging import os.path +from pathlib import Path +from typing import Union import wx # IMPORTANT: wx.html needs to be imported for the HTMLWindow defined in the XRC # file to be correctly identified. See: http://trac.wxwidgets.org/ticket/3626 @@ -55,11 +57,19 @@ AngularSpectrumViewport, ThetaViewport from odemis.gui.conf import get_acqui_conf from odemis.gui.cont import settings +from odemis.gui.cont.tabs import PYRAMIDAL_CONVERSION_SUFFIX from odemis.gui.cont.tabs.tab import Tab from odemis.gui.model import TOOL_POINT, TOOL_LINE, TOOL_ACT_ZOOM_FIT, TOOL_RULER, TOOL_LABEL, \ TOOL_NONE from odemis.gui.util import call_in_wx_main -from odemis.util.dataio import data_to_static_streams, open_acquisition, open_files_and_stitch +from odemis.util.dataio import ( + data_to_static_streams, + export_data_as_pyramidal, + open_acquisition, + open_files_and_stitch, + is_pyramidal, + convert_file_to_pyramidal, +) class AnalysisTab(Tab): @@ -93,6 +103,9 @@ def __init__(self, name, button, panel, main_frame, main_data): # displayed tab_data = guimod.AnalysisGUIData(main_data) super(AnalysisTab, self).__init__(name, button, panel, main_frame, tab_data) + + self.main_data = main_data + if main_data.role in ("sparc-simplex", "sparc", "sparc2"): # Different name on the SPARC to reflect the slightly different usage self.set_label("ANALYSIS") @@ -318,13 +331,52 @@ def _on_add_file(self): def _on_add_tileset(self): self.select_acq_file(extend=True, tileset=True) + def _get_pyramidal_path(self, filename: Union[str, Path]): + """ + Returns the output path for a given input file, used in the pyramidal conversion process. + + :param filename: The input filename of the file that is to be converted + :return: The adjusted output path + """ + filename = Path(filename) + name_converted = f"{filename.stem}{PYRAMIDAL_CONVERSION_SUFFIX}.tif" + + project_folder = ( + None + if getattr(self.main_data, "is_viewer", False) + else self.main_data.project_path.value + ) + + if project_folder: + return Path(project_folder) / name_converted + else: + # For simplicity, always convert to tiff format + return filename.with_name(name_converted) + def load_tileset(self, filenames, extend=False): - data = open_files_and_stitch(filenames) # TODO: allow user defined registration / weave methods - self.display_new_data(filenames[0], data, extend=extend) + data = open_files_and_stitch(filenames) # TODO: allow user defined registration / weave methods + # For the lack of a better name, use the name of the first tile + source_filename = filenames[0] + filename_converted = self._get_pyramidal_path(source_filename) + try: + export_data_as_pyramidal(data, filename_converted, compressed=True) + self.display_new_data(filename_converted, open_acquisition(filename_converted), extend=extend) + except Exception: + logging.exception("Failed to export/reopen stitched pyramidal tileset") + self.display_new_data(filenames[0], data, extend=extend) def load_data(self, filename, fmt=None, extend=False): - data = open_acquisition(filename, fmt) - self.display_new_data(filename, data, extend=extend) + filename = Path(filename) + if is_pyramidal(filename): + self.display_new_data(filename, open_acquisition(filename, fmt), extend=extend) + return + filename_converted = self._get_pyramidal_path(filename) + try: + convert_file_to_pyramidal(filename, filename_converted, compressed=True) + self.display_new_data(filename_converted, open_acquisition(filename_converted, fmt), extend=extend) + except Exception as ex: + logging.exception("Failed to process TIFF import, loading original. Error: %s", ex) + self.display_new_data(filename, open_acquisition(filename, fmt), extend=extend) def _get_time_spectrum_streams(self, spec_streams): """ @@ -358,6 +410,7 @@ def display_new_data(self, filename, data, extend=False): extend (bool): if False, will ensure that the previous streams are closed. If True, will add the new file to the current streams opened. """ + filename = str(filename) if not extend: # Remove all the previous streams self._stream_bar_controller.clear() diff --git a/src/odemis/gui/cont/tabs/correlation_tab.py b/src/odemis/gui/cont/tabs/correlation_tab.py index fa3d1bbff6..d6e0d7bcf0 100644 --- a/src/odemis/gui/cont/tabs/correlation_tab.py +++ b/src/odemis/gui/cont/tabs/correlation_tab.py @@ -25,26 +25,33 @@ import collections import logging -import os.path -from typing import List, Optional +import os +from pathlib import Path +from typing import List, Optional, Union import wx - from odemis.gui import conf - from odemis import dataio from odemis import model import odemis.gui import odemis.gui.cont.export as exportcont -from odemis.gui.cont.stream_bar import StreamBarController import odemis.gui.cont.views as viewcont import odemis.gui.model as guimod import odemis.gui.util as guiutil from odemis.acq.stream import OpticalStream, EMStream, StaticStream from odemis.gui.cont.correlation import CorrelationController +from odemis.gui.cont.stream_bar import StreamBarController +from odemis.gui.cont.tabs import PYRAMIDAL_CONVERSION_SUFFIX +from odemis.gui.cont.tabs.tab import Tab from odemis.gui.model import TOOL_ACT_ZOOM_FIT from odemis.gui.util import call_in_wx_main -from odemis.gui.cont.tabs.tab import Tab -from odemis.util.dataio import data_to_static_streams, open_acquisition, open_files_and_stitch +from odemis.util.dataio import ( + convert_file_to_pyramidal, + data_to_static_streams, + export_data_as_pyramidal, + is_pyramidal, + open_acquisition, + open_files_and_stitch, +) class CorrelationTab(Tab): @@ -166,13 +173,53 @@ def _on_add_file(self) -> None: def _on_add_tileset(self) -> None: self.select_acq_file(extend=True, tileset=True) - def load_tileset(self, filenames: List[str], extend: bool = False) -> None: - data = open_files_and_stitch(filenames) # TODO: allow user defined registration / weave methods - self.load_streams(data) + def _get_pyramidal_path(self, filename: Union[str, Path]): + """ + Returns the output path for a given input file, used in the pyramidal conversion process. - def load_data(self, filename: str, fmt: str = None, extend: bool = False) -> None: - data = open_acquisition(filename, fmt) - self.load_streams(data) + :param filename: The input filename of the file that is to be converted + :return: The adjusted output path + """ + filename = Path(filename) + name_converted = f"{filename.stem}{PYRAMIDAL_CONVERSION_SUFFIX}.tif" + + project_folder = ( + None + if getattr(self.main_data, "is_viewer", False) + else self.main_data.project_path.value + ) + + if project_folder: + return Path(project_folder) / name_converted + else: + # For simplicity, always convert to tiff format + return filename.with_name(name_converted) + + def load_tileset(self, filenames: List[str], extend: bool = False) -> None: + data = open_files_and_stitch(filenames) # TODO: allow user defined registration / weave methods + # For the lack of a better name, use the name of the first tile + source_filename = filenames[0] + filename_converted = self._get_pyramidal_path(source_filename) + try: + export_data_as_pyramidal(data, filename_converted, compressed=True) + self.load_streams(open_acquisition(filename_converted)) + except Exception: + logging.exception("Failed to export/reopen stitched pyramidal tileset") + self.load_streams(data) + + + def load_data(self, filename: Union[str, Path], fmt: Optional[str] = None, extend: bool = False) -> None: + filename = Path(filename) + if is_pyramidal(filename): + self.load_streams(open_acquisition(filename, fmt)) + return + filename_converted = self._get_pyramidal_path(filename) + try: + convert_file_to_pyramidal(filename, filename_converted, compressed=True) + self.load_streams(open_acquisition(filename_converted, fmt)) + except Exception as ex: + logging.exception("Failed to process TIFF import, loading original. Error: %s", ex) + self.load_streams(open_acquisition(filename, fmt)) def select_acq_file(self, extend: bool = False, tileset: bool = False): """ Open an image file using a file dialog box diff --git a/src/odemis/util/dataio.py b/src/odemis/util/dataio.py index f5f3840426..ecd59d7aed 100644 --- a/src/odemis/util/dataio.py +++ b/src/odemis/util/dataio.py @@ -21,8 +21,9 @@ """ import json import logging +from pathlib import Path import os -from typing import Any, Optional +from typing import Any, List, Optional, Union import numpy @@ -199,15 +200,16 @@ def _split_planes(data): return data -def open_acquisition(filename, fmt=None): +def open_acquisition(filename, fmt=None) -> List[Union[model.DataArray, model.DataArrayShadow]]: """ Opens the data according to the type of file, and returns the opened data. If it's a pyramidal image, do not fetch the whole data from the image. If the image is not pyramidal, it reads the entire image and returns it filename (string): Name of the file where the image is fmt (string): The format of the file - return (list of DataArrays or DataArrayShadows): The opened acquisition source + return: The opened acquisition source """ + filename = str(filename) if fmt: converter = dataio.get_converter(fmt) else: @@ -225,6 +227,60 @@ def open_acquisition(filename, fmt=None): return data +def is_pyramidal(filename: Union[str, Path]) -> bool: + """ + Check whether a file is in pyramidal format. + + :param filename: Path to file + :return: True if the file is pyramidal, False otherwise + :raises IOError: If the file cannot be read or converter doesn't support pyramid detection + """ + reader = dataio.find_fittest_converter(str(filename), mode=os.O_RDONLY) + if not hasattr(reader, "is_pyramidal"): + return False + + return reader.is_pyramidal(str(filename)) + + +def export_data_as_pyramidal( + data: Union[model.DataArray, List[model.DataArray]], + dst_filename: Union[str, Path], + compressed: bool = True, +) -> None: + """ + Export provided data to a pyramidal file format selected by destination extension. + + :param data: Array that holds image data + :param dst_filename: Destination file path + :param compressed: Whether to request compressed output + """ + dst_filename = str(dst_filename) + exporter = dataio.find_fittest_converter(dst_filename, mode=os.O_WRONLY) + if not getattr(exporter, "CAN_SAVE_PYRAMID", False): + raise IOError( + "Destination format %s does not support pyramidal export" + % (getattr(exporter, "FORMAT", type(exporter).__name__),) + ) + exporter.export(dst_filename, data, compressed=compressed, pyramid=True) + + +def convert_file_to_pyramidal(src_filename: Union[str, Path], dst_filename: Union[str, Path], compressed: bool = True) -> None: + """ + Convert any readable acquisition file into a pyramidal file format. + + The source is read via the source converter's read_data API. The destination + converter is selected from dst_filename and must support pyramidal export. + + :param src_filename: Source file path + :param dst_filename: Destination file path + :param compressed: Whether to request compressed output + :raises IOError: If reading/exporting fails or destination doesn't support pyramids + """ + reader = dataio.find_fittest_converter(str(src_filename), mode=os.O_RDONLY) + data = reader.read_data(str(src_filename)) + export_data_as_pyramidal(data, dst_filename, compressed) + + def splitext(path): """ Split a pathname into basename + ext (.XXX). diff --git a/src/odemis/util/test/dataio_test.py b/src/odemis/util/test/dataio_test.py index c2c06fa111..49c8dc58bb 100644 --- a/src/odemis/util/test/dataio_test.py +++ b/src/odemis/util/test/dataio_test.py @@ -31,6 +31,7 @@ from odemis.util import testing from odemis.util.dataio import ( _split_planes, + convert_file_to_pyramidal, data_to_static_streams, open_acquisition, open_files_and_stitch, @@ -289,6 +290,16 @@ def test_open_files_and_stitch(self): testing.assert_tuple_almost_equal(rdata[0].metadata[model.MD_POS], (25e-6, 25e-6)) testing.assert_tuple_almost_equal(rdata[0].metadata[model.MD_PIXEL_SIZE], (1e-6, 1e-6)) + def test_convert_to_pyramidal(self): + data = model.DataArray(numpy.random.randint(0, 255, (256, 256), dtype=numpy.uint8)) + with tempfile.TemporaryDirectory(prefix="odemis-dataio-pyr-") as tmpdir: + source_file = os.path.join(tmpdir, "source_non_pyramidal.tif") + dest_file = os.path.join(tmpdir, "converted_pyramidal.tif") + tiff.export(source_file, data, pyramid=False, imagej=True) + convert_file_to_pyramidal(source_file, dest_file, compressed=True) + self.assertTrue(os.path.exists(dest_file)) + self.assertTrue(tiff.is_pyramidal(dest_file)) + class TestSplitPlanes(unittest.TestCase): @classmethod