"""Import's test data: one set of fields, written in every format and naming.
A small set of fields from the toxo_mito example plate, converted into the
container formats spaCR reads and renamed into every filename convention
Mask's ``metadata_type`` offers, published together with their masks and a
measurements database. Importing any variant should give back exactly the
wells, fields and channels the fields started as, and the manifest says what
those are.
Published contents: :data:`IMPORT_EXAMPLE_REPO` holds one archive. Inside
it, under ``import_example/``:
* ``variants/<key>/plate1/`` -- the same twelve planes (wells E01 and E02,
fields 9 and 10, three channels, cropped from the toxo_mito plate) written
once per variant: once per filename convention, once as Zeiss CZI, once as
OME-TIFF and once as an ImageJ TIFF stack;
* ``variants/<key>/masks/{cell,nucleus,pathogen}/`` -- the Mask run's label
images for exactly those fields, named the way that variant names images;
* ``variants/<key>/measurements.db`` -- the Measure rows of exactly the cells
in those masks, keyed by that variant's own file names, as a collaborator's
table would be;
* ``reference/measurements.db`` -- the same rows as spaCR wrote them;
* ``variants/nikon_nd2`` and ``variants/leica_lif`` -- public sample files,
because no open library writes ND2 or LIF. They are third-party images
under CC BY 4.0, with no masks, and they go through the Format Converter;
* ``manifest.csv`` -- one row per image file stating its actual well, field,
channel, z and t. It is the last member of the archive, so an interrupted
transfer leaves no manifest and the set reads as absent.
The variant registry is defined in this module, not in the archive.
:data:`IMPORT_VARIANTS` is what the Import screen offers before anything is
downloaded, and what ``tools/build_import_example.py`` writes. Using one
list for both keeps every button pointing at a folder the archive contains.
This module imports nothing from Qt, so the round-trip test and the builder
can use it headless.
"""
from __future__ import annotations
import csv
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, List, Optional, Tuple
__all__ = [
"IMPORT_EXAMPLE_ARCHIVE",
"IMPORT_EXAMPLE_FOLDER",
"IMPORT_EXAMPLE_REPO",
"IMPORT_VARIANTS",
"ImportVariant",
"import_example_folder",
"import_variant",
"is_present",
"manifest_rows",
"variant_inputs",
]
#: The Hugging Face dataset repository Import's test data is published in.
IMPORT_EXAMPLE_REPO = "einarolafsson/spacr-example-import"
#: The one archive that repository ships.
IMPORT_EXAMPLE_ARCHIVE = "spacr-example-import.tar"
#: The folder every member of the archive sits under. The archive unpacks
#: into the shared example plate like the other example sets, and this prefix
#: is what keeps its files from being mistaken for that plate's own -- the
#: Mask demo counts any ``*.tif`` at the plate's top level as itself.
IMPORT_EXAMPLE_FOLDER = "import_example"
#: The file that states what every image truly is.
MANIFEST_NAME = "manifest.csv"
@dataclass(frozen=True)
[docs]
class ImportVariant:
"""One way the same fields were written, and how Import reads it back.
:param key: the folder under ``variants/`` and the chooser's route key.
:param label: the button text.
:param route: ``'import'`` for the Import screen itself (images, masks
and a measurement table), ``'convert'`` for the Format Converter
(images only).
:param metadata_type: the filename convention to parse by, or ``'auto'``
for folders and tokens.
:param custom_regex: the pattern, for ``metadata_type='custom'``.
:param file_format: what the image files are, in words.
:param example: one image path from the variant, relative to its
``plate1`` folder, so the chooser can show what the names look like
before anything is downloaded.
:param note: anything the reader should know about this variant.
"""
key: str
label: str
route: str
metadata_type: str
custom_regex: str
file_format: str
example: str
note: str = ""
@property
[docs]
def description(self) -> str:
"""The paragraph the chooser shows on hover."""
where = ("the Import screen: images, cell / nucleus / pathogen masks "
"and a measurements table"
if self.route == "import" else
"the Format Converter: images only")
convention = (f"Naming: {self.metadata_type}"
+ (f" with the pattern {self.custom_regex}"
if self.custom_regex else ""))
text = (f"{self.file_format}, named like {self.example}. "
f"{convention}. Fills {where}, then previews.")
if self.note:
text += f"\n\n{self.note}"
return text
_CUSTOM_REGEX = (r"(?P<plateID>[^_]+)__(?P<wellID>[A-Z]\d{2})"
r"__site(?P<fieldID>\d+)__(?P<chanID>[A-Za-z0-9]+)")
#: Every variant, in the order the chooser lists them. Zeiss CZI first,
#: because it is the one a demonstration starts from.
IMPORT_VARIANTS: Tuple[ImportVariant, ...] = (
ImportVariant(
"zeiss_czi", "Test Zeiss CZI import", "import",
"zeiss_zen_split_tiles", "", "Zeiss CZI, one plane per file",
"E01/plate1_S00001_T00009_C00001.czi",
"Written with pylibCZIrw, each file carrying its channel name in the "
"CZI metadata. The scene index is not a well name, so the well comes "
"from the folder."),
ImportVariant(
"cellvoyager", "Test Yokogawa CellVoyager import", "import",
"cellvoyager", "", "TIFF", "plate1_E01_T0001F009L01A01Z01C01.tif"),
ImportVariant(
"cq1", "Test Yokogawa CQ1 import", "import", "cq1", "", "TIFF",
"W0097F0009T0001Z000C1.tif",
"The CQ1 writes a well INDEX over 24 columns: W0097 is E01."),
ImportVariant(
"opera_phenix", "Test Opera Phenix import", "import",
"opera_phenix", "", "TIFF", "Images/r05c01f09p01-ch1sk1fk1fl1.tiff",
"Harmony writes row and column numbers, never a well name: r05c01 "
"is E01."),
ImportVariant(
"imagexpress", "Test ImageXpress import", "import", "imagexpress", "",
"TIFF", "TimePoint_1/plate1_E01_s9_w1<GUID>.TIF"),
ImportVariant(
"arrayscan", "Test ArrayScan import", "import", "arrayscan", "",
"TIFF", "plate1_E01f08d0.TIF",
"Fields and dyes count from zero: f08d0 is field 9, channel 1."),
ImportVariant(
"arrayscan_kinetic", "Test ArrayScan kinetic import", "import",
"arrayscan_kinetic", "", "TIFF", "plate1i3t001E01f08d0.TIF"),
ImportVariant(
"evos", "Test EVOS import", "import", "evos", "", "TIFF",
"scan_R_p1_z1_0_E01f09d0.tif"),
ImportVariant(
"incell", "Test IN Cell import", "import", "incell", "", "TIFF",
"E - 01(fld 9 wv DAPI - DAPI).tif",
"Channels are named by filter, so their order is the order of the "
"names."),
ImportVariant(
"scanr", "Test ScanR import", "import", "scanr", "", "TIFF",
"data/E1--W00097--P00009--Z00000--T00000--DAPI.tif"),
ImportVariant(
"cytation", "Test Cytation import", "import", "cytation", "", "TIFF",
"E1_01_1_9_DAPI_001.tif"),
ImportVariant(
"leica_matrix_screener", "Test Leica Matrix Screener import",
"import", "leica_matrix_screener", "", "OME-TIFF, one plane per file",
"slide--S00/chamber--U00--V04/field--X08--Y00/image--L00--S00--U00--"
"V04--J20--E00--O00--X08--Y00--T00--Z00--C00.ome.tif",
"The Matrix Screener names a field by its X/Y position, not a number, "
"so fields are numbered in order."),
ImportVariant(
"leica_lasx_series", "Test Leica LAS X series import", "import",
"leica_lasx_series", "", "TIFF", "E01/Series009_z00_ch00.tif",
"A series export carries no well, so the well comes from the folder."),
ImportVariant(
"leica_lasx_series_time", "Test Leica LAS X timelapse import",
"import", "leica_lasx_series_time", "", "TIFF",
"E01/Pos008_t000_z00_ch00.tif"),
ImportVariant(
"nikon_nis_xy", "Test Nikon NIS-Elements import", "import",
"nikon_nis_xy", "", "TIFF", "E01xy09c1.tif"),
ImportVariant(
"nikon_jobs", "Test Nikon JOBS import", "import", "nikon_jobs", "",
"TIFF", "WellE01_ChannelDAPI_Seq0009.tif"),
ImportVariant(
"micromanager_mda", "Test Micro-Manager import", "import",
"micromanager_mda", "", "TIFF",
"E01/img_channel000_position008_time000000000_z000.tif"),
ImportVariant(
"zeiss_zen_split_tiles", "Test Zeiss ZEN split tiles import",
"import", "zeiss_zen_split_tiles", "", "TIFF",
"E01/plate1_S00001_T00009_C00001.tiff"),
ImportVariant(
"custom", "Test custom naming import", "import", "custom",
_CUSTOM_REGEX, "TIFF", "toxo-plate1__E01__site09__DAPI.tif"),
ImportVariant(
"auto", "Test folder-layout import", "import", "auto", "",
"TIFF", "E01/fov09_ch1.tif",
"No convention: the well is the folder and the channel is the ch "
"token. Fields are numbered in order, because a folder layout "
"states no field numbers to keep."),
ImportVariant(
"ome_tiff", "Test OME-TIFF import", "import", "auto", "",
"OME-TIFF, three channels per file", "E01/field009.ome.tif",
"Channels are read from inside the file."),
ImportVariant(
"tiff_stack", "Test TIFF stack import", "import", "auto", "",
"ImageJ TIFF stack, three channels per file", "E01/field009.tif",
"Channels are read from inside the file."),
ImportVariant(
"nikon_nd2", "Test Nikon ND2 import", "convert", "auto", "",
"Nikon ND2 (public sample)", "A01/WellA01_ChannelBF_Seq0001.nd2",
"No open library writes ND2, so this is a public sample file by "
"Maxime Woringer, CC BY 4.0: one brightfield plane, renamed the way "
"NIS-Elements JOBS names a well-plate acquisition. It has no masks, "
"so it goes through the Format Converter."),
ImportVariant(
"leica_lif", "Test Leica LIF import", "convert", "auto", "",
"Leica LIF (public samples)", "A01/FRAP.lif",
"No open library writes LIF, so these are public sample files by "
"Sean Warren and Michael Goelzer, CC BY 4.0: time series, a z-stack "
"and a tile scan. They have no masks, so they go through the Format "
"Converter."),
)
[docs]
def import_variant(key: str) -> ImportVariant:
"""The variant called ``key``.
:param key: a key from :data:`IMPORT_VARIANTS`.
:raises KeyError: naming the keys that exist.
"""
for candidate in IMPORT_VARIANTS:
if candidate.key == key:
return candidate
raise KeyError(f"no import test variant named {key!r}; there is "
f"{', '.join(v.key for v in IMPORT_VARIANTS)}")
[docs]
def import_example_folder(plate_folder: Optional[Path] = None) -> Path:
"""Where the unpacked set lives.
:param plate_folder: the shared example plate the archive unpacks into;
:func:`spacr.example_archives.example_plate_folder` when omitted.
:returns: ``<plate folder>/import_example``. Not created here.
"""
if plate_folder is None:
from .example_archives import example_plate_folder
plate_folder = example_plate_folder()
return Path(plate_folder) / IMPORT_EXAMPLE_FOLDER
[docs]
def manifest_rows(root) -> List[Dict[str, str]]:
"""Every row of the set's ``manifest.csv``.
:param root: the unpacked ``import_example`` folder.
:returns: the rows, or ``[]`` when there is no readable manifest.
"""
try:
with (Path(root) / MANIFEST_NAME).open(
newline="", encoding="utf-8") as handle:
return list(csv.DictReader(handle))
except (OSError, csv.Error, UnicodeDecodeError):
return []
[docs]
def is_present(root, key: Optional[str] = None) -> bool:
"""Whether a complete copy of the set -- or of one variant -- is here.
:param root: the unpacked ``import_example`` folder.
:param key: one variant, or every variant when omitted.
:returns: ``True`` only when the manifest exists and every file it lists
for the variant(s) is on disk. A partly unpacked set reads as absent,
so it is downloaded again.
"""
root = Path(root)
rows = [row for row in manifest_rows(root)
if key is None or row.get("variant") == key]
return bool(rows) and all((root / row["path"]).is_file() for row in rows)