# Copyright (C) 2017 DESY, Christoph Rosemann, Notkestr. 85, D-22607 Hamburg
#
# lavue is an image viewing program for photon science imaging detectors.
# Its usual application is as a live viewer using hidra as data source.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation in version 2
# of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
#
# Authors:
# Christoph Rosemann <christoph.rosemann@desy.de>
# Andre Rothkirch <andre.rothkirch@desy.de>
# Jan Kotanski <jan.kotanski@desy.de>
#
""" this a simple file handler that loads image files
and delivers just the actual array """
import struct
import numpy as np
import sys
import json
import logging
from . import filewriter
if sys.version_info > (3,):
long = int
try:
import fabio
#: (:obj:`bool`) fabio can be imported
FABIO = True
except ImportError:
FABIO = False
try:
import PIL
import PIL.Image
#: (:obj:`bool`) PIL can be imported
PILLOW = True
except ImportError:
PILLOW = False
#: (:obj:`dict` <:obj:`str`, :obj:`module`> ) nexus writer modules
WRITERS = {}
try:
from . import h5pywriter
WRITERS["h5py"] = h5pywriter
except Exception:
pass
try:
from . import h5cppwriter
WRITERS["h5cpp"] = h5cppwriter
except Exception:
pass
logger = logging.getLogger("lavue")
[docs]class NexusFieldHandler(object):
"""Nexus file handler class.
Reads image from file and returns the numpy array."""
def __init__(self, fname=None, writer=None):
""" constructor
:param fname: file name
:type fname: :obj:`str`
:param writer: h5 writer module: "h5cpp" or "h5py"
:type writer: :obj:`str`
"""
#: (:obj:`any`) module image object
self.__image = None
#: (:obj:`numpy.ndarray`) image data
self.__data = None
#: (:obj:`str`) file name
self.__fname = fname
#: (:obj:`dict` <:obj:`str`, :obj:`dict` <:obj:`str`, :obj:`any`>>)
#: image field dictionary
self.__fields = {}
# (:class:`lavuelib.filewriter.root`) nexus file root
self.__root = None
# (:class:`lavuelib.filewriter.FTFile') nexus file object
self.__fl = None
#: (:obj:`str`) json dictionary with metadata or empty string
self.__metadata = ""
if not writer:
if "h5cpp" in WRITERS.keys():
writer = "h5cpp"
else:
writer = "h5py"
if writer not in WRITERS.keys():
raise Exception("Writer '%s' cannot be opened" % writer)
wrmodule = WRITERS[writer.lower()]
if fname:
try:
self.__fl = filewriter.open_file(
fname, writer=wrmodule, readonly=True,
libver='latest',
swmr=(True if writer in ["h5py", "h5cpp"] else False)
)
except Exception:
try:
self.__fl = filewriter.open_file(
fname, writer=wrmodule, readonly=True)
except Exception:
raise Exception("File '%s' cannot be opened \n" % (fname))
# except Exception as e:
# raise Exception("File '%s' cannot be opened %s\n"
# % (fname, str(e)))
self.__root = self.__fl.root()
[docs] def frombuffer(self, membuffer, fname=None, writer=None):
""" constructor
:param membuffer: memory buffer
:type membuffer: :obj:`bytes` or :obj:`io.BytesIO`
:param fname: file name
:type fname: :obj:`str`
:param writer: h5 writer module: "h5py" or "h5py"
:type writer: :obj:`str`
"""
if fname is not None:
self.__fname = fname
if not writer:
if "h5cpp" in WRITERS.keys() and \
WRITERS["h5cpp"].is_image_file_supported():
writer = "h5cpp"
elif "h5py" in WRITERS.keys() and \
WRITERS["h5py"].is_image_file_supported():
writer = "h5py"
elif "h5cpp" in WRITERS.keys():
writer = "h5cpp"
else:
writer = "h5py"
if writer not in WRITERS.keys():
raise Exception("Writer '%s' cannot be opened" % writer)
wrmodule = WRITERS[writer.lower()]
try:
self.__fl = filewriter.load_file(
membuffer,
fname=self.__fname, writer=wrmodule, readonly=True,
libver='latest',
swmr=(True if writer in ["h5py", "h5cpp"] else False)
)
except Exception:
try:
self.__fl = filewriter.load_file(
membuffer,
fname, writer=wrmodule, readonly=True)
except Exception:
raise Exception(
"File '%s' cannot be loaded \n" % (self.__fname))
self.__root = self.__fl.root()
[docs] def findImageFields(self):
""" provides a dictionary with of all image fields
:returns: dictionary of the field names and the field objects
:rtype: :obj:`dict` <:obj:`str`, :obj:`dict` <:obj:`str`, :obj:`any`>>
"""
#: (:obj:`dict` <:obj:`str`, :obj:`dict` <:obj:`str`, :obj:`any`>>)
#: image field dictionary
self.__fields = {}
self.__parseimages(self.__root)
return self.__fields
@classmethod
def __getpath(cls, path):
""" converts full_path with NX_classes into nexus_path
:param path: nexus full_path
:type path: :obj:`str`
"""
spath = path.split("/")
return "/".join(
[(dr if ":" not in dr else dr.split(":")[0])
for dr in spath])
def __addimage(self, node, tgpath):
"""adds the node into the description list
:param node: nexus node
:type node: :class:`lavuelib.filewriter.FTField` or \
:class:`lavuelib.filewriter.FTGroup`
:param path: path of the link target or `None`
:type path: :obj:`str`
"""
desc = {}
path = filewriter.first(node.path)
desc["full_path"] = str(path)
desc["nexus_path"] = str(self.__getpath(path))
if hasattr(node, "shape"):
desc["shape"] = list(node.shape or [])
else:
return
if len(desc["shape"]) < 2:
return
if hasattr(node, "dtype"):
desc["dtype"] = str(node.dtype)
else:
return
if node.is_valid:
desc["node"] = node
else:
return
self.__fields[desc["nexus_path"]] = desc
def __parseimages(self, node, tgpath=None):
"""parses the node and add it into the description list
:param node: nexus node
:type node: :class:`lavuelib.filewriter.FTField` or \
:class:`lavuelib.filewriter.FTGroup` or
:param path: path of the link target or `None`
:type path: :obj:`str`
"""
self.__addimage(node, tgpath)
names = []
if isinstance(node, filewriter.FTGroup):
names = [
(ch.name,
str(ch.target_path) if hasattr(ch, "target_path") else None)
for ch in filewriter.get_links(node)]
for nm in names:
try:
ch = node.open(nm[0])
self.__parseimages(ch, nm[1])
# except Exception:
# pass
finally:
pass
[docs] def getNode(self, field=None):
""" get node
:param field: field path
:type field: :obj:`str`
:returns: nexus field node
:rtype: :class:`lavuelib.filewriter.FTField`
"""
node = None
if field is not None:
sfield = str(field).split("/")
node = self.__root
for name in sfield:
if name:
node = node.open(name)
else:
node = self.__fl.default_field()
return node
[docs] @classmethod
def getFrameCount(cls, node, growing=0, refresh=True):
""" provides the last frame number
:param node: nexus field node
:type node: :class:`lavuelib.filewriter.FTField`
:param growing: growing dimension
:type growing: :obj:`int`
:param refresh: refresh image node
:type refresh: :obj:`bool`
:returns: a number of frames
:rtype: :obj:`int`
"""
if refresh:
node.refresh()
if node:
shape = node.shape
if shape:
if len(shape) > growing and growing > -1:
return shape[growing]
return 0
[docs] @classmethod
def extract(cls, value):
if isinstance(value, np.ndarray):
if len(value.shape) == 1 and value.shape[0] == 1:
value = value[0]
return value
[docs] @classmethod
def getImage(cls, node, frame=-1, growing=0, refresh=True):
"""parses the field and add it into the description list
:param node: nexus field node
:type node: :class:`lavuelib.filewriter.FTField`
:param frame: frame to take, the last one is -1
:type frame: :obj:`int`
:param growing: growing dimension
:type growing: :obj:`int`
:param refresh: refresh image node
:type refresh: :obj:`bool`
:returns: get the image
:rtype: :class:`numpy.ndarray`
"""
if refresh:
node.refresh()
if node:
shape = node.shape
if shape:
if frame is None:
return node.read()
if len(shape) == 2:
return node[...]
elif len(shape) == 3:
if growing == 0:
if frame < 0 or shape[0] > frame:
return node[frame, :, :]
elif growing == 1:
if frame < 0 or shape[1] > frame:
return node[:, frame, :]
else:
if frame < 0 or shape[2] > frame:
return node[:, :, frame]
elif len(shape) == 4:
if growing == 0:
if frame < 0 or shape[0] > frame:
return node[frame, :, :, :]
elif growing == 1:
if frame < 0 or shape[1] > frame:
return node[:, frame, :, :]
elif growing == 2:
if frame < 0 or shape[2] > frame:
return node[:, :, frame, :]
else:
if frame < 0 or shape[3] > frame:
return node[:, :, :, frame]
[docs]class ImageFileHandler(object):
"""Simple file handler class.
Reads image from file and returns the numpy array."""
def __init__(self, fname):
""" constructor
:param fname: file name
:type fname: :obj:`str`
"""
#: (:obj:`any`) module image object
self.__image = None
#: (:obj:`numpy.ndarray`) image data
self.__data = None
#: (:obj:`str`) json dictionary with metadata or empty string
self.__metadata = ""
try:
if FABIO:
self.__image = fabio.open(fname)
self.__data = self.__image.data
if self.__data is None:
raise Exception("no data")
if "_array_data.header_convention" \
in self.__image.header.keys():
rmeta = self.__image.header["_array_data.header_contents"]
self.__metadata = CBFLoader().metadata(rmeta)
elif PILLOW:
self.__image = PIL.Image.open(fname)
self.__data = np.array(self.__image)
except Exception as e:
logger.debug(str(e))
try:
if FABIO and PILLOW:
self.__image = PIL.Image.open(fname)
self.__data = np.array(self.__image)
except Exception as e:
logger.debug(str(e))
try:
self.__image = np.fromfile(str(fname), dtype='uint8')
if fname.endswith(".cbf"):
self.__data = CBFLoader().load(self.__image)
self.__metadata = CBFLoader().metadata(self.__image)
else:
self.__data = TIFLoader().load(self.__image)
except Exception as e:
# print(str(e))
logger.warning(str(e))
[docs] def getImage(self):
""" provides the image data
:returns: image data
:rtype: :class:`numpy.ndarray`
"""
return self.__data
[docs]class CBFLoader(object):
""" CBF loader """
[docs] @classmethod
def load(cls, flbuffer):
""" loads CBF file image data into numpy array
:param flbuffer: numpy array with CBF file image data
:type flbuffer: :class:`numpy.ndarray`
:returns: image data
:rtype: :class:`numpy.ndarray`
"""
image = np.array([0])
inpoint = np.array([26, 4, 213], dtype='uint8')
# array with '--CIF-BINARY-FORMAT-SECTION---'
outpoint = np.array(
[45, 45, 67, 73, 70, 45, 66, 73, 78, 65, 82, 89, 45, 70,
79, 82, 77, 65, 84, 45, 83, 69, 67, 84, 73, 79, 78, 45, 45, 45],
dtype='uint8')
flag = 0
# check if byte offset compress ('x-CBF_BYTE_OFFSET')
boc = np.array(
[120, 45, 67, 66, 70, 95, 66, 89, 84,
69, 95, 79, 70, 70, 83, 69, 84],
dtype='uint8')
try:
# iscbf
flbuffer.tostring().index(boc.tostring()) // flbuffer.itemsize
except Exception:
flag = 1
# additional parms for cross check if decompress worked out
# ('X-Binary-Number-of-Elements:')
dset_num_ele = np.array(
[88, 45, 66, 105, 110, 97, 114, 121, 45, 78, 117, 109, 98,
101, 114, 45, 111, 102, 45, 69, 108, 101, 109, 101, 110,
116, 115, 58],
dtype='uint8')
# array with 'X-Binary-Size-Fastest-Dimension:'
dset_fast_dim = np.array(
[88, 45, 66, 105, 110, 97, 114, 121, 45, 83, 105, 122, 101, 45, 70,
97, 115, 116, 101, 115, 116, 45, 68, 105, 109, 101, 110, 115, 105,
111, 110, 58], dtype='uint8')
# array with 'X-Binary-Size-Second-Dimension:'
dset_sec_dim = np.array(
[88, 45, 66, 105, 110, 97, 114, 121, 45, 83, 105, 122, 101, 45, 83,
101, 99, 111, 110, 100, 45, 68, 105, 109, 101, 110, 115, 105, 111,
110, 58], dtype='uint8')
# array with 'X-Binary-Size-Padding:'
dset_pad = np.array(
[88, 45, 66, 105, 110, 97, 114, 121, 45, 83, 105, 122, 101, 45, 80,
97, 100, 100, 105, 110, 103, 58], dtype='uint8')
# search for data stream start
if flag == 0:
try:
idstart = flbuffer.tostring().index(
inpoint.tostring()) // flbuffer.itemsize
idstart += inpoint.size
except Exception:
flag = 1
try:
idstop = flbuffer.tostring().index(
outpoint.tostring()) // flbuffer.itemsize
idstop -= 3 # cr / extra -1 due to '10B' -- linefeed
except Exception:
flag = 1
vals = np.zeros(4, dtype='int')
spos = np.zeros(5, dtype='int')
spos[4] = idstart
try:
spos[0] = flbuffer.tostring().index(
dset_num_ele.tostring()) // flbuffer.itemsize
spos[1] = flbuffer.tostring().index(
dset_fast_dim.tostring()) // flbuffer.itemsize
spos[2] = flbuffer.tostring().index(
dset_sec_dim.tostring()) // flbuffer.itemsize
spos[3] = flbuffer.tostring().index(
dset_pad.tostring()) // flbuffer.itemsize
# by A.R., Apr 24, 2017
vals[0] = int(
flbuffer[
spos[0] + dset_num_ele.size:spos[1] - 2].tostring())
vals[1] = int(
flbuffer[
spos[1] + dset_fast_dim.size:spos[2] - 2].tostring())
vals[2] = int(
flbuffer[
spos[2] + dset_sec_dim.size:spos[3] - 2].tostring())
vals[3] = int(
flbuffer[
spos[3] + dset_pad.size:spos[4] - 8].tostring())
except Exception:
flag = 1
if flag == 0:
image = 0
image = cls._decompress_cbf_c(
flbuffer[idstart:idstop + 1], vals)
else:
image = np.array([0])
return np.transpose(image)
@classmethod
def _decompress_cbf_c(cls, stream, vals):
""" decompresses CBF
:param stream: a part of cbf data
:type stream: :class:`numpy.ndarray`
:param val: decompress parameters, i.e. n_out, xdim, ydum padding
:type val: :class:`numpy.ndarray`
:returns: image data
:rtype: :class:`numpy.ndarray`
"""
xdim = long(487)
ydim = 619
padding = long(4095)
n_out = xdim * ydim
# simply assume content fits here
if vals.size == 4 and sum(vals) != 0:
xdim = vals[1]
ydim = vals[2]
padding = vals[3]
n_out = vals[0]
flbuffer = np.zeros(stream.size, dtype='int32') + stream
mymap = np.zeros(stream.size, dtype='uint8') + 1
isvalid = np.zeros(stream.size, dtype='uint8') + 1
id_relevant = np.where(stream == 128)
# overcome issue if 128 exists in padding (seems so that
# either this does not happened before or padding was 0 in any case)
try:
idd = np.where(id_relevant < (flbuffer.size - padding))
id_relevant = id_relevant[idd]
except Exception:
pass
for dummy, dummy2 in enumerate(id_relevant):
for j, i in enumerate(dummy2):
if mymap[i] != 0:
if stream[i + 1] != 0 or stream[i + 2] != 128:
mymap[i:i + 3] = 0
isvalid[i + 1:i + 3] = 0
delta = flbuffer[i + 1] + flbuffer[i + 2] * 256
if delta > 32768:
delta -= 65536
flbuffer[i] = delta
else:
mymap[i:i + 7] = 0
isvalid[i + 1:i + 7] = 0
# delta=sum(np.multiply(flbuffer[i+3:i+7],
# np.array([1,256,65536,16777216],dtype='int64')))
delta = (
np.multiply(
flbuffer[i + 3:i + 7],
np.array([1, 256, 65536, 16777216],
dtype='int64'))).sum()
if delta > 2147483648:
delta -= 4294967296
flbuffer[i] = delta
try:
id8sign = np.where((stream > 128) & (mymap != 0))
flbuffer[id8sign] -= 256
# print ("adjusting 8Bit vals")
# for i, j in enumerate(stream):
# if j > 128 and mymap[i] !=0:
# flbuffer[i]=flbuffer[i]-256
# print stream[0:11]
# print flbuffer[0:11]
except Exception:
pass
try:
# print sum(isvalid) #should be 305548
idd = np.where(isvalid != 0)
flbuffer = flbuffer[idd]
except Exception:
pass
# print stream[0:11]
# print flbuffer[0:11]
res = np.cumsum(flbuffer, dtype='int32')
# print max(res)
if res.size - padding != n_out:
return np.array([0])
# by A.R., Apr 24, 2017
# return res[0:n_out].reshape(xdim, ydim)
return res[0:n_out].reshape(xdim, ydim, order='F')
[docs]class TIFLoader(object):
""" TIF loader """
[docs] @classmethod
def load(cls, flbuffer):
""" loads TIF file image data into numpy array
:param flbuffer: numpy array with TIF file image data
:type flbuffer: :class:`numpy.ndarray`
:returns: image data
:rtype: :class:`numpy.ndarray`
"""
image = np.float64(-1)
# define unsigned default if undefined - i.e. like MAR165 data
sample_format = 1
flbuffer_endian = 'none'
if sum(abs(flbuffer[0:2] - [73, 73])) == 0:
flbuffer_endian = "<" # little
if sum(abs(flbuffer[0:2] - [77, 77])) == 0:
flbuffer_endian = ">" # big
if flbuffer_endian == "none":
return image # or better to raise exception?
numfortiff = np.uint16(
struct.unpack_from(flbuffer_endian + "H", flbuffer[2:4])[0])
if numfortiff != 42:
return image # or better to raise exception?
ifd_off = np.uint32(
struct.unpack_from(flbuffer_endian + "I", flbuffer[4:8])[0])
#
# jump to/eval image file directory (ifd)
num_of_ifd = np.uint16(
struct.unpack_from(
flbuffer_endian + "H", flbuffer[ifd_off:ifd_off + 2])[0])
for ifd_entry in range(num_of_ifd):
field_tag = np.uint16(
struct.unpack_from(
flbuffer_endian + "H",
flbuffer[ifd_off + 2 + ifd_entry * 12:ifd_off
+ 4 + ifd_entry * 12])[0])
field_type = np.uint16(
struct.unpack_from(
flbuffer_endian + "H",
flbuffer[ifd_off + 4 + ifd_entry * 12:ifd_off
+ 6 + ifd_entry * 12])[0])
# num_vals = np.uint32(
# struct.unpack_from(
# flbuffer_endian + "I",
# flbuffer[ifd_off + 6 + ifd_entry * 12:ifd_off + 10
# + ifd_entry * 12])[0])
# given tiff 6.0 there are 12 type entries, currently not all of
# them are accounted, A.R.
val_or_off = 0
if field_type == 1: # check flbuffer addressing!
val_or_off = np.uint8(
struct.unpack_from(
flbuffer_endian + "B",
flbuffer[ifd_off + 10 + ifd_entry * 12:ifd_off + 15
+ ifd_entry * 12])[0])
if field_type == 3:
val_or_off = np.uint16(
struct.unpack_from(
flbuffer_endian + "H",
flbuffer[ifd_off + 10 + ifd_entry * 12:ifd_off + 15
+ ifd_entry * 12])[0])
if field_type == 4:
val_or_off = np.uint32(
struct.unpack_from(
flbuffer_endian + "I",
flbuffer[ifd_off + 10 + ifd_entry * 12:ifd_off + 15
+ ifd_entry * 12])[0])
if field_type == 8:
val_or_off = np.int16(
struct.unpack_from(
flbuffer_endian + "h",
flbuffer[ifd_off + 10 + ifd_entry * 12:ifd_off
+ 15 + ifd_entry * 12])[0])
if field_type == 9:
val_or_off = np.int32(
struct.unpack_from(
flbuffer_endian + "i",
flbuffer[ifd_off + 10 + ifd_entry * 12:ifd_off + 15
+ ifd_entry * 12])[0])
if field_type == 11:
val_or_off = np.float32(
struct.unpack_from(
flbuffer_endian + "f",
flbuffer[ifd_off + 10 + ifd_entry * 12:ifd_off
+ 15 + ifd_entry * 12])[0])
# eval (hopefully) tags needed to allow for getting an image
if field_tag == 256:
width = int(val_or_off)
if field_tag == 257:
length = int(val_or_off)
if field_tag == 258:
bit_per_sample = int(val_or_off)
# compression scheme - return invalid if NOT none,
# i.e. only uncompressed data is supported (forever!?)
if field_tag == 259:
if val_or_off != 1:
return image
# photometric interpretation - 2 denotes RGB which is refused
# otherwise don't mind/care ...
if field_tag == 262:
if val_or_off == 2:
return image
if field_tag == 273:
strip_offsets = val_or_off
# likely equals image width
# if field_tag == 278:
# rows_per_strip = val_or_off
if field_tag == 279:
strip_byte_counts = val_or_off
if field_tag == 339:
sample_format = val_or_off
next_idf = np.uint32(
struct.unpack_from(
flbuffer_endian + "I",
flbuffer[ifd_off + 2 + (ifd_entry + 1) * 12:
ifd_off + 6 + (ifd_entry + 1) * 12]
)[0])
if next_idf != 0:
# print('another ifd exists ... NOT read')
pass
if width * length * bit_per_sample // 8 != strip_byte_counts:
return image
if sample_format == 1 and bit_per_sample == 8:
image = np.uint8(
struct.unpack_from(
flbuffer_endian + str(width * length) + "B",
flbuffer[strip_offsets:strip_offsets
+ strip_byte_counts + 1]))
if sample_format == 1 and bit_per_sample == 16:
image = np.uint16(
struct.unpack_from(
flbuffer_endian + str(width * length) + "H",
flbuffer[strip_offsets:strip_offsets
+ strip_byte_counts + 1]))
if sample_format == 1 and bit_per_sample == 32:
image = np.uint32(
struct.unpack_from(
flbuffer_endian + str(width * length) + "I",
flbuffer[strip_offsets:strip_offsets
+ strip_byte_counts + 1]))
# if sample_format == 2 and bit_per_sample == 8:
# image=np.int8(struct.unpack_from(
# flbuffer_endian+str(width*length)+"b",
# flbuffer[strip_offsets:strip_offsets
# +strip_byte_counts+1]))
if sample_format == 2 and bit_per_sample == 16:
image = np.int16(
struct.unpack_from(
flbuffer_endian + str(width * length) + "h",
flbuffer[strip_offsets:strip_offsets
+ strip_byte_counts + 1]))
if sample_format == 2 and bit_per_sample == 32:
image = np.int32(
struct.unpack_from(
flbuffer_endian + str(width * length) + "i",
flbuffer[strip_offsets:strip_offsets
+ strip_byte_counts + 1]))
if sample_format == 3 and bit_per_sample == 32:
image = np.float32(
struct.unpack_from(
flbuffer_endian + str(width * length) + "f",
flbuffer[strip_offsets:strip_offsets
+ strip_byte_counts + 1]))
try:
return np.transpose(image.reshape(width, length, order='F'))
except Exception:
return np.transpose(image)
if __name__ == "__main__":
# filename =
# '/afs/desy.de/user/r/rothkirc/public/P03/lost_00001_00001.tif' # input
# file name
# input file name
filename = '/afs/desy.de/user/r/rothkirc/public/20131129_eugen/' \
+ 'mar165_agbeh_00001.tif'
tmp = np.fromfile(filename, dtype='uint8') # read all content as unit 8
resu = TIFLoader().load(tmp)
print("Return value shape and dtype")
print("%s %s" % (resu.shape, resu.dtype))