mirror of
https://github.com/c-sooyoung/fold_slice.git
synced 2026-09-17 23:59:11 +09:00
initial commit
This commit is contained in:
@@ -0,0 +1,63 @@
|
||||
ifndef MATLAB_HOME
|
||||
$(error "MATLAB_HOME is undefined, load matlab/2018a module!")
|
||||
endif
|
||||
|
||||
ifndef HDF5_SERIAL_HOME
|
||||
$(error "HDF5_SERIAL_HOME is undefined, load gcc/6.3.0 and hdf5_serial/1.8.18 modules!")
|
||||
endif
|
||||
|
||||
ifndef TIFF_INCLUDE_DIR
|
||||
ifndef TIFF_LIBRARY_DIR
|
||||
$(error "TIFF_INCLUDE_DIR or TIFF_LIBRARY_DIR undefined, load tiff/4.0.9 module!")
|
||||
endif
|
||||
endif
|
||||
|
||||
MEXOPTIONS += -R2018a
|
||||
|
||||
IO_DIR := ../..
|
||||
RDOBJ_TARGET_BASE := $(IO_DIR)/ptycho_read
|
||||
RDMSR_TARGET_BASE := $(IO_DIR)/read_measurement
|
||||
TARGET_SUFFIX := mexa64
|
||||
RDOBJ_TARGET_FILE := $(RDOBJ_TARGET_BASE).$(TARGET_SUFFIX)
|
||||
RDMSR_TARGET_FILE := $(RDMSR_TARGET_BASE).$(TARGET_SUFFIX)
|
||||
|
||||
all: $(RDOBJ_TARGET_FILE) $(RDMSR_TARGET_FILE)
|
||||
|
||||
ptycho_read: $(RDOBJ_TARGET_FILE)
|
||||
|
||||
read_measurement: $(RDMSR_TARGET_FILE)
|
||||
|
||||
debug_helper.o : debug_helper.cc debug_helper.h
|
||||
mex $(MEXOPTIONS) -c $<
|
||||
|
||||
hdf5_helper.o: hdf5_helper.cc hdf5_helper.h
|
||||
mex $(MEXOPTIONS) -c $<
|
||||
|
||||
read_object_data.o: read_object_data.cc read_object_data.h precision.h hdf5_helper.h debug_helper.h env_helper.h multi_processing.h
|
||||
mex $(MEXOPTIONS) -c $<
|
||||
|
||||
read_eiger_data.o: read_eiger_data.cc read_eiger_data.h precision.h hdf5_helper.h debug_helper.h env_helper.h multi_processing.h
|
||||
mex $(MEXOPTIONS) -c $<
|
||||
|
||||
read_data_threaded.o: read_data_threaded.cc read_data_threaded.h precision.h debug_helper.h env_helper.h ${TIFF_INCLUDE_DIR}/tiffio.h
|
||||
mex $(MEXOPTIONS) -I$(TIFF_INCLUDE_DIR) -c $<
|
||||
|
||||
# Put the resultinf mex file into +io/+HDF folder
|
||||
$(RDOBJ_TARGET_FILE): readObjectData.cc debug_helper.o hdf5_helper.o read_object_data.o mex_helper.h env_helper.h precision.h
|
||||
mex $(MEXOPTIONS) $< read_object_data.o hdf5_helper.o debug_helper.o $(HDF5_SERIAL_HOME)/lib/libhdf5.a -lz -ldl -output $(RDOBJ_TARGET_BASE)
|
||||
|
||||
$(RDMSR_TARGET_FILE): readMeasurementData.cc read_eiger_data.o hdf5_helper.o read_data_threaded.o debug_helper.o env_helper.h mex_helper.h precision.h read_eiger_data.h read_data_threaded.h $(TIFF_LIBRARY_DIR)/libtiff.a
|
||||
mex $(MEXOPTIONS) $< read_eiger_data.o hdf5_helper.o read_data_threaded.o debug_helper.o $(HDF5_SERIAL_HOME)/lib/libhdf5.a $(TIFF_LIBRARY_DIR)/libtiff.a -lz -ldl -output $(RDMSR_TARGET_BASE)
|
||||
|
||||
doc:
|
||||
test -r doxygen.conf && rm -rf doc/html
|
||||
test -d doc || mkdir doc
|
||||
doxygen doxygen.conf
|
||||
|
||||
clean:
|
||||
rm -f *~ *.o
|
||||
|
||||
proper: clean
|
||||
rm -f $(RDOBJ_TARGET_FILE) $(RDMSR_TARGET_FILE)
|
||||
|
||||
.PHONY: all clean proper ptycho_read read_measurement doc
|
||||
@@ -0,0 +1,149 @@
|
||||
This directory contains code for reading ptychographic reconstruction results and measurement data.
|
||||
|
||||
ptycho_read
|
||||
===========
|
||||
|
||||
This is a MEX function for reading ptychographically constructed object datasets from HDF5 files.
|
||||
|
||||
The parameters to the function are
|
||||
1. number of reader processes
|
||||
2. precision of the returned array: either 'single' or 'double'
|
||||
3. desired dimension of the returned objects
|
||||
This is an optional parameter. If absent, the dimension of the object in the first file is assumed to be the desired object dimension.
|
||||
Objects that have different dimension will be adapted (either some pixels at the borders left away, or zero pixels added)
|
||||
4. path to the object dataset within the HDF5 file (e.g. '/reconstruction/object')
|
||||
5. cell array with the file paths (e.g. {'the/path/to/example.h5'})
|
||||
If there is only one file, the parameter can also be a character array
|
||||
|
||||
From MATLAB call it in one of the following ways:
|
||||
|
||||
[A, ind] = io.ptycho_read(1, 'single', [21,22], '/reconstruction/object', {'test1_20x20_c.h5', 'test2_20x20_c.h5'});
|
||||
A = io.ptycho_read(1, 'single', [21,22], '/reconstruction/object', {'test1_20x20_c.h5', 'test2_20x20_c.h5'});
|
||||
|
||||
The array *A* will contain the object datasets as a multidimensional array (Ncols, Nrows, Nmodes, Nslices, Nobjects). Nslices and Nmodes dimensions will be dropped if they are of size one. *ind* will contain a list of file indices that could not be read, if the first form is used. In the second form you'll get an error message if a file cannot be read. The function will print out some information about the read speed, if the environment variable *PTYCHO_READ_VERBOSE* is set to one of the values '1', 'yes', 'true'.
|
||||
|
||||
setenv('PTYCHO_READ_VERBOSE', 'yes')
|
||||
|
||||
The above command accomplishes this inside MATLAB.
|
||||
|
||||
read_measurement
|
||||
================
|
||||
|
||||
This is a MEX function for reading data from Eiger detector files in HDF5 format. The data must be located at /eh5/images within the file.
|
||||
|
||||
The parameters to the function are
|
||||
1. parameter structure
|
||||
|
||||
The parameter structure contains some general fields:
|
||||
* 'asize' with the result image dimensions (Ncols, Nrows) (optional, derived from data by default)
|
||||
* 'ctr' with the regioin of interest center [col, row] relative to the data (optional, derived from data by default)
|
||||
* 'precision' with the desired precision - 'single' or 'double'
|
||||
|
||||
For the **Eiger** detector, the parameter structure must contain the following fields:
|
||||
* 'extension' with value 'h5'
|
||||
* 'data_path' cell array (size 1 or N) with with the paths to the HDF5 files conaining the Eiger data
|
||||
* 'data_location' cell array (size 1 or N) with the dataset location(s) within the file(s)
|
||||
* 'nthreads' with the desired number of parallel read processes
|
||||
|
||||
Data is expected to be present in the following form:
|
||||
|
||||
GROUP "/" {
|
||||
GROUP "eh5" {
|
||||
DATASET "images" {
|
||||
DATATYPE H5T_STD_U32LE
|
||||
DATASPACE SIMPLE { ( 464, 514, 1030 ) / ( 464, 514, 1030 ) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Here, there are 464 images with 514x1030 pixels. The file must be readable with the standard HDF5 library version 1.10.2
|
||||
|
||||
For the **Pilatus** detector
|
||||
* 'extension' with value 'cbf'
|
||||
* 'data_path' cell array with the paths of the CBF data files
|
||||
* 'nthreads' with the desired number of parallel reader threads
|
||||
Every data file must contain lines like
|
||||
|
||||
conversions="x-CBF_BYTE_OFFSET"
|
||||
X-Binary-Size-Fastest-Dimension: 1475
|
||||
X-Binary-Size-Second-Dimension: 1679
|
||||
|
||||
where 1475x1679 are the image dimensions and the image data should be byte offset encoded.
|
||||
|
||||
For the **Moench** detector
|
||||
* 'extension' with value 'tiff'
|
||||
* 'data_path' cell array with the paths of the TIFF data files
|
||||
* 'nthreads' with the desired number of parallel reader threads
|
||||
The files are expected to contain raw single precision floating point data in one image plane, stored in IMAGELENGTH chunks of size IMAGEWIDTH. These tags must also give the image dimensions.
|
||||
|
||||
From MATLAB call it in the following way:
|
||||
|
||||
arg = struct()
|
||||
arg.extension = 'h5'
|
||||
arg.precision = 'single'
|
||||
arg.nthreads = 1
|
||||
arg.data_path = { 'test.h5' }
|
||||
arg.data_location = { '/eiger/images' }
|
||||
A = io.read_measurement(arg);
|
||||
|
||||
The array *A* will contain the measurement datasets as a multidimensional array [Ncols, Nrows, Nimages]. The function will print out some information about the arguments, if the environment variable *PTYCHO_READ_VERBOSE* is set to one of the values '1', 'yes', 'true'.
|
||||
|
||||
setenv('PTYCHO_READ_VERBOSE', 'yes')
|
||||
|
||||
The above command accomplishes this inside MATLAB.
|
||||
|
||||
Debug output
|
||||
------------
|
||||
|
||||
If debug output about the inner workings of the functions is desired, the environment variable *PTYCHO_READ_DEBUG* should be set to the filename where debug output will end up.
|
||||
|
||||
setenv('PTYCHO_READ_DEBUG', '/tmp/debug_output.txt')
|
||||
|
||||
The above command accomplishes this inside MATLAB.
|
||||
|
||||
Compilation
|
||||
-----------
|
||||
|
||||
The code works only with MATLAB versions at or above 2018a, because the code requires the new interleaved complex array format. You need to load the matlab/2018a or a later MATLAB environment module in order to compile the code. Aditionally the gcc version and an apropriate HDF5 serial and TIFF environment module need to be loaded. This can be done by sourcing the *setup-environment.sh* script. After these steps module list shoud approximately like this:
|
||||
|
||||
[stadler_h@ra-l-002 ~]$ module list
|
||||
Currently Loaded Modulefiles:
|
||||
1) gcc/6.3.0 2) hdf5_serial/1.8.18 3) matlab/2018a 4) tiff/4.0.9
|
||||
|
||||
Now you should be ready to compile the code:
|
||||
|
||||
[stadler_h@ra-l-002 ptycho_reader]$ make
|
||||
mex -R2018a -c debug_helper.cc
|
||||
Building with 'g++'.
|
||||
MEX completed successfully.
|
||||
...
|
||||
mex -R2018a readObjectData.cc read_object_data.o hdf5_helper.o debug_helper.o /opt/psi/Compiler/hdf5_serial/1.8.18/gcc/6.3.0/lib/libhdf5.a -lz -ldl -output ../../ptycho_read
|
||||
Building with 'g++'.
|
||||
MEX completed successfully.
|
||||
...
|
||||
mex -R2018a readMeasurementData.cc read_eiger_data.o hdf5_helper.o read_data_threaded.o debug_helper.o /opt/psi/Compiler/hdf5_serial/1.8.18/gcc/6.3.0/lib/libhdf5.a -lz -ldl -output ../../read_measurement
|
||||
Building with 'g++'.
|
||||
MEX completed successfully.
|
||||
|
||||
The make process will procude the ptycho_read and the read_measurement MEX file in the io package: *io.ptycho_read* and *io.read_measurement*
|
||||
|
||||
Intermediate files of the make process can be cleaned using the command
|
||||
|
||||
[stadler_h@ra-l-002 ptycho_reader]$ make clean
|
||||
rm -f *~ *.o
|
||||
|
||||
If you also want to delete the ptycho_read MEX file, use the command
|
||||
|
||||
[stadler_h@ra-l-002 ptycho_reader]$ make proper
|
||||
rm -f *~ *.o
|
||||
rm -f ../../ptycho_read.mexa64 ../../read_measurement.mexa64
|
||||
|
||||
Code documentation in HTML format can be produced using doxygen (tested with version 1.8.13)
|
||||
|
||||
[stadler_h@ra-l-002 ptycho_reader]$ make doc
|
||||
test -r doxygen.conf && rm -rf doc/html
|
||||
test -d doc || mkdir doc
|
||||
doxygen doxygen.conf
|
||||
...
|
||||
|
||||
Hope everything works as expected!
|
||||
@@ -0,0 +1,6 @@
|
||||
#include <fstream>
|
||||
#include <memory>
|
||||
|
||||
namespace debug {
|
||||
std::unique_ptr<std::ofstream> out; //!< pointer to debug log file
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
/*!
|
||||
* \file
|
||||
* Helper code for debug handling
|
||||
*/
|
||||
|
||||
#ifndef DEBUG_HELPER
|
||||
#define DEBUG_HELPER
|
||||
|
||||
/*!
|
||||
* \brief Debug functionality
|
||||
*
|
||||
* The environment variable PTYCHO_READ_DEBUG gives the name of the debug log file
|
||||
*/
|
||||
namespace debug {
|
||||
extern std::unique_ptr<std::ofstream> out; //!< pointer to debug file
|
||||
|
||||
/*!
|
||||
* \brief Check debug output stream
|
||||
* \return true if stream is operational, otherwise false
|
||||
*/
|
||||
inline bool check_out()
|
||||
{
|
||||
if (out.get())
|
||||
return out.get()->good();
|
||||
return false;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Initialize debug output stream
|
||||
*/
|
||||
inline void debug_init()
|
||||
{
|
||||
out.reset(nullptr);
|
||||
const char *fname = std::getenv("PTYCHO_READ_DEBUG");
|
||||
if (fname) {
|
||||
out.reset(new std::ofstream(fname, std::ios::app));
|
||||
if (! out.get()->good())
|
||||
out.reset(nullptr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Initialize debug stream
|
||||
*/
|
||||
#define DEBUG_INIT debug::debug_init()
|
||||
|
||||
/*!
|
||||
* \brief Start debug code block
|
||||
* Only access the debug stream inside such a block
|
||||
*/
|
||||
#define DEBUG if (debug::check_out())
|
||||
|
||||
/*!
|
||||
* \brief Access debug output stream
|
||||
* \return debug output stream reference
|
||||
*/
|
||||
#define OUT (*debug::out.get())
|
||||
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,34 @@
|
||||
#ifndef ENV_HELPER_H
|
||||
#define ENV_HELPER_H
|
||||
|
||||
/*!
|
||||
* \file
|
||||
* Helper functions for environment parsing
|
||||
*/
|
||||
|
||||
namespace env {
|
||||
|
||||
/*!
|
||||
* \brief Check if environment variable steered feature is enabled
|
||||
*
|
||||
* \param envvar environment variable name
|
||||
* \return true if feature is enabled
|
||||
*/
|
||||
inline bool is_enabled (const std::string &envvar)
|
||||
{
|
||||
char *val = std::getenv(envvar.c_str());
|
||||
if (! val)
|
||||
return false;
|
||||
std::string value(val);
|
||||
if (value == "1")
|
||||
return true;
|
||||
if (value == "yes")
|
||||
return true;
|
||||
if (value == "true")
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,39 @@
|
||||
/*!
|
||||
* \file
|
||||
* Helper definitions for HDF5 handling
|
||||
*/
|
||||
|
||||
#include "hdf5_helper.h"
|
||||
|
||||
namespace {
|
||||
|
||||
bool initialized = false; //!< Datastructures initialized ?
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace hdf5 {
|
||||
|
||||
template<> hid_t type_trait<double>::type = H5T_NATIVE_DOUBLE; //!< HDF5 double
|
||||
template<> hid_t type_trait<float>::type = H5T_NATIVE_FLOAT; //!< HDF5 float
|
||||
template<> hid_t type_trait<uint64_t>::type = H5T_NATIVE_UINT64; //!< HDF5 uint64_t
|
||||
template<> hid_t type_trait<unsigned int>::type = H5T_NATIVE_UINT; //!< HDF5 unsigned int
|
||||
template<> hid_t type_trait<std::complex<float>>::type = type_trait<std::complex<float>>::invalid; //!< HDF5 complex<float>
|
||||
template<> hid_t type_trait<std::complex<double>>::type = type_trait<std::complex<double>>::invalid; //!< HDF5 complex<double>
|
||||
|
||||
void initialize()
|
||||
{
|
||||
if (! initialized) {
|
||||
type_trait<std::complex<float>>::type = create_complex_type<double>();
|
||||
type_trait<std::complex<double>>::type = create_complex_type<float>();
|
||||
}
|
||||
}
|
||||
|
||||
void cleanup()
|
||||
{
|
||||
if (initialized) {
|
||||
h5type_trait::close(type_trait<std::complex<float>>::type);
|
||||
h5type_trait::close(type_trait<std::complex<double>>::type);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
/*!
|
||||
* \file
|
||||
* Helper code for HDF5 handling
|
||||
*/
|
||||
|
||||
#include <complex>
|
||||
#include <utility>
|
||||
#include "hdf5.h"
|
||||
|
||||
/*!
|
||||
* \brief HDF5 helper classes and functions
|
||||
*/
|
||||
namespace hdf5 {
|
||||
|
||||
/*!
|
||||
* \brief Type specific HDF5 features
|
||||
*
|
||||
* \tparam T native type
|
||||
*/
|
||||
template <typename T>
|
||||
struct type_trait final {
|
||||
static constexpr hid_t invalid = -1; //!< invalid type value
|
||||
static hid_t type; //!< HDF5 type handle for native type
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief HDF5 object with id
|
||||
*
|
||||
* \tparam trait trait for HDF5 object type
|
||||
*/
|
||||
template <typename trait>
|
||||
class obj final {
|
||||
obj(const obj &rhs) = delete; //!< no copy
|
||||
obj& operator= (const obj &rhs) = delete; //!< no copy
|
||||
hid_t id; //!< Wrapped object id
|
||||
public:
|
||||
static constexpr hid_t invalid = -1; //!< invalid id value
|
||||
/*!
|
||||
* \brief Constructor
|
||||
*
|
||||
* \param id_ set id to this
|
||||
*/
|
||||
explicit obj(hid_t id_) noexcept : id(id_)
|
||||
{}
|
||||
|
||||
/*!
|
||||
* \brief Destructor
|
||||
*/
|
||||
~obj() noexcept
|
||||
{
|
||||
if (valid())
|
||||
trait::close(id);
|
||||
}
|
||||
|
||||
operator hid_t() noexcept { return id; } //!< Conversion to hid_t
|
||||
bool valid() const noexcept { return id >= 0; } //!< Valid? (true if id is valid)
|
||||
|
||||
/*!
|
||||
* \brief Set new and return old id value
|
||||
*
|
||||
* \param val new id value
|
||||
* \return new id value
|
||||
*/
|
||||
hid_t set(hid_t val) noexcept
|
||||
{
|
||||
if (valid())
|
||||
trait::close(id);
|
||||
id = val;
|
||||
return val;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Grab id and invalidate this object
|
||||
*
|
||||
* \return id
|
||||
*/
|
||||
hid_t grab() noexcept
|
||||
{
|
||||
hid_t val = id;
|
||||
id = invalid;
|
||||
return val;
|
||||
}
|
||||
};
|
||||
|
||||
/*! \brief HDF5 File trait */
|
||||
struct file_trait final {
|
||||
static void close(hid_t id) noexcept { H5Fclose(id); } //!< Close operation
|
||||
};
|
||||
|
||||
/*! \brief HDF5 Attribute trait */
|
||||
struct attribute_trait final {
|
||||
static void close(hid_t id) noexcept { H5Aclose(id); } //!< Close operation
|
||||
};
|
||||
|
||||
/*! \brief HDF5 Group trait */
|
||||
struct group_trait final {
|
||||
static void close(hid_t id) noexcept { H5Gclose(id); } //!< Close operation
|
||||
};
|
||||
|
||||
/*! \brief HDF5 Dataset trait */
|
||||
struct dataset_trait final {
|
||||
static void close(hid_t id) noexcept { H5Dclose(id); } //!< Close operation
|
||||
};
|
||||
|
||||
/*! \brief HDF5 Dataspace trait */
|
||||
struct dataspace_trait final {
|
||||
static void close(hid_t id) noexcept { H5Sclose(id); } //!< Close operation
|
||||
};
|
||||
|
||||
/*! \brief HDF5 Type trait */
|
||||
struct h5type_trait final {
|
||||
static void close(hid_t id) noexcept { H5Tclose(id); } //!< Close operation
|
||||
};
|
||||
|
||||
/*! \brief HDF5 property trait */
|
||||
struct property_trait final {
|
||||
static void close(hid_t id) noexcept { H5Pclose(id); } //!< Close operation
|
||||
};
|
||||
|
||||
using file = obj<file_trait>; //!< HDF5 file
|
||||
using attribute = obj<attribute_trait>; //!< HDF5 attribute
|
||||
using group = obj<group_trait>; //!< HDF5 group
|
||||
using dataset = obj<dataset_trait>; //!< HDF5 dataset
|
||||
using dataspace = obj<dataspace_trait>; //!< HDF5 dataspace
|
||||
using h5type = obj<h5type_trait>; //!< HDF5 type
|
||||
using property = obj<property_trait>; //!< HDF5 property
|
||||
|
||||
/*!
|
||||
* \brief HDF5 exception
|
||||
*/
|
||||
class exception final : public std::exception {
|
||||
std::string msg; //!< error message
|
||||
|
||||
public:
|
||||
/*!
|
||||
* \brief Constructor
|
||||
*
|
||||
* \param message error message
|
||||
*/
|
||||
exception(std::string &&message)
|
||||
: msg(message)
|
||||
{}
|
||||
|
||||
/*!
|
||||
* \brief Destructor
|
||||
*/
|
||||
virtual ~exception() {}
|
||||
|
||||
/*!
|
||||
* \brief Get error message
|
||||
*
|
||||
* \return error message
|
||||
*/
|
||||
const char* what() const noexcept override
|
||||
{
|
||||
return msg.c_str();
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief Check hdf5 result
|
||||
*
|
||||
* \throw exception if result is not ok
|
||||
*/
|
||||
inline void check_result(herr_t id, std::string &&message) {
|
||||
if (id < 0)
|
||||
throw exception(std::forward<decltype(message)>(message));
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Check hdf5 object
|
||||
*
|
||||
* \throw exception if object is not ok
|
||||
*/
|
||||
inline void check_object(hid_t id, std::string &&message) {
|
||||
if (id < 0)
|
||||
throw exception(std::forward<decltype(message)>(message));
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Initialize data
|
||||
*
|
||||
* Must be called before using the complex data types
|
||||
*/
|
||||
void initialize();
|
||||
|
||||
/*!
|
||||
* \brief Cleanup data
|
||||
*/
|
||||
void cleanup();
|
||||
|
||||
/*!
|
||||
* \brief Convenience initializer class
|
||||
*/
|
||||
struct initializer final {
|
||||
initializer() { initialize(); } //!< Constructor initializes internal hdf5 helper data structures
|
||||
~initializer() { cleanup(); } //!< Destructor cleans up internal hdf5 helper data structures
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief Create complex compound type
|
||||
*
|
||||
* \return type id
|
||||
* \throw hdf5::exception if an error happens
|
||||
*/
|
||||
template <typename T>
|
||||
hid_t create_complex_type()
|
||||
{
|
||||
hdf5::h5type type(H5Tcreate(H5T_COMPOUND, sizeof(std::complex<T>)));
|
||||
hdf5::check_result(type, "Unable to create complex compound type");
|
||||
hdf5::check_result(H5Tinsert(type, "r", 0, hdf5::type_trait<T>::type), "Unable to insert first complex compound member");
|
||||
hdf5::check_result(H5Tinsert(type, "i", sizeof(T), hdf5::type_trait<T>::type), "Unable to insert second complex compound member");
|
||||
return type.grab();
|
||||
}
|
||||
|
||||
} // namespace hdf5
|
||||
@@ -0,0 +1,444 @@
|
||||
#ifndef MEX_HELPER
|
||||
#define MEX_HELPER
|
||||
|
||||
/*!
|
||||
* \file
|
||||
* Helper code for MEX data
|
||||
*
|
||||
* include \<cstdint\> before this one
|
||||
*/
|
||||
|
||||
/*!
|
||||
* \brief Unique pointer for MATLAB allocated object
|
||||
*
|
||||
* \tparam T type of object
|
||||
*/
|
||||
template <typename T>
|
||||
struct mx_deleter final {
|
||||
/*!
|
||||
* \brief deleter for pointer
|
||||
*
|
||||
* \param p pointer
|
||||
*/
|
||||
void operator()(T *p) const noexcept {
|
||||
if (p)
|
||||
mxFree(p);
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief Unique pointer for MATLAB array
|
||||
*/
|
||||
template <>
|
||||
struct mx_deleter<mxArray> final {
|
||||
/*!
|
||||
* \brief deleter for MATLAB array
|
||||
*
|
||||
* \param p MATLAB array
|
||||
*/
|
||||
void operator()(mxArray *p) const noexcept {
|
||||
mxDestroyArray(p);
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief unique_ptr for MATLAB objects
|
||||
*
|
||||
* \tparam T object type
|
||||
*/
|
||||
template <typename T>
|
||||
using mx_ptr = std::unique_ptr<T, mx_deleter<T>>;
|
||||
|
||||
/*!
|
||||
* \brief Type specific MATLAB features
|
||||
*
|
||||
* \tparam T native type
|
||||
*/
|
||||
template <typename T>
|
||||
struct mx_trait final {
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief Numeric real
|
||||
*/
|
||||
struct mx_numeric {
|
||||
/*!
|
||||
* \brief first int64_t
|
||||
* \param pa array
|
||||
* \param index array index
|
||||
* \return first int64_t in data
|
||||
*/
|
||||
virtual int64_t get_int64(const mxArray *pa, std::size_t index=0) = 0;
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief Double specific MATLAB features
|
||||
*/
|
||||
template <>
|
||||
struct mx_trait<double> final : public mx_numeric {
|
||||
using type = double; //!< base type
|
||||
using complex_type = mxComplexDouble; //!< complex type
|
||||
constexpr static mxClassID class_id = mxDOUBLE_CLASS; //!< MATLAB class id
|
||||
|
||||
/*!
|
||||
* \brief Get data pointer
|
||||
*
|
||||
* \param pa complex MATLAB array
|
||||
* \return pointer to complex array data
|
||||
*/
|
||||
static complex_type* get_complex(mxArray *pa)
|
||||
{
|
||||
return mxGetComplexDoubles(pa);
|
||||
}
|
||||
/*!
|
||||
* \brief Get data pointer
|
||||
* \param pa MATLAB array
|
||||
* \return pointer to array data
|
||||
*/
|
||||
static type* get(mxArray *pa)
|
||||
{
|
||||
return mxGetDoubles(pa);
|
||||
}
|
||||
/*!
|
||||
* \brief first int64_t
|
||||
* \param pa array
|
||||
* \param index array index
|
||||
* \return first int64_t in data
|
||||
*/
|
||||
int64_t get_int64(const mxArray *pa, std::size_t index=0) override
|
||||
{
|
||||
return *(get(const_cast<mxArray*>(pa)) + index);
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief Float specific MATLAB features
|
||||
*/
|
||||
template <>
|
||||
struct mx_trait<float> final : public mx_numeric {
|
||||
using type = float; //!< base type
|
||||
using complex_type = mxComplexSingle; //!< complex type
|
||||
constexpr static mxClassID class_id = mxSINGLE_CLASS; //!< MATLAB class id
|
||||
|
||||
/*!
|
||||
* \brief Get data pointer
|
||||
*
|
||||
* \param pa complex MATLAB array
|
||||
* \return pointer to complex array data
|
||||
*/
|
||||
static complex_type* get_complex(mxArray *pa)
|
||||
{
|
||||
return mxGetComplexSingles(pa);
|
||||
}
|
||||
/*!
|
||||
* \brief Get data pointer
|
||||
* \param pa MATLAB array
|
||||
* \return pointer to array data
|
||||
*/
|
||||
static type* get(mxArray *pa)
|
||||
{
|
||||
return mxGetSingles(pa);
|
||||
}
|
||||
/*!
|
||||
* \brief first int64_t
|
||||
* \param pa array
|
||||
* \param index array index
|
||||
* \return first int64_t in data
|
||||
*/
|
||||
int64_t get_int64(const mxArray *pa, std::size_t index=0) override
|
||||
{
|
||||
return *(get(const_cast<mxArray*>(pa)) + index);
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief int8 specific MATLAB features
|
||||
*/
|
||||
template <>
|
||||
struct mx_trait<int8_t> final : public mx_numeric {
|
||||
using type = int8_t; //!< base type
|
||||
constexpr static mxClassID class_id = mxINT8_CLASS; //!< MATLAB class id
|
||||
/*!
|
||||
* \brief Get data pointer
|
||||
* \param pa MATLAB array
|
||||
* \return pointer to array data
|
||||
*/
|
||||
static type* get(mxArray *pa)
|
||||
{
|
||||
return mxGetInt8s(pa);
|
||||
}
|
||||
/*!
|
||||
* \brief first int64_t
|
||||
* \param pa array
|
||||
* \param index array index
|
||||
* \return first int64_t in data
|
||||
*/
|
||||
int64_t get_int64(const mxArray *pa, std::size_t index=0) override
|
||||
{
|
||||
return *(get(const_cast<mxArray*>(pa)) + index);
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief uint8 specific MATLAB features
|
||||
*/
|
||||
template <>
|
||||
struct mx_trait<uint8_t> final : public mx_numeric {
|
||||
using type = uint8_t; //!< base type
|
||||
constexpr static mxClassID class_id = mxUINT8_CLASS; //!< MATLAB class id
|
||||
/*!
|
||||
* \brief Get data pointer
|
||||
* \param pa MATLAB array
|
||||
* \return pointer to array data
|
||||
*/
|
||||
static type* get(mxArray *pa)
|
||||
{
|
||||
return mxGetUint8s(pa);
|
||||
}
|
||||
/*!
|
||||
* \brief first int64_t
|
||||
* \param pa array
|
||||
* \param index array index
|
||||
* \return first int64_t in data
|
||||
*/
|
||||
int64_t get_int64(const mxArray *pa, std::size_t index=0) override
|
||||
{
|
||||
return *(get(const_cast<mxArray*>(pa)) + index);
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief int16 specific MATLAB features
|
||||
*/
|
||||
template <>
|
||||
struct mx_trait<int16_t> final : public mx_numeric {
|
||||
using type = int16_t; //!< base type
|
||||
constexpr static mxClassID class_id = mxINT16_CLASS; //!< MATLAB class id
|
||||
/*!
|
||||
* \brief Get data pointer
|
||||
* \param pa MATLAB array
|
||||
* \return pointer to array data
|
||||
*/
|
||||
static type* get(mxArray *pa)
|
||||
{
|
||||
return mxGetInt16s(pa);
|
||||
}
|
||||
/*!
|
||||
* \brief first int64_t
|
||||
* \param pa array
|
||||
* \param index array index
|
||||
* \return first int64_t in data
|
||||
*/
|
||||
int64_t get_int64(const mxArray *pa, std::size_t index=0) override
|
||||
{
|
||||
return *(get(const_cast<mxArray*>(pa)) + index);
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief uint16 specific MATLAB features
|
||||
*/
|
||||
template <>
|
||||
struct mx_trait<uint16_t> final : public mx_numeric {
|
||||
using type = uint16_t; //!< base type
|
||||
constexpr static mxClassID class_id = mxUINT16_CLASS; //!< MATLAB class id
|
||||
/*!
|
||||
* \brief Get data pointer
|
||||
* \param pa MATLAB array
|
||||
* \return pointer to array data
|
||||
*/
|
||||
static type* get(mxArray *pa)
|
||||
{
|
||||
return mxGetUint16s(pa);
|
||||
}
|
||||
/*!
|
||||
* \brief first int64_t
|
||||
* \param pa array
|
||||
* \param index array index
|
||||
* \return first int64_t in data
|
||||
*/
|
||||
int64_t get_int64(const mxArray *pa, std::size_t index=0) override
|
||||
{
|
||||
return *(get(const_cast<mxArray*>(pa)) + index);
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief int32 specific MATLAB features
|
||||
*/
|
||||
template <>
|
||||
struct mx_trait<int32_t> final : public mx_numeric {
|
||||
using type = int32_t; //!< base type
|
||||
constexpr static mxClassID class_id = mxINT32_CLASS; //!< MATLAB class id
|
||||
/*!
|
||||
* \brief Get data pointer
|
||||
* \param pa MATLAB array
|
||||
* \return pointer to array data
|
||||
*/
|
||||
static type* get(mxArray *pa)
|
||||
{
|
||||
return mxGetInt32s(pa);
|
||||
}
|
||||
/*!
|
||||
* \brief first int64_t
|
||||
* \param pa array
|
||||
* \param index array index
|
||||
* \return first int64_t in data
|
||||
*/
|
||||
int64_t get_int64(const mxArray *pa, std::size_t index=0) override
|
||||
{
|
||||
return *(get(const_cast<mxArray*>(pa)) + index);
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief uint32 specific MATLAB features
|
||||
*/
|
||||
template <>
|
||||
struct mx_trait<uint32_t> final : public mx_numeric {
|
||||
using type = uint32_t; //!< base type
|
||||
constexpr static mxClassID class_id = mxUINT32_CLASS; //!< MATLAB class id
|
||||
/*!
|
||||
* \brief Get data pointer
|
||||
* \param pa MATLAB array
|
||||
* \return pointer to array data
|
||||
*/
|
||||
static type* get(mxArray *pa)
|
||||
{
|
||||
return mxGetUint32s(pa);
|
||||
}
|
||||
/*!
|
||||
* \brief first int64_t
|
||||
* \param pa array
|
||||
* \param index array index
|
||||
* \return first int64_t in data
|
||||
*/
|
||||
int64_t get_int64(const mxArray *pa, std::size_t index=0) override
|
||||
{
|
||||
return *(get(const_cast<mxArray*>(pa)) + index);
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief int64 specific MATLAB features
|
||||
*/
|
||||
template <>
|
||||
struct mx_trait<int64_t> final : public mx_numeric {
|
||||
using type = int64_t; //!< base type
|
||||
constexpr static mxClassID class_id = mxINT64_CLASS; //!< MATLAB class id
|
||||
/*!
|
||||
* \brief Get data pointer
|
||||
* \param pa MATLAB array
|
||||
* \return pointer to array data
|
||||
*/
|
||||
static type* get(mxArray *pa)
|
||||
{
|
||||
return mxGetInt64s(pa);
|
||||
}
|
||||
/*!
|
||||
* \brief first int64_t
|
||||
* \param pa array
|
||||
* \param index array index
|
||||
* \return first int64_t in data
|
||||
*/
|
||||
int64_t get_int64(const mxArray *pa, std::size_t index=0) override
|
||||
{
|
||||
return *(get(const_cast<mxArray*>(pa)) + index);
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief uint64 specific MATLAB features
|
||||
*/
|
||||
template <>
|
||||
struct mx_trait<uint64_t> final : public mx_numeric {
|
||||
using type = uint64_t; //!< base type
|
||||
constexpr static mxClassID class_id = mxUINT64_CLASS; //!< MATLAB class id
|
||||
/*!
|
||||
* \brief Get data pointer
|
||||
* \param pa MATLAB array
|
||||
* \return pointer to array data
|
||||
*/
|
||||
static type* get(mxArray *pa)
|
||||
{
|
||||
return mxGetUint64s(pa);
|
||||
}
|
||||
/*!
|
||||
* \brief first int64_t
|
||||
* \param pa array
|
||||
* \param index array index
|
||||
* \return first int64_t in data
|
||||
*/
|
||||
int64_t get_int64(const mxArray *pa, std::size_t index=0) override
|
||||
{
|
||||
return *(get(const_cast<mxArray*>(pa)) + index);
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief get numeric trait for real numeric array
|
||||
* \param pa array
|
||||
* \return numeric trait or nullptr if not real numeric
|
||||
*/
|
||||
inline mx_numeric* get_numeric_trait(const mxArray *pa)
|
||||
{
|
||||
if (mxIsComplex(pa))
|
||||
return nullptr;
|
||||
switch (mxGetClassID(pa)) {
|
||||
case mxDOUBLE_CLASS:
|
||||
return new mx_trait<double>;
|
||||
case mxSINGLE_CLASS:
|
||||
return new mx_trait<float>;
|
||||
case mxINT8_CLASS:
|
||||
return new mx_trait<int8_t>;
|
||||
case mxUINT8_CLASS:
|
||||
return new mx_trait<uint8_t>;
|
||||
case mxINT16_CLASS:
|
||||
return new mx_trait<int16_t>;
|
||||
case mxUINT16_CLASS:
|
||||
return new mx_trait<uint16_t>;
|
||||
case mxINT32_CLASS:
|
||||
return new mx_trait<int32_t>;
|
||||
case mxUINT32_CLASS:
|
||||
return new mx_trait<uint32_t>;
|
||||
case mxINT64_CLASS:
|
||||
return new mx_trait<int64_t>;
|
||||
case mxUINT64_CLASS:
|
||||
return new mx_trait<uint64_t>;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
template <>
|
||||
struct mx_trait<char> final {
|
||||
constexpr static mxClassID class_id = mxCHAR_CLASS; //!< MATLAB class id
|
||||
/*!
|
||||
* \brief Get string
|
||||
* \param pa array
|
||||
* \return string
|
||||
*/
|
||||
static std::string get_string(const mxArray *pa)
|
||||
{
|
||||
const std::size_t nchars = mxGetN(pa);
|
||||
std::unique_ptr<char> buf(new char[nchars+1]);
|
||||
if (mxGetString(pa, buf.get(), nchars+1))
|
||||
return std::string();
|
||||
return std::string(buf.get(), nchars);
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief get char trait for class id
|
||||
* \param pa array
|
||||
* \return numeric trait or nullptr if not numeric
|
||||
*/
|
||||
inline mx_trait<char>* get_char_trait(const mxArray *pa)
|
||||
{
|
||||
switch (mxGetClassID(pa)) {
|
||||
case mxCHAR_CLASS:
|
||||
return new mx_trait<char>;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
#endif // ifndef MEX_HELPER
|
||||
@@ -0,0 +1,268 @@
|
||||
#ifndef MULTI_PROCESSING_H
|
||||
#define MULTI_PROCESSING_H
|
||||
|
||||
/*!
|
||||
* \file
|
||||
* Provide functionality for multi processing
|
||||
*/
|
||||
|
||||
/*!
|
||||
* \brief Multiprocessing support functions and types
|
||||
*/
|
||||
namespace mp {
|
||||
|
||||
/*!
|
||||
* \brief Buffer object managing destination and shared memory areas
|
||||
* \tparam e_type buffer element type
|
||||
*/
|
||||
template <typename e_type>
|
||||
struct buf {
|
||||
const unsigned int msg_size; //!< maximum message size per process
|
||||
e_type *destination; //!< pointer to destination memory
|
||||
e_type *shared; //!< pointer to shared memory
|
||||
std::size_t length; //!< total data elements
|
||||
std::size_t chunk_size; //!< data elements for one chunk
|
||||
int chunks_per_proc; //!< number of chunks for a process to handle
|
||||
int first_full; //!< first process with non reduced number of chunks
|
||||
int num_procs; //!< number of read processes
|
||||
|
||||
/*!
|
||||
* \brief Constructor
|
||||
*
|
||||
* \param dest Destination pointer
|
||||
* \param dims Result matrix dimensions
|
||||
* \param nprocs Number of read processes
|
||||
* \param message_size Size of message area per child process in bytes
|
||||
*/
|
||||
buf(e_type *dest, const std::vector<mwSize> &dims, int nprocs, unsigned int message_size=64u)
|
||||
: msg_size(message_size ? message_size : 64u), destination(dest), shared(nullptr), length(0), chunk_size(1), chunks_per_proc(0), first_full(0), num_procs(nprocs)
|
||||
{
|
||||
assert(! dims.empty());
|
||||
assert(nprocs >= 1);
|
||||
int nchunks = dims[0];
|
||||
chunks_per_proc = (nchunks + nprocs - 1) / nprocs;
|
||||
first_full = (nprocs * chunks_per_proc) - nchunks;
|
||||
|
||||
for (int i=1; i<dims.size(); i++)
|
||||
chunk_size *= dims[i];
|
||||
length = nchunks * chunk_size;
|
||||
|
||||
DEBUG {
|
||||
OUT << nprocs << "np " << nchunks << "nc " << chunks_per_proc << "cpp " << first_full << "ff " << chunk_size << "cs " << length << 'l' << std::endl;
|
||||
}
|
||||
|
||||
if (nprocs > 1) {
|
||||
void *sbuf = mmap(nullptr, (length - offset(1) * chunk_size) * sizeof(e_type) + (num_procs - 1) * msg_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
|
||||
if (sbuf == MAP_FAILED)
|
||||
throw std::runtime_error(std::string("mmap error - ") + strerror(errno));
|
||||
shared = static_cast<e_type *>(sbuf);
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Destructor
|
||||
*/
|
||||
~buf() noexcept
|
||||
{
|
||||
if (shared)
|
||||
munmap(shared, (length - offset(1) * chunk_size) * sizeof(e_type) + (num_procs - 1) * msg_size);
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Get shareable buffer pointer
|
||||
*
|
||||
* \param proc process number
|
||||
* \return Pointer to buffer area accessible from parent process
|
||||
*/
|
||||
e_type *get(int proc) noexcept
|
||||
{
|
||||
if (! proc) {
|
||||
DEBUG {
|
||||
OUT << "buffer " << proc << ": destination+0 at address " << destination << std::endl;
|
||||
}
|
||||
return destination;
|
||||
}
|
||||
std::size_t buf_start = offset(proc, true) * chunk_size;
|
||||
DEBUG {
|
||||
OUT << "buffer " << proc << ": shared+" << buf_start << " at address " << &shared[buf_start] << std::endl;
|
||||
}
|
||||
return &shared[buf_start];
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Destination buffer
|
||||
*
|
||||
* \param proc process number
|
||||
* \return Pointer to destination buffer area for process proc
|
||||
*/
|
||||
e_type *get_dest(int proc) noexcept
|
||||
{
|
||||
return destination + offset(proc) * chunk_size;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Copy data in shared memory to destination
|
||||
*/
|
||||
void copy() noexcept
|
||||
{
|
||||
if (shared) {
|
||||
DEBUG {
|
||||
OUT << "Copy " << (length - offset(1) * chunk_size) << " elements to destination+" << (offset(1) * chunk_size) << std::endl;
|
||||
}
|
||||
std::copy(shared, shared + (length - offset(1) * chunk_size), destination + (offset(1) * chunk_size));
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Object offset
|
||||
*
|
||||
* \param proc process number
|
||||
* \param shared offset into shared memory?
|
||||
* \return First chunk index for proc
|
||||
*/
|
||||
int offset(int proc, bool shared=false) noexcept
|
||||
{
|
||||
int offset = proc * chunks_per_proc;
|
||||
if (proc < first_full)
|
||||
offset -= proc;
|
||||
else
|
||||
offset -= first_full;
|
||||
if (shared)
|
||||
offset -= chunks_per_proc - (first_full ? 1 : 0);
|
||||
return offset;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Get pointer to message
|
||||
*
|
||||
* For child processes returns a pointer to the message area of size msg_size
|
||||
*
|
||||
* \param proc process number (must not be the parent process 0)
|
||||
* \return pointer to message area
|
||||
*/
|
||||
char *get_msg(int proc)
|
||||
{
|
||||
if (proc)
|
||||
return reinterpret_cast<char *>(&shared[length - offset(1) * chunk_size]) + (proc - 1) * msg_size;
|
||||
throw std::logic_error("no message area for parent process");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Record message
|
||||
*
|
||||
* Used to record a child process error message in the shared memory area
|
||||
*
|
||||
* \param msg message
|
||||
* \param proc process number (must not be the parent process 0)
|
||||
*/
|
||||
void message(const char *msg, int proc)
|
||||
{
|
||||
std::strncpy(get_msg(proc), msg, msg_size-1);
|
||||
}
|
||||
}; // struct buf
|
||||
|
||||
struct exception : public std::exception {
|
||||
bool converted = false;
|
||||
int process = 0;
|
||||
std::string error_message;
|
||||
|
||||
inline exception ()
|
||||
{}
|
||||
|
||||
inline exception (int proc, const std::string &msg)
|
||||
: process(proc), error_message(msg)
|
||||
{}
|
||||
|
||||
inline const char* what() const noexcept override
|
||||
{
|
||||
try {
|
||||
if (! converted) {
|
||||
std::string &msg = const_cast<std::string&>(error_message);
|
||||
std::ostringstream oss;
|
||||
oss << msg << " (proc " << process << ')';
|
||||
msg = oss.str();
|
||||
const_cast<bool&>(converted) = true;
|
||||
}
|
||||
} catch (...) {}
|
||||
return error_message.c_str();
|
||||
}
|
||||
}; // struct exception
|
||||
|
||||
template<typename e_type>
|
||||
inline void run (int nprocs, buf<e_type> &buffer,
|
||||
std::function<void(int, buf<e_type>&)> &&func)
|
||||
{
|
||||
if (nprocs <= 0)
|
||||
return;
|
||||
|
||||
int proc = 0; // process number starting with 0
|
||||
int pids[nprocs] = {0}; // first entry (parent) unused
|
||||
try {
|
||||
for (proc=nprocs-1; proc; --proc) {
|
||||
if ((pids[proc] = fork()) < 0) {
|
||||
proc = 0;
|
||||
throw exception(0, "process spawning failed");
|
||||
}
|
||||
if (! pids[proc])
|
||||
break;
|
||||
}
|
||||
|
||||
DEBUG_INIT;
|
||||
func(proc, buffer);
|
||||
|
||||
if (proc) {
|
||||
DEBUG {
|
||||
OUT << "Process " << proc << " finished" << std::endl;
|
||||
}
|
||||
std::_Exit((EXIT_SUCCESS));
|
||||
}
|
||||
} catch (std::exception &ex) {
|
||||
DEBUG {
|
||||
OUT << "Process " << proc << " exception: " << ex.what() << std::endl;
|
||||
}
|
||||
if (proc) {
|
||||
buffer.message(ex.what(), proc);
|
||||
std::_Exit((EXIT_FAILURE));
|
||||
}
|
||||
for (proc=nprocs-1; proc; --proc) {
|
||||
if (pids[proc] > 0) {
|
||||
int status;
|
||||
waitpid(pids[proc], &status, 0); // prevent zombies
|
||||
}
|
||||
}
|
||||
throw;
|
||||
}
|
||||
|
||||
bool failed = false;
|
||||
exception ex;
|
||||
for (proc=nprocs-1; proc; --proc) {
|
||||
int status;
|
||||
bool with_error = false;
|
||||
if (waitpid(pids[proc], &status, 0) != pids[proc]) {
|
||||
with_error = true;
|
||||
ex.process = proc;
|
||||
ex.error_message = std::strerror(errno);
|
||||
} else if (! WIFEXITED(status)) {
|
||||
with_error = true;
|
||||
ex.process = proc;
|
||||
ex.error_message = "did not exit normally";
|
||||
} else if (WEXITSTATUS(status) != (EXIT_SUCCESS)) {
|
||||
with_error = true;
|
||||
ex.process = proc;
|
||||
ex.error_message = buffer.get_msg(proc);
|
||||
}
|
||||
DEBUG {
|
||||
OUT << "Process " << proc << ": finished " << (with_error ? "with error" : "successfully") << std::endl;
|
||||
}
|
||||
failed |= with_error;
|
||||
}
|
||||
if (failed)
|
||||
throw ex;
|
||||
|
||||
buffer.copy();
|
||||
}
|
||||
|
||||
} // namespace mp
|
||||
|
||||
#endif // ifndef MULTI_PROCESSING_H
|
||||
@@ -0,0 +1,59 @@
|
||||
#ifndef PRECISION_H
|
||||
#define PRECISION_H
|
||||
|
||||
/*!
|
||||
* \file
|
||||
* Precision handling support
|
||||
*
|
||||
* \pre needs standard exceptions and string
|
||||
*/
|
||||
|
||||
/*!
|
||||
* \brief Precision related functions and types
|
||||
*/
|
||||
namespace precision {
|
||||
|
||||
/*!
|
||||
* \brief Single or double precision
|
||||
*/
|
||||
enum type : unsigned char {
|
||||
Single, //!< single precision
|
||||
Double //!< double precision
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief Precision from string
|
||||
* \arg str precision string
|
||||
* \return precision
|
||||
*/
|
||||
inline type from_str(const std::string &str)
|
||||
{
|
||||
type prec = Single;
|
||||
if (str == "double")
|
||||
prec = Double;
|
||||
else if (str != "single")
|
||||
throw std::invalid_argument("undefined precision string, must be 'single' or 'double'");
|
||||
return prec;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Precision to string
|
||||
* \arg prec precision
|
||||
* \return string
|
||||
*/
|
||||
inline const char* to_str(type prec)
|
||||
{
|
||||
switch (prec) {
|
||||
case Single:
|
||||
return "single";
|
||||
case Double:
|
||||
return "double";
|
||||
default:
|
||||
throw std::invalid_argument("undefined precision value");
|
||||
}
|
||||
return "undef";
|
||||
}
|
||||
|
||||
} // namespace precision
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,310 @@
|
||||
/*!
|
||||
* \file
|
||||
* Read Measurement Data
|
||||
*
|
||||
* This file contains the main mex code for reading measurement data into MATLAB memory.
|
||||
*/
|
||||
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <stdexcept>
|
||||
#include <fstream>
|
||||
#include "mex.h"
|
||||
#include "precision.h"
|
||||
#include "read_eiger_data.h"
|
||||
#include "read_data_threaded.h"
|
||||
#include "debug_helper.h"
|
||||
|
||||
namespace {
|
||||
|
||||
namespace p {
|
||||
constexpr char format[] = "extension"; //!< extension field
|
||||
constexpr char image_size[] = "asize"; //!< image size field
|
||||
constexpr char roi_center[] = "ctr"; //!< ROI center field
|
||||
constexpr char scan_number[] = "scan_number"; //!< scan number field
|
||||
constexpr char data_path[] = "data_path"; //!< data path field
|
||||
constexpr char data_location[] = "data_location"; //!< data location field
|
||||
constexpr char data_prefix[] = "data_prefix"; //!< data prefix field
|
||||
constexpr char scan_string_format[] = "scan_string_format"; //!< scan string format field
|
||||
constexpr char precision[] = "precision"; //!< precision string ('single' or 'double')
|
||||
constexpr char nthreads[] = "nthreads"; //!< number of parallel threads or processes
|
||||
} // namespace p
|
||||
|
||||
#include "env_helper.h"
|
||||
#include "mex_helper.h"
|
||||
|
||||
/*!
|
||||
* \brief Get field from struct
|
||||
*
|
||||
* \param pa Pointer to MATLAB struct array
|
||||
* \param field_name Field name
|
||||
* \return Pointer to MATALAB array of field value
|
||||
*/
|
||||
mxArray* getField(const mxArray *pa, const char *field_name)
|
||||
{
|
||||
mxArray *field = mxGetField(pa, 0, field_name);
|
||||
if (! field)
|
||||
throw std::invalid_argument(std::string("Field ")+field_name+" not found in structure!");
|
||||
return field;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Get char field
|
||||
*
|
||||
* \param pa MATLAB struct array
|
||||
* \param field_name field name
|
||||
* \return field string value
|
||||
*/
|
||||
std::string getCharField(const mxArray *pa, const char *field_name)
|
||||
{
|
||||
mxArray *field = getField(pa, field_name);
|
||||
if (! mxIsChar(field))
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:field:type:notchar", "Field %s not a char array!", field_name);
|
||||
if (mxGetNumberOfDimensions(field)!=2 || mxGetM(field)!=1)
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:field:dim", "Field %s is not a single char array!", field_name);
|
||||
std::size_t nchars = mxGetN(field);
|
||||
char buf[nchars+1];
|
||||
if (mxGetString(field, buf, nchars+1))
|
||||
mexErrMsgIdAndTxt("psi:ptycho:string:extract", "Unable to extract characters of field %s!", field_name);
|
||||
return buf;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Assign values to vector
|
||||
*
|
||||
* \param a Vector of length at least nelements
|
||||
* \param b Pointer to at least nelements values
|
||||
* \param nelements number of elements to assign
|
||||
* \tparam A Vector element type
|
||||
* \tparam B Value type
|
||||
*/
|
||||
template <typename A, typename B>
|
||||
void assign(std::vector<A> &a, const B *b, std::size_t nelements)
|
||||
{
|
||||
if (b == nullptr)
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:field:retrieval", "Unable to extract values from numeric vector field!");
|
||||
for (std::size_t i=0; i<nelements; i++)
|
||||
a[i] = b[i];
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Get vector field with long integer values
|
||||
*
|
||||
* \param pa MATLAB struct array
|
||||
* \param field_name field name
|
||||
* \param nelements expected number of elements (0 means undefined and is the default)
|
||||
* \return field string value
|
||||
* \tparam T Vector element type
|
||||
*/
|
||||
template <typename T>
|
||||
std::vector<T> getVectorField(const mxArray *pa, const char *field_name, unsigned int nelements=0)
|
||||
{
|
||||
mxArray *field = getField(pa, field_name);
|
||||
if (! mxIsNumeric(field))
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:field:type:notnumeric", "Field %s not a numeric array!", field_name);
|
||||
if (mxGetNumberOfDimensions(field)!=2 || mxGetM(field)!=1)
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:field:dim", "Field %s is not a single numeric array!", field_name);
|
||||
std::size_t N = mxGetN(field);
|
||||
if (nelements && N != nelements)
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:field:length", "Field %s doesn't have %u elements!", field_name, nelements);
|
||||
mxClassID mx_class = mxGetClassID(field);
|
||||
std::vector<T> buf(N);
|
||||
switch (mx_class) {
|
||||
case mxINT8_CLASS:
|
||||
assign(buf, mxGetInt8s(field), N);
|
||||
break;
|
||||
case mxUINT8_CLASS:
|
||||
assign(buf, mxGetUint8s(field), N);
|
||||
break;
|
||||
case mxINT16_CLASS:
|
||||
assign(buf, mxGetInt16s(field), N);
|
||||
break;
|
||||
case mxUINT16_CLASS:
|
||||
assign(buf, mxGetUint16s(field), N);
|
||||
break;
|
||||
case mxINT32_CLASS:
|
||||
assign(buf, mxGetInt32s(field), N);
|
||||
break;
|
||||
case mxUINT32_CLASS:
|
||||
assign(buf, mxGetUint32s(field), N);
|
||||
break;
|
||||
case mxINT64_CLASS:
|
||||
assign(buf, mxGetInt64s(field), N);
|
||||
break;
|
||||
case mxUINT64_CLASS:
|
||||
assign(buf, mxGetUint64s(field), N);
|
||||
break;
|
||||
case mxSINGLE_CLASS:
|
||||
assign(buf, mxGetSingles(field), N);
|
||||
break;
|
||||
case mxDOUBLE_CLASS:
|
||||
assign(buf, mxGetDoubles(field), N);
|
||||
break;
|
||||
default:
|
||||
std::string class_name(mxGetClassName(field));
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:field:class", "Field %s doesn't contain integer elements (it has class %s)!", field_name, class_name.c_str());
|
||||
}
|
||||
return buf;
|
||||
}
|
||||
|
||||
std::vector<std::string> getStrings(const mxArray *pa, const char *field_name, unsigned int nelements=0)
|
||||
{
|
||||
mxArray *field = getField(pa, field_name);
|
||||
if (mxGetM(field) != 1)
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:wrongDimensions:notOne", "Argument %s must be a 1xN one-dimensional character or cell array!", field_name);
|
||||
std::unique_ptr<mx_trait<char>> ct(get_char_trait(field));
|
||||
if (ct.get()) { // char array
|
||||
return std::vector<std::string>({ ct->get_string(field) });
|
||||
} else { // cell array
|
||||
if (! mxIsCell(field))
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:type", "Argument %s must be either character or cell array!", field_name);
|
||||
std::vector<std::string> result_vec;
|
||||
for (std::size_t j=0; j<mxGetN(field); j+=1) {
|
||||
mxArray *pa = mxGetCell(field, j);
|
||||
if (mxGetM(pa) != 1)
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:wrongDimensions:notOne", "Cell fields in argument %s must be one-dimensional character arrays!", field_name);
|
||||
ct.reset(get_char_trait(pa));
|
||||
if (! ct.get())
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:wrongType:notChar", "Cell field in argument %s must be of type char!", field_name);
|
||||
result_vec.push_back(ct->get_string(pa));
|
||||
if (! result_vec.back().size())
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:empty", "Empty string in %s argument!", field_name);
|
||||
}
|
||||
if (! result_vec.size())
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:empty", "Empty %s argument!", field_name);
|
||||
return result_vec;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
/*!
|
||||
* \brief MEX function readMeasurementData
|
||||
*
|
||||
* Input arguments:
|
||||
* 0 : csax p structure
|
||||
*
|
||||
* Output arguments:
|
||||
* 0 : Measurement data (3d or 4d array [cols, rows, n_bursts, n_positions], n_burst is optional and only used if the HHDF5 dataset is 4d)
|
||||
*
|
||||
* \param nlhs Number of left hand side (result) arguments
|
||||
* \param plhs Pointer array to left hand side arguments
|
||||
* \param nrhs Number of right hand side (input) arguments
|
||||
* \param prhs Pointer array to right hand side arguments
|
||||
*/
|
||||
void mexFunction(int nlhs, mxArray *plhs[],
|
||||
int nrhs, const mxArray *prhs[])
|
||||
{
|
||||
constexpr unsigned int ninputs = 1;
|
||||
constexpr unsigned int noutputs = 1;
|
||||
|
||||
DEBUG_INIT;
|
||||
if (nrhs != ninputs)
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:in:wrongNumber", "Wrong number of arguments, use: array = readMeasurementData(argument structure)");
|
||||
if (nlhs != noutputs)
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:out:wrongNumber", "Wrong number of output arguments, use: array = readObjectData(argument structure)");
|
||||
|
||||
if (! mxIsStruct(prhs[0]))
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:wrongType:notStruct", "Argument must be of type struct!");
|
||||
if (mxGetNumberOfDimensions(prhs[0])!=2 || mxGetM(prhs[0])!=1 || mxGetN(prhs[0])!=1)
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:wrongDimensions:notOne", "Argument must be a 1x1 struct array!");
|
||||
|
||||
bool verbose = env::is_enabled("PTYCHO_READ_VERBOSE");
|
||||
|
||||
precision::type prec = precision::type::Single;
|
||||
try {
|
||||
prec = precision::from_str(getCharField(prhs[0], p::precision));
|
||||
} catch (std::exception &ex) {
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:precision", "Precision field error: %s", ex.what());
|
||||
}
|
||||
try {
|
||||
std::string format(getCharField(prhs[0], p::format));
|
||||
std::vector<long> image_size;
|
||||
try {
|
||||
image_size = getVectorField<long>(prhs[0], p::image_size, 2);
|
||||
} catch (std::invalid_argument &ex) {}
|
||||
std::vector<long> roi_center;
|
||||
try {
|
||||
roi_center = getVectorField<long>(prhs[0], p::roi_center, 2);
|
||||
} catch (std::invalid_argument &ex) {}
|
||||
std::vector<long> nthreads(getVectorField<long>(prhs[0], p::nthreads, 1));
|
||||
std::vector<std::string> data_path(getStrings(prhs[0], p::data_path));
|
||||
// std::string scan_string_format(getCharField(prhs[0], p::scan_string_format));
|
||||
// std::string data_prefix(getCharField(prhs[0], p::data_prefix));
|
||||
// std::vector<long> scan_number(getVectorField<long>(prhs[0], p::scan_number));
|
||||
|
||||
// Format dependent arguments
|
||||
std::vector<std::string> data_location;
|
||||
if (format == "h5")
|
||||
data_location = getStrings(prhs[0], p::data_location);
|
||||
|
||||
if (verbose) {
|
||||
mexPrintf("extension: %s\n", format.c_str());
|
||||
if (data_path.size() == 1) {
|
||||
mexPrintf("data_path: %s\n", data_path[0].c_str());
|
||||
} else {
|
||||
mexPrintf("data_path first: %s\n", data_path[0].c_str());
|
||||
mexPrintf("data_path last : %s\n", data_path.back().c_str());
|
||||
}
|
||||
mexPrintf("roi_center: ");
|
||||
if (roi_center.empty()) {
|
||||
mexPrintf("from data\n");
|
||||
} else {
|
||||
mexPrintf("%ld %ld\n", roi_center[0], roi_center[1]);
|
||||
}
|
||||
mexPrintf("image_size: ");
|
||||
if (image_size.empty()) {
|
||||
mexPrintf("from data\n");
|
||||
} else {
|
||||
mexPrintf("%ld %ld\n", image_size[0], image_size[1]);
|
||||
}
|
||||
mexPrintf("precision: %s\n", precision::to_str(prec));
|
||||
// mexPrintf("data_prefix: %s\n", data_prefix.c_str());
|
||||
// mexPrintf("scan_number: [ ");
|
||||
// for (long l : scan_number)
|
||||
// mexPrintf("%ld ", l);
|
||||
// mexPrintf("]\n");
|
||||
const char *debuglog = std::getenv("PTYCHO_READ_DEBUG");
|
||||
mexPrintf("debuglog: %s - ", (debuglog ? debuglog : "not set"));
|
||||
DEBUG {
|
||||
mexPrintf("ok\n");
|
||||
OUT << "----------- read measurement data ----------------\n";
|
||||
} else {
|
||||
mexPrintf("not functional\n");
|
||||
}
|
||||
|
||||
// Format dependend verbose output
|
||||
if (format == "h5") {
|
||||
if (data_location.size() == 1) {
|
||||
mexPrintf("data_location: %s\n", data_location[0].c_str());
|
||||
} else {
|
||||
mexPrintf("data_location first: %s\n", data_location[0].c_str());
|
||||
mexPrintf("data_location last : %s\n", data_location.back().c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (format == "h5") {
|
||||
DEBUG {
|
||||
OUT << "Reading Eiger data..." << std::endl;
|
||||
}
|
||||
plhs[0] = data_prep::read_eiger_data(nthreads[0], data_path, data_location, image_size, roi_center, prec);
|
||||
} else if (format == "cbf") {
|
||||
DEBUG {
|
||||
OUT << "Reading Pilatus data..." << std::endl;
|
||||
}
|
||||
plhs[0] = data_prep::read_data_threaded("pilatus", nthreads[0], data_path, image_size, roi_center, prec);
|
||||
} else if (format == "tiff") {
|
||||
DEBUG {
|
||||
OUT << "Reading Moench data..." << std::endl;
|
||||
}
|
||||
plhs[0] = data_prep::read_data_threaded("moench", nthreads[0], data_path, image_size, roi_center, prec);
|
||||
} else {
|
||||
mexErrMsgIdAndTxt("psi:ptycho:format:unknown", "Unknown extension: %s", format.c_str());
|
||||
}
|
||||
} catch (std::exception &ex) {
|
||||
mexErrMsgIdAndTxt("psi:ptycho:struct:arg", "%s", ex.what());
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
/*!
|
||||
* \file
|
||||
* Read Object Data
|
||||
*
|
||||
* This file contains the main mex code for reading object data into MATLAB memory.
|
||||
*/
|
||||
|
||||
#include <string>
|
||||
#include <cstdlib>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <stdexcept>
|
||||
#include <fstream>
|
||||
#include "precision.h"
|
||||
#include "env_helper.h"
|
||||
#include "mex.h"
|
||||
#include "mex_helper.h"
|
||||
#include "read_object_data.h"
|
||||
#include "debug_helper.h"
|
||||
|
||||
#define USAGE "use: array = readObjectData(nProcesses, precision, [nrows, ncols], datasetPath, { filePaths, ... })"
|
||||
|
||||
/*!
|
||||
* \brief MEX function readObjectData
|
||||
*
|
||||
* Input arguments:
|
||||
* 0 : Number of read processes
|
||||
* 1 : 'double' or 'single' precision
|
||||
* 2 : Output image size [nrows, ncolumns] (optional, default value = size of first image)
|
||||
* 3 : Input object dataset path within the HDF5 file
|
||||
* 4 : Input HDF5 file paths
|
||||
*
|
||||
* Output arguments:
|
||||
* 0 : Object data (3d, 4d[with modes], or 5d[with slices]) [nobjects, nslices*, nmodes*, nrows, ncolumns]
|
||||
* 1 : (Optional) 1d array of file indices that could not be read
|
||||
*
|
||||
* \param nlhs Number of left hand side (result) arguments
|
||||
* \param plhs Pointer array to left hand side arguments
|
||||
* \param nrhs Number of right hand side (input) arguments
|
||||
* \param prhs Pointer array to right hand side arguments
|
||||
*/
|
||||
void mexFunction(int nlhs, mxArray *plhs[],
|
||||
int nrhs, const mxArray *prhs[])
|
||||
{
|
||||
constexpr int max_nprocs = 512;
|
||||
constexpr unsigned int min_inputs = 4;
|
||||
constexpr unsigned int max_inputs = 5;
|
||||
constexpr unsigned int min_outputs = 1;
|
||||
constexpr unsigned int max_outputs = 2;
|
||||
|
||||
DEBUG_INIT;
|
||||
if ((nrhs < min_inputs) || (nrhs > max_inputs))
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:in:wrongNumber", "Wrong number of arguments, " USAGE);
|
||||
if ((nlhs < min_outputs) || (nlhs > max_outputs))
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:out:wrongNumber", "Wrong number of output arguments, " USAGE);
|
||||
|
||||
unsigned int i = 0;
|
||||
int64_t nprocs = 1;
|
||||
{ // 0
|
||||
if (! mxIsScalar(prhs[i]))
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:wrongType:notScalar", "Argument %d must be scalar!", i+1);
|
||||
std::unique_ptr<mx_numeric> nt(get_numeric_trait(prhs[i]));
|
||||
if (! nt.get())
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:wrongType:notScalar", "Argument %d must be numeric!", i+1);
|
||||
nprocs = nt->get_int64(prhs[i]);
|
||||
if (nprocs < 1)
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:notPositive", "Number of processes must be positive!");
|
||||
if (nprocs > max_nprocs)
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:tooMany", "Too many processes, max is %d!", max_nprocs);
|
||||
i += 1;
|
||||
}
|
||||
precision::type prec = precision::type::Single;
|
||||
{ // 1
|
||||
if (mxGetM(prhs[i]) != 1)
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:wrongDimensions:notOne", "Argument %d must be one-dimensional character array!", i+1);
|
||||
std::unique_ptr<mx_trait<char>> ct(get_char_trait(prhs[i]));
|
||||
if (! ct.get())
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:wrongType:notChar", "Argument %d must be of type char!", i+1);
|
||||
std::string precString = ct->get_string(prhs[i]);
|
||||
if (precString == "double") {
|
||||
prec = precision::type::Double;
|
||||
} else if (precString != "single") {
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:type", "Illegal type argument (%s): must be either 'single' or 'double'!", precString.c_str());
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
std::array<int64_t, 2> dims{0, 0};
|
||||
do { // 2 optionial
|
||||
std::unique_ptr<mx_numeric> nt(get_numeric_trait(prhs[i]));
|
||||
if (! nt.get())
|
||||
break;
|
||||
if ((mxGetN(prhs[i]) != 2) || (mxGetM(prhs[i]) != 1))
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:wrongDimensions:oneByTwo", "Argument %d must be 2x1 numeric array!", i+1);
|
||||
dims[0] = nt->get_int64(prhs[i], 0);
|
||||
dims[1] = nt->get_int64(prhs[i], 1);
|
||||
if ((dims[0] < 1) || (dims[1] < 1))
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:illegalSize", "Size argument must be at least 1 in every dimension!");
|
||||
i += 1;
|
||||
} while(false);
|
||||
std::string object_path;
|
||||
{ // 3
|
||||
if (mxGetM(prhs[i]) != 1)
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:wrongDimensions:notOne", "Argument %d must be one-dimensional character array!", i+1);
|
||||
std::unique_ptr<mx_trait<char>> ct(get_char_trait(prhs[i]));
|
||||
if (! ct.get())
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:wrongType:notChar", "Argument %d must be of type char!", i+1);
|
||||
object_path = ct->get_string(prhs[i]);
|
||||
if (! object_path.size())
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:empty", "Illegal object path argument!");
|
||||
i += 1;
|
||||
}
|
||||
if (i == nrhs)
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:in:wrongNumber", "Missing arguments, " USAGE);
|
||||
std::vector<std::string> file_paths;
|
||||
{ // 4
|
||||
if (mxGetM(prhs[i]) != 1)
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:wrongDimensions:notOne", "Argument %d must be a 1xN one-dimensional character or cell array!", i+1);
|
||||
std::unique_ptr<mx_trait<char>> ct(get_char_trait(prhs[i]));
|
||||
if (ct.get()) { // char array
|
||||
file_paths.push_back(ct->get_string(prhs[i]));
|
||||
} else { // cell array
|
||||
if (! mxIsCell(prhs[i]))
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:type", "Argument %d must be either character or cell array!", i+1);
|
||||
for (std::size_t j=0; j<mxGetN(prhs[i]); j+=1) {
|
||||
mxArray *pa = mxGetCell(prhs[i], j);
|
||||
if (mxGetM(pa) != 1)
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:wrongDimensions:notOne", "Cell fields in argument %d must be one-dimensional character arrays!", i+1);
|
||||
ct.reset(get_char_trait(pa));
|
||||
if (! ct.get())
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:wrongType:notChar", "Cell field in argument %d must be of type char!", i+1);
|
||||
file_paths.push_back(ct->get_string(pa));
|
||||
if (! file_paths.back().size())
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:empty", "Illegal file path argument!");
|
||||
}
|
||||
if (! file_paths.size())
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:empty", "Empty file path argument!");
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
if (i != nrhs)
|
||||
mexErrMsgIdAndTxt("psi:ptycho:arg:in:wrongNumber", "Superfluous arguments, " USAGE);
|
||||
|
||||
bool verbose = env::is_enabled("PTYCHO_READ_VERBOSE");
|
||||
if (verbose) {
|
||||
mexPrintf("precision: %s\n", precision::to_str(prec));
|
||||
const char *debuglog = std::getenv("PTYCHO_READ_DEBUG");
|
||||
mexPrintf("debuglog: %s - ", (debuglog ? debuglog : "not set"));
|
||||
DEBUG {
|
||||
mexPrintf("ok\n");
|
||||
OUT << "----------- read object data ----------------\n";
|
||||
} else {
|
||||
mexPrintf("not functional\n");
|
||||
}
|
||||
mexPrintf("object_location: %s\n", object_path.c_str());
|
||||
if (file_paths.size() == 1) {
|
||||
mexPrintf("data_location: %s\n", file_paths[0].c_str());
|
||||
} else {
|
||||
mexPrintf("data_location first: %s\n", file_paths[0].c_str());
|
||||
mexPrintf("data_location last : %s\n", file_paths.back().c_str());
|
||||
}
|
||||
}
|
||||
|
||||
data_prep::read_stat r_stat;
|
||||
plhs[0] = data_prep::read_object_data((int)nprocs, prec, dims, object_path, file_paths, (nlhs == 2) ? &plhs[1] : nullptr, verbose ? &r_stat : nullptr);
|
||||
if (verbose)
|
||||
mexPrintf("Read %fMB in %fs, bandwidth is %fMB/s\n", (r_stat.nbytes / 1000000.), r_stat.seconds, (r_stat.nbytes / (r_stat.seconds * 1000000.)));
|
||||
return;
|
||||
}
|
||||
@@ -0,0 +1,625 @@
|
||||
/*!
|
||||
* \file
|
||||
* Support for data formats with many separate compnents that can be read using threads
|
||||
*
|
||||
* TODO: make the code safe to type inconsistencies
|
||||
*/
|
||||
|
||||
#include <map>
|
||||
#include <cerrno>
|
||||
#include <cstring>
|
||||
#include <algorithm>
|
||||
#include <fstream>
|
||||
#include <cassert>
|
||||
#include <cstdint>
|
||||
#include <regex>
|
||||
#include <mutex>
|
||||
#include <future>
|
||||
#include <atomic>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <stdio.h>
|
||||
#include <dirent.h>
|
||||
#include "tiffio.h"
|
||||
#include "precision.h"
|
||||
#include "mex.h"
|
||||
#include "mex_helper.h"
|
||||
#include "debug_helper.h"
|
||||
#include "read_data_threaded.h"
|
||||
|
||||
namespace {
|
||||
thread_local unsigned int thread_id = 0;
|
||||
std::mutex mutex_cout;
|
||||
std::atomic<bool> thread_error;
|
||||
|
||||
#define LOCK(m) std::lock_guard<std::mutex> _lock(m);
|
||||
|
||||
/*!
|
||||
* \brief Adapt file and destination dimensions
|
||||
*
|
||||
* \param fdims (IN) file image dimensions (2D)
|
||||
* \param mdims (IN) MATLAB destination array dimensions (> 2D) (last dimensions are image dimensions)
|
||||
* \param im_ctr (IN) image center
|
||||
* \param fstride (OUT) file image strides
|
||||
* \param mstride (OUT) destination image strides
|
||||
* \param count (OUT) elements to copy in each image dimension
|
||||
*/
|
||||
void adapt_dimensions(const std::vector<long> &fdims, const std::vector<mwSize> mdims,
|
||||
const std::vector<long> &im_ctr,
|
||||
std::vector<long> &fstride,
|
||||
std::vector<long> &mstride,
|
||||
std::vector<long> &count)
|
||||
{
|
||||
const mwSize *md = &(*(mdims.end() - 2));
|
||||
const std::vector<mwSize> mh{md[0]/2, md[1]/2};
|
||||
for (unsigned int i=0; i<2; i++) {
|
||||
if (im_ctr[i] >= mh[i]) {
|
||||
mstride[i] = 0;
|
||||
fstride[i] = im_ctr[i] - mh[i];
|
||||
if (fstride[i] + md[i] > fdims[i])
|
||||
count[i] = fdims[i] - fstride[i];
|
||||
else
|
||||
count[i] = md[i];
|
||||
} else {
|
||||
fstride[i] = 0;
|
||||
mstride[i] = mh[i] - im_ctr[i];
|
||||
if (mstride[i] + fdims[i] <= md[i])
|
||||
count[i] = fdims[i];
|
||||
else
|
||||
count[i] = md[i] - mstride[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Append element to file path
|
||||
*
|
||||
* Path separator is '/'
|
||||
*
|
||||
* \param path path appended to
|
||||
* \param elem element to append
|
||||
* \return new path with extra element
|
||||
*/
|
||||
std::string path_append (const std::string &path, const std::string &elem)
|
||||
{
|
||||
std::string result(path);
|
||||
if (result.size())
|
||||
result.push_back('/');
|
||||
return result + elem;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
|
||||
/* PILATUS CBF DATA
|
||||
_array_data.data
|
||||
;
|
||||
--CIF-BINARY-FORMAT-SECTION--
|
||||
Content-Type: application/octet-stream;
|
||||
conversions="x-CBF_BYTE_OFFSET"
|
||||
Content-Transfer-Encoding: BINARY
|
||||
X-Binary-Size: 2499331
|
||||
X-Binary-ID: 1
|
||||
X-Binary-Element-Type: "signed 32-bit integer"
|
||||
X-Binary-Element-Byte-Order: LITTLE_ENDIAN
|
||||
Content-MD5: XoY7+gfct1+OKiJlzKOiRw==
|
||||
X-Binary-Number-of-Elements: 2476525
|
||||
X-Binary-Size-Fastest-Dimension: 1475
|
||||
X-Binary-Size-Second-Dimension: 1679
|
||||
X-Binary-Size-Padding: 4095
|
||||
*/
|
||||
std::regex pilatus_ncols_regex(R"(X-Binary-Size-Fastest-Dimension: (\d+))"); //!< Regex for pilatus ncols
|
||||
std::regex pilatus_nrows_regex(R"(X-Binary-Size-Second-Dimension: (\d+))"); //!< Regex for pilatus nrows
|
||||
|
||||
/*!
|
||||
* \brief Read CBF image dimensions
|
||||
*
|
||||
* \param data (IN) CBF image file data
|
||||
* \param dims (OUT) to be filled with [nrows, ncols] from the data
|
||||
*/
|
||||
void cbf_dims (const std::vector<char> &data, std::vector<long> &dims)
|
||||
{
|
||||
assert(dims.size() == 2);
|
||||
{
|
||||
std::cmatch match;
|
||||
if (! std::regex_search(&data[0], &data[0] + data.size(), match, pilatus_ncols_regex))
|
||||
throw std::runtime_error("unable to find number of columns");
|
||||
dims[1] = std::stol(match[1].str());
|
||||
if (dims[1] <= 0)
|
||||
throw std::runtime_error("dimension along row is not positive");
|
||||
}
|
||||
{
|
||||
std::cmatch match;
|
||||
if (! std::regex_search(&data[0], &data[0] + data.size(), match, pilatus_nrows_regex))
|
||||
throw std::runtime_error("unable to find number of rows");
|
||||
dims[0] = std::stol(match[1].str());
|
||||
if (dims[0] <= 0)
|
||||
throw std::runtime_error("dimension along column is not positive");
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Read pilatus metadata
|
||||
*
|
||||
* \param paths (IN) paths to pilatus CBF image files
|
||||
* \param dims (OUT) to be filled with [n_images, n_series(only if several series are present), nrows, ncols], the last two from the first data file
|
||||
* \param im_sz (INOUT) if empty, fill it with [nrows, ncols] from the first data file
|
||||
*/
|
||||
void pilatus_read_meta(const std::vector<std::string> &paths,
|
||||
std::vector<long> &dims,
|
||||
std::vector<long> &im_sz)
|
||||
{
|
||||
// Take the first file to determine image dimensions size and center if not given
|
||||
if (paths.empty())
|
||||
throw std::invalid_argument("no CBF data files");
|
||||
DEBUG {
|
||||
OUT << paths.size() << " data files\n"
|
||||
<< "first file " << paths[0] << std::endl;
|
||||
}
|
||||
std::vector<long> fdim{0, 0};
|
||||
{
|
||||
std::vector<char> buf(4*1024); // 4K max header size in CBF file
|
||||
std::ifstream ifs(paths[0]);
|
||||
ifs.read(&buf[0], buf.size());
|
||||
if (ifs.fail() && !ifs.eof())
|
||||
throw std::runtime_error("unable to read first data file");
|
||||
buf.resize(ifs.gcount());
|
||||
cbf_dims(buf, fdim);
|
||||
}
|
||||
dims.resize(3);
|
||||
dims[0] = paths.size();
|
||||
dims[1] = fdim[0];
|
||||
dims[2] = fdim[1];
|
||||
if (im_sz.empty()) {
|
||||
im_sz.resize(2);
|
||||
im_sz[0] = fdim[0];
|
||||
im_sz[1] = fdim[1];
|
||||
DEBUG {
|
||||
OUT << "setting image size to " << im_sz[0] << 'x' << im_sz[1] << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Read image data from pilatus CBF file
|
||||
*
|
||||
* \param tid thread id starting from 0
|
||||
* \param destination MATLAB array image data destination
|
||||
* \param path pilatus CBF data file path
|
||||
* \param mdims MATLAB array dimensions
|
||||
* \param im_ctr image center relative to data
|
||||
* \tparam f_type float or double array element type
|
||||
*/
|
||||
template<typename f_type>
|
||||
void pilatus_read_data(unsigned int tid,
|
||||
f_type * destination,
|
||||
const std::string &path,
|
||||
const std::vector<mwSize> &mdims,
|
||||
const std::vector<long> &im_ctr)
|
||||
{
|
||||
// Use Heiners method to read the data
|
||||
// Adapt dimensions for every image file
|
||||
std::vector<char> fbuf;
|
||||
{
|
||||
FILE *fin = fopen(path.c_str(), "r");
|
||||
if (! fin)
|
||||
throw std::runtime_error(std::string("unable to open file ") + path + ": " + std::strerror(errno));
|
||||
try {
|
||||
off_t fsz;
|
||||
{
|
||||
struct stat sbuf;
|
||||
if (fstat(fileno(fin), &sbuf) == -1) {
|
||||
throw std::runtime_error(std::string("unable to stat file ") + path + ": " + std::strerror(errno));
|
||||
}
|
||||
fsz = sbuf.st_size;
|
||||
}
|
||||
fbuf.resize(fsz);
|
||||
fread(&fbuf[0], 1, fsz, fin);
|
||||
if (ferror(fin))
|
||||
throw std::runtime_error(std::string("unable to read file ") + path + ": " + std::strerror(errno));
|
||||
fclose(fin);
|
||||
} catch (...) {
|
||||
fclose(fin);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
std::vector<long> fdims{0, 0};
|
||||
cbf_dims(fbuf, fdims);
|
||||
unsigned long finger; // compressed data index
|
||||
{
|
||||
std::vector<char> sig{ '\x0c', '\x1a', '\x04', '\xd5' };
|
||||
auto p = std::search(fbuf.begin(), fbuf.end(), sig.begin(), sig.end());
|
||||
if (p == fbuf.end())
|
||||
throw std::runtime_error(std::string("data signature not found within file " + path));
|
||||
finger = p - fbuf.begin() + 4;
|
||||
}
|
||||
unsigned long nelems = fdims[0] * fdims[1];
|
||||
std::vector<f_type> data(nelems);
|
||||
int current = 0;
|
||||
for (unsigned int i=0; i<nelems; i++) {
|
||||
if (*((uint8_t *)&fbuf[finger]) != 0x80) { // | xx |
|
||||
current += *((int8_t *)&fbuf[finger]);
|
||||
finger += 1;
|
||||
} else if (*((uint16_t *)&fbuf[finger+1]) != 0x8000) { // | 80 | xx | xx |
|
||||
current += *((int16_t *)&fbuf[finger+1]);
|
||||
finger += 3;
|
||||
} else { // | 80 | 80 | 00 | xx | xx | xx | xx |
|
||||
current += *((int32_t *)&fbuf[finger+3]);
|
||||
finger += 7;
|
||||
}
|
||||
if (finger + 7 > fbuf.size())
|
||||
throw std::runtime_error(std::string("data inconsistency in file ") + path);
|
||||
//if (current < -1) // allow value -1, which is used to mark detector gaps
|
||||
// throw std::runtime_error(std::string("data error in file ") + path);
|
||||
data[i] = current;
|
||||
}
|
||||
fbuf.clear();
|
||||
std::vector<long> fstride{0, 0};
|
||||
std::vector<long> mstride{0, 0};
|
||||
std::vector<long> count{0, 0};
|
||||
adapt_dimensions(fdims, mdims, im_ctr, fstride, mstride, count);
|
||||
DEBUG {
|
||||
LOCK(mutex_cout);
|
||||
OUT << thread_id << ": fstride=[" << fstride[0] << ',' << fstride[1] << "], mstride=[" << mstride[0] << ',' << mstride[1] << "], count=[" << count[0] << ',' << count[1] <<']' << std::endl;
|
||||
}
|
||||
{
|
||||
auto sz = mdims.size() - 2;
|
||||
auto msize = mdims[sz] * mdims[sz+1];
|
||||
std::memset(destination, 0, msize * sizeof(f_type));
|
||||
}
|
||||
for (unsigned long row=0; row<count[0]; row++) {
|
||||
auto col = data.begin() + fdims[1] * (row + fstride[0]) + fstride[1];
|
||||
std::copy(col, col + count[1], destination + mdims.back() * (row + mstride[0]) + mstride[1]);
|
||||
}
|
||||
}
|
||||
|
||||
//-----------------------------------------------------
|
||||
|
||||
/*!
|
||||
* \brief Read moench tiff image dimensions
|
||||
*
|
||||
* \param tiff_handle (IN) TIFF file handle
|
||||
* \param fdims (OUT) to be filled with [nrows, ncols]
|
||||
*/
|
||||
void tiff_dims (TIFF *tiff_handle, std::vector<long> &fdims)
|
||||
{
|
||||
assert(fdims.size() == 2);
|
||||
uint32_t image_length, image_width;
|
||||
if (! TIFFGetField(tiff_handle, TIFFTAG_IMAGELENGTH, &image_length))
|
||||
throw std::runtime_error("unable to get image length");
|
||||
if (! image_length)
|
||||
throw std::runtime_error("dimension along column is not positive");
|
||||
if (! TIFFGetField(tiff_handle, TIFFTAG_IMAGEWIDTH, &image_width))
|
||||
throw std::runtime_error("unable to get image width");
|
||||
if (! image_width)
|
||||
throw std::runtime_error("dimension along row is not positive");
|
||||
fdims[0] = image_length;
|
||||
fdims[1] = image_width;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Read moench metadata
|
||||
*
|
||||
* \param paths (IN) list of moench TIFF image data file paths
|
||||
* \param dims (OUT) data dimensions [n_images, nrows, ncols], the last two from the first data file
|
||||
* \param im_sz (INOUT) if empty, set to [nrows, ncols] from first data file
|
||||
*/
|
||||
void moench_read_meta(const std::vector<std::string> &paths,
|
||||
std::vector<long> &dims,
|
||||
std::vector<long> &im_sz)
|
||||
{
|
||||
// Take the first file to determine image dimensions size and center if not given
|
||||
if (paths.empty())
|
||||
throw std::invalid_argument("no TIFF data files");
|
||||
DEBUG {
|
||||
OUT << paths.size() << " data files\n"
|
||||
<< "first file " << paths[0] << std::endl;
|
||||
}
|
||||
std::vector<long> fdim(2);
|
||||
{
|
||||
TIFF *tiff_handle = TIFFOpen(paths[0].c_str(), "r");
|
||||
if (! tiff_handle)
|
||||
throw std::runtime_error("unable to open first image data file");
|
||||
try {
|
||||
tiff_dims(tiff_handle, fdim);
|
||||
} catch (...) {
|
||||
TIFFClose(tiff_handle);
|
||||
throw;
|
||||
}
|
||||
TIFFClose(tiff_handle);
|
||||
}
|
||||
dims.resize(3);
|
||||
dims[0] = paths.size();
|
||||
dims[1] = fdim[0];
|
||||
dims[2] = fdim[1];
|
||||
if (im_sz.empty()) {
|
||||
im_sz.resize(2);
|
||||
im_sz[0] = fdim[0];
|
||||
im_sz[1] = fdim[1];
|
||||
DEBUG {
|
||||
OUT << "setting image size to " << im_sz[0] << 'x' << im_sz[1] << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Read TIFF image data
|
||||
*
|
||||
* \param tf TIFF file descriptor
|
||||
* \param buf char buffer with enough space for a data strip
|
||||
* \param image image data buffer
|
||||
* \param num_strip number of data strips
|
||||
* \tparam sample_type image data sample type
|
||||
* \tparam result_type image buffer data type
|
||||
*/
|
||||
template<typename sample_type, typename result_type>
|
||||
void tiff_read(TIFF *tf, std::vector<char> &buf, std::vector<result_type> &image, tstrip_t num_strips)
|
||||
{
|
||||
std::uint32_t idx = 0;
|
||||
for (tstrip_t strip=0; strip<num_strips; strip++) {
|
||||
tsize_t nbytes = TIFFReadEncodedStrip(tf, strip, buf.data(), buf.size());
|
||||
if (nbytes < 0)
|
||||
throw std::runtime_error("unable to read strip from tiff file");
|
||||
for (tsize_t i=0; i<nbytes; i+=sizeof(sample_type))
|
||||
image[idx++] = *((sample_type *)&buf[i]);
|
||||
}
|
||||
if (idx != image.size())
|
||||
throw std::runtime_error("tiff image data size mismatch");
|
||||
}
|
||||
|
||||
|
||||
/*!
|
||||
* \brief Read moench TIFF image data into MATLAB array
|
||||
*
|
||||
* \param tid thread id, starting from 0
|
||||
* \param destination MATLAB array destination data buffer
|
||||
* \param path tiff file path
|
||||
* \param mdims MTLAB array dimensions
|
||||
* \param im_ctr image center relative to data
|
||||
*/
|
||||
template<typename f_type>
|
||||
void moench_read_data (unsigned int tid,
|
||||
f_type * destination,
|
||||
const std::string &path,
|
||||
const std::vector<mwSize> &mdims,
|
||||
const std::vector<long> &im_ctr)
|
||||
{
|
||||
/*
|
||||
std::vector<long> fdims{0, 0};
|
||||
std::vector<float> fbuf(0);
|
||||
{
|
||||
TIFF *tiff_handle = TIFFOpen(path.c_str(), "r");
|
||||
if (! tiff_handle)
|
||||
throw std::runtime_error(std::string("unable to open file ") + path);
|
||||
try {
|
||||
tiff_dims(tiff_handle, fdims);
|
||||
long nelems = fdims[0] * fdims[1];
|
||||
fbuf.resize(nelems);
|
||||
tmsize_t res, sz = fdims[1] * sizeof(float);
|
||||
for (uint32_t strip=0; strip<fdims[0]; strip++) {
|
||||
float *pos = &fbuf[fdims[1] * strip];
|
||||
res = TIFFReadRawStrip(tiff_handle, strip, pos, sz);
|
||||
if (res != sz)
|
||||
throw std::runtime_error("unable to read tiff data");
|
||||
}
|
||||
} catch (...) {
|
||||
TIFFClose(tiff_handle);
|
||||
throw;
|
||||
}
|
||||
TIFFClose(tiff_handle);
|
||||
}
|
||||
*/
|
||||
std::vector<long> fdims{0, 0};
|
||||
std::vector<f_type> fbuf(0);
|
||||
{
|
||||
TIFF *tiff_handle = TIFFOpen(path.c_str(), "r");
|
||||
if (! tiff_handle)
|
||||
throw std::runtime_error(std::string("unable to open file ") + path);
|
||||
try {
|
||||
tiff_dims(tiff_handle, fdims);
|
||||
long nelems = fdims[0] * fdims[1];
|
||||
fbuf.resize(nelems);
|
||||
std::uint16_t bps;
|
||||
if (TIFFGetField(tiff_handle, TIFFTAG_BITSPERSAMPLE, &bps) != 1)
|
||||
throw std::runtime_error(std::string("unable to read number of bits per sample for file ") + path);
|
||||
std::uint16_t format;
|
||||
if (TIFFGetFieldDefaulted(tiff_handle, TIFFTAG_SAMPLEFORMAT, &format) != 1)
|
||||
throw std::runtime_error(std::string("unable to read sample format for file") + path);
|
||||
tstrip_t num_strips = TIFFNumberOfStrips(tiff_handle);
|
||||
tsize_t strip_sz = TIFFStripSize(tiff_handle);
|
||||
std::vector<char> cbuf(strip_sz);
|
||||
switch (format) {
|
||||
case 1:
|
||||
switch (bps) {
|
||||
case 8:
|
||||
tiff_read<std::uint8_t, f_type>(tiff_handle, cbuf, fbuf, num_strips); break;
|
||||
case 16:
|
||||
tiff_read<std::uint16_t, f_type>(tiff_handle, cbuf, fbuf, num_strips); break;
|
||||
case 32:
|
||||
tiff_read<std::uint32_t, f_type>(tiff_handle, cbuf, fbuf, num_strips); break;
|
||||
default:
|
||||
throw std::runtime_error(std::string("unsupported number of bits per unsigned integer sample in file ") + path);
|
||||
}
|
||||
break;
|
||||
case 2:
|
||||
switch (bps) {
|
||||
case 8:
|
||||
tiff_read<std::int8_t, f_type>(tiff_handle, cbuf, fbuf, num_strips); break;
|
||||
case 16:
|
||||
tiff_read<std::int16_t, f_type>(tiff_handle, cbuf, fbuf, num_strips); break;
|
||||
case 32:
|
||||
tiff_read<std::int32_t, f_type>(tiff_handle, cbuf, fbuf, num_strips); break;
|
||||
default:
|
||||
throw std::runtime_error(std::string("unsupported number of bits per integer sample in file ") + path);
|
||||
}
|
||||
break;
|
||||
case 3:
|
||||
switch (bps) {
|
||||
case 32:
|
||||
tiff_read<float, f_type>(tiff_handle, cbuf, fbuf, num_strips); break;
|
||||
case 64:
|
||||
tiff_read<double, f_type>(tiff_handle, cbuf, fbuf, num_strips); break;
|
||||
default:
|
||||
throw std::runtime_error(std::string("unsupported number of bits per ieee sample in file ") + path);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
throw std::runtime_error(std::string("unsupported sample format in file ") + path);
|
||||
}
|
||||
} catch (...) {
|
||||
TIFFClose(tiff_handle);
|
||||
throw;
|
||||
}
|
||||
TIFFClose(tiff_handle);
|
||||
}
|
||||
std::vector<long> fstride{0, 0};
|
||||
std::vector<long> mstride{0, 0};
|
||||
std::vector<long> count{0, 0};
|
||||
adapt_dimensions(fdims, mdims, im_ctr, fstride, mstride, count);
|
||||
DEBUG {
|
||||
LOCK(mutex_cout);
|
||||
OUT << thread_id << ": fstride=[" << fstride[0] << ',' << fstride[1] << "], mstride=[" << mstride[0] << ',' << mstride[1] << "], count=[" << count[0] << ',' << count[1] <<']' << std::endl;
|
||||
}
|
||||
{
|
||||
unsigned long msize = *(mdims.end() - 2) * mdims.back();
|
||||
std::memset(destination, 0, msize * sizeof(f_type));
|
||||
}
|
||||
for (unsigned long row=0; row<count[0]; row++) {
|
||||
auto col = fbuf.begin() + fdims[1] * (row + fstride[0]) + fstride[1];
|
||||
std::copy(col, col + count[1], destination + mdims.back() * (row + mstride[0]) + mstride[1]);
|
||||
}
|
||||
}
|
||||
|
||||
//-----------------------------------------------------
|
||||
|
||||
/*!
|
||||
* \brief Read data files in parallel
|
||||
*
|
||||
* \param format data format name
|
||||
* \param nthreads desired number of threads
|
||||
* \param paths paths to detector data files
|
||||
* \param image_size desired image size [nrows, ncols]
|
||||
* \param roi_center desired image center [row, col] relative to data
|
||||
* \return MATLAB array [n_images, n_series(only if several are present), nrows, ncols]
|
||||
*/
|
||||
template<typename f_type>
|
||||
mxArray* read_data_parallel(const std::string &format,
|
||||
long nthreads,
|
||||
const std::vector<std::string> &paths,
|
||||
std::vector<long> &image_size, std::vector<long> &roi_center)
|
||||
{
|
||||
struct read_func final {
|
||||
void (*read_meta) (const std::vector<std::string> &paths,
|
||||
std::vector<long> &dims,
|
||||
std::vector<long> &im_sz);
|
||||
void (*read_data) (unsigned int tid,
|
||||
f_type * destination,
|
||||
const std::string &path,
|
||||
const std::vector<mwSize> &mdims,
|
||||
const std::vector<long> &im_ctr);
|
||||
} detector_functions[2] = {
|
||||
{ pilatus_read_meta, pilatus_read_data<f_type> }, // pilatus
|
||||
{ moench_read_meta, moench_read_data<f_type> } // moench
|
||||
};
|
||||
|
||||
const std::map<std::string, read_func&> dfunc {
|
||||
{ "pilatus", detector_functions[0] },
|
||||
{ "moench", detector_functions[1] }
|
||||
};
|
||||
|
||||
const auto elem = dfunc.find(format);
|
||||
if (elem == dfunc.end())
|
||||
throw std::invalid_argument("unsupported data format");
|
||||
const auto &func = elem->second;
|
||||
std::vector<long> fdims;
|
||||
func.read_meta(paths, fdims, image_size);
|
||||
if (fdims.size() < 3)
|
||||
throw std::invalid_argument("inconsistent data (dimensionality too low)");
|
||||
if (paths.size() != fdims[0])
|
||||
throw std::invalid_argument("inconsistent data (number of paths / array dimension mismatch)");
|
||||
if (image_size.size() != 2)
|
||||
throw std::invalid_argument("bad image size, must be two dimensional");
|
||||
if (roi_center.empty()) {
|
||||
roi_center.resize(2);
|
||||
roi_center[0] = fdims[fdims.size() - 2] / 2;
|
||||
roi_center[1] = fdims.back() / 2;
|
||||
}
|
||||
std::vector<mwSize> mdims(fdims.size());
|
||||
std::transform(fdims.begin(), fdims.end(), mdims.begin(), [](long e)->mwSize { return (mwSize)e; });
|
||||
auto sz = mdims.size() - 2;
|
||||
mdims[sz] = image_size[0];
|
||||
mdims[sz + 1] = image_size[1];
|
||||
DEBUG {
|
||||
OUT << "fdims " << fdims[0] << 'x' << fdims[1] << 'x' << fdims[2] << '\n'
|
||||
<< "mdims " << mdims[0] << 'x' << mdims[1] << 'x' << mdims[2] << '\n'
|
||||
<< "center " << roi_center[0] << 'x' << roi_center[1] << std::endl;
|
||||
}
|
||||
mx_ptr<mxArray> matrix;
|
||||
{
|
||||
std::vector<mwSize> rdims(mdims);
|
||||
std::reverse(rdims.begin(), rdims.end());
|
||||
matrix.reset(mxCreateNumericArray(rdims.size(), &rdims[0], mx_trait<f_type>::class_id, mxREAL));
|
||||
}
|
||||
if (! matrix.get())
|
||||
throw std::runtime_error("matrix creation failed");
|
||||
f_type *destination = mx_trait<f_type>::get(matrix.get());
|
||||
|
||||
if (nthreads > paths.size())
|
||||
nthreads = paths.size();
|
||||
std::vector<std::future<void>> threads;
|
||||
for (unsigned int tid=0; tid<nthreads; tid++) {
|
||||
threads.push_back(std::async(std::launch::async, [nthreads, tid, &destination, &func, &paths, &fdims, &mdims, &roi_center]() {
|
||||
thread_id = tid;
|
||||
const unsigned int npaths = (paths.size() + nthreads - 1) / nthreads;
|
||||
const unsigned int first_full = (nthreads * npaths) - paths.size();
|
||||
const unsigned int first = (tid * npaths) - (tid < first_full ? tid : first_full);
|
||||
const unsigned int last = first + npaths - (tid < first_full ? 1 : 0);
|
||||
const auto pdim = &mdims[mdims.size() - 2];
|
||||
const auto sz = pdim[0] * pdim[1];
|
||||
DEBUG {
|
||||
LOCK(mutex_cout);
|
||||
OUT << thread_id << ": range " << first << '-' << last << ", sz=" << sz << std::endl;
|
||||
}
|
||||
for (unsigned int i=first; i<last; i++) {
|
||||
if (thread_error.load())
|
||||
throw std::runtime_error("received stop signal");
|
||||
DEBUG {
|
||||
LOCK(mutex_cout);
|
||||
OUT << thread_id << ": " << paths[i] << std::endl;
|
||||
}
|
||||
func.read_data(tid, &destination[i * sz], paths[i], mdims, roi_center);
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
bool success{true};
|
||||
for (unsigned int tid=0; tid<nthreads; tid++) {
|
||||
try {
|
||||
threads[tid].wait();
|
||||
threads[tid].get();
|
||||
} catch (const std::exception &ex) {
|
||||
mexPrintf("Thread %u error: %s\n", tid, ex.what());
|
||||
thread_error.store(false);
|
||||
}
|
||||
}
|
||||
|
||||
if (thread_error)
|
||||
throw std::runtime_error("Failed to read data!");
|
||||
|
||||
return matrix.release();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
namespace data_prep {
|
||||
mxArray* read_data_threaded(const std::string &format,
|
||||
long nthreads,
|
||||
const std::vector<std::string> &data_path,
|
||||
std::vector<long> &image_size, std::vector<long> &roi_center,
|
||||
precision::type prec)
|
||||
{
|
||||
if (prec == precision::type::Double)
|
||||
return read_data_parallel<double>(format, nthreads, data_path, image_size, roi_center);
|
||||
else
|
||||
return read_data_parallel<float>(format, nthreads, data_path, image_size, roi_center);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
#ifndef READ_DATA_THREADED_H
|
||||
#define READ_DATA_THREADED_H
|
||||
|
||||
/*!
|
||||
* \file
|
||||
* Header file for the read Moench and Pilatus data functionality
|
||||
*/
|
||||
|
||||
/*!
|
||||
* \brief Data preparator functions
|
||||
*/
|
||||
namespace data_prep {
|
||||
/*!
|
||||
* \brief read Moench or Pilatus data
|
||||
*
|
||||
* \param format data format string
|
||||
* \param nthreads number of parallel read threads
|
||||
* \param data_path paths to data files
|
||||
* \param image_size size of returned image data
|
||||
* \param roi_center region of interest center relative to measured data
|
||||
* \param prec precision of returned data
|
||||
* \return MATLAB array with resized and recentered images
|
||||
*/
|
||||
mxArray* read_data_threaded(const std::string &format,
|
||||
long nthreads,
|
||||
const std::vector<std::string> &data_path,
|
||||
std::vector<long> &image_size, std::vector<long> &roi_center,
|
||||
precision::type prec);
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,700 @@
|
||||
/*!
|
||||
* \file
|
||||
* Read Object Data
|
||||
*
|
||||
* This file contains the main mex code for reading object data into MATLAB memory.
|
||||
*/
|
||||
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <chrono>
|
||||
#include <cassert>
|
||||
#include <array>
|
||||
#include <vector>
|
||||
#include <cstring>
|
||||
#include <cerrno>
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
#include <exception>
|
||||
#include <stdexcept>
|
||||
#include <cstdio>
|
||||
#include <functional>
|
||||
#include <algorithm>
|
||||
#include <unistd.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/wait.h>
|
||||
#include "hdf5_helper.h"
|
||||
#include "mex.h"
|
||||
#include "debug_helper.h"
|
||||
#include "precision.h"
|
||||
#include "multi_processing.h"
|
||||
|
||||
namespace {
|
||||
|
||||
#include "mex_helper.h"
|
||||
|
||||
/*!
|
||||
* \defgroup dataLayoutKind Kind of data layout
|
||||
* @{
|
||||
*/
|
||||
constexpr int kind_11 = 0; //!< 1 file - 1 dataset (new eiger)
|
||||
constexpr int kind_N1 = 1; //!< N files - 1 dataset (old eiger)
|
||||
constexpr int kind_1N = 2; //!< 1 file - N datasets (ESRF)
|
||||
constexpr int kind_NN = 3; //!< N (file, dataset) pairs
|
||||
/*! @} */
|
||||
|
||||
/*!
|
||||
* \brief Vector to string
|
||||
* Outputs a string containing v0[,v1[,v2...]]
|
||||
* \param v vector
|
||||
* \param separator separator between vector elements, defaults to comma
|
||||
* \tparam T vector element type
|
||||
*/
|
||||
template <typename T>
|
||||
std::string vec_to_str(const std::vector<T> &v, char separator=',')
|
||||
{
|
||||
std::ostringstream oss;
|
||||
oss << v[0];
|
||||
for (std::size_t i=1; i<v.size(); i++)
|
||||
oss << separator << v[i];
|
||||
return oss.str();
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Adapt data space layout
|
||||
*
|
||||
* Adapt the memory and file dataset layouts by selecting apropriate hyperslabs
|
||||
*
|
||||
* \param mspace memory data space layout (must fit to parameters first, last)
|
||||
* \param fspace file dataset layout
|
||||
* \param mdim memory space dimensions (must fit to parameters first last if kind=kind_11)
|
||||
* \param fdim file dataset dimensions
|
||||
* \param first first chunk of file space to include
|
||||
* \param last one past last chunk of file space to include
|
||||
* \param roi_center image center
|
||||
* \tparam kind kind of data layout
|
||||
*/
|
||||
template<int kind>
|
||||
void adapt_spaces(hdf5::dataspace &mspace, hdf5::dataspace &fspace,
|
||||
const std::vector<hsize_t> &mdim, const std::vector<hsize_t> &fdim,
|
||||
int first , int last,
|
||||
const std::vector<long> &roi_center)
|
||||
{
|
||||
if (mdim.size() <= roi_center.size())
|
||||
throw std::invalid_argument("result matrix must have more dimensions than the image");
|
||||
|
||||
std::vector<hsize_t> fstart(fdim.size());
|
||||
std::vector<hsize_t> mstart(fdim.size());
|
||||
std::vector<hsize_t> fcount(fdim);
|
||||
std::vector<hsize_t> mcount(mdim);
|
||||
bool mselect = false;
|
||||
bool fselect = false;
|
||||
|
||||
if (kind == kind_11) {
|
||||
// adapt first dimension (chunk dimension)
|
||||
if ((first != 0) || (last != fdim[0])) {
|
||||
fstart[0] = first;
|
||||
fcount[0] = last - first;
|
||||
fselect = true;
|
||||
// DEBUG {
|
||||
// OUT << " adapted chunk dimension: " << fstart[0] << '+' << fcount[0] << std::endl;
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
// adapt last dimensions (normally detector image dimensions)
|
||||
{
|
||||
auto fi = fdim.size();
|
||||
auto mi = mdim.size();
|
||||
auto i = roi_center.size();
|
||||
do {
|
||||
i--; fi--; mi--;
|
||||
hsize_t fh = (fdim[fi] + 1) / 2;
|
||||
hsize_t mh = (mdim[mi] + 1) / 2;
|
||||
hsize_t rc = roi_center[i];
|
||||
if (rc<0 || rc>=fdim[fi])
|
||||
throw std::invalid_argument("roi_center out of image bounds");
|
||||
if (mh <= rc) {
|
||||
fstart[fi] = rc - mh;
|
||||
if (fstart[fi] + mdim[mi] >= fdim[fi]) {
|
||||
fcount[fi] = fdim[fi] - fstart[fi];
|
||||
mcount[mi] = fcount[fi];
|
||||
mselect = mselect || (mcount[mi] != mdim[mi]);
|
||||
} else {
|
||||
fcount[fi] = mdim[mi];
|
||||
}
|
||||
fselect = fselect || ((fcount[fi] != fdim[fi]) || fstart[fi]);
|
||||
} else {
|
||||
if (rc + mh <= fdim[fi]) {
|
||||
mcount[mi] = fcount[fi] = rc + mh;
|
||||
mstart[mi] = mdim[mi] - mcount[mi];
|
||||
fselect = fselect || (fcount[fi] != fdim[fi]);
|
||||
} else {
|
||||
mcount[mi] = fcount[fi];
|
||||
mstart[mi] = rc - mh;
|
||||
}
|
||||
mselect = true;
|
||||
}
|
||||
// DEBUG {
|
||||
// OUT << " adapting image dimension " << i << ": fdim=" << fdim[fi] << '/' << fh << ", mdim=" << mdim[mi] << '/' << mh << ", rc=" << rc << "\n fsel=" << fselect << ", msel=" << mselect << std::endl;
|
||||
// }
|
||||
} while(i);
|
||||
}
|
||||
if (mselect) {
|
||||
DEBUG {
|
||||
OUT << "memory subspace start=" << vec_to_str(mstart) << " count=" << vec_to_str(mcount) << "\nfile space is " << vec_to_str(fdim) << std::endl;
|
||||
}
|
||||
if (H5Sselect_hyperslab(mspace, H5S_SELECT_SET, &mstart[0], nullptr, &mcount[0], nullptr) < 0)
|
||||
throw hdf5::exception("unable to set hdf5 memory space");
|
||||
}
|
||||
if (fselect) {
|
||||
DEBUG {
|
||||
OUT << "file subspace start=" << vec_to_str(fstart) << " count=" << vec_to_str(fcount) << "\nmem space is " << vec_to_str(mdim) << std::endl;
|
||||
}
|
||||
if (H5Sselect_hyperslab(fspace, H5S_SELECT_SET, &fstart[0], nullptr, &fcount[0], nullptr) < 0)
|
||||
throw hdf5::exception("unable to set hdf5 dataset space");
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Read object from file into buffer
|
||||
* \param file HDF5 file
|
||||
* \param dataset_path path to dataset
|
||||
* \param first first buffer chunk to read
|
||||
* \param last last buffer chunk to read
|
||||
* \param mdims buffer dimensions
|
||||
* \param ddims dataset dimensions
|
||||
* \param buf pointer to parent acessible buffer
|
||||
* \param roi_center image center
|
||||
* \tparam f_type buffer element type
|
||||
* \tparam kind kind of data layout
|
||||
*/
|
||||
template <typename f_type, int kind>
|
||||
void read_data_chunk(hdf5::file &file, const std::string &dataset_path,
|
||||
int first, int last,
|
||||
const std::vector<mwSize> &mdims,
|
||||
const std::vector<hsize_t> &ddims,
|
||||
f_type *buf,
|
||||
const std::vector<long> &roi_center)
|
||||
{
|
||||
// Open dataset and data space
|
||||
hdf5::dataset dataset(H5Dopen(file, dataset_path.c_str(), H5P_DEFAULT));
|
||||
if (! dataset.valid())
|
||||
throw hdf5::exception("unable to open dataset");
|
||||
hdf5::dataspace dataspace(H5Dget_space(dataset));
|
||||
if (! dataspace.valid())
|
||||
throw hdf5::exception("unable to open data space for dataset");
|
||||
|
||||
// Read and check data space
|
||||
std::vector<hsize_t> sdims(ddims.size()); // source dimensions
|
||||
{
|
||||
int ndims = H5Sget_simple_extent_ndims(dataspace);
|
||||
if (ndims != ddims.size())
|
||||
throw hdf5::exception("data space dimension mismatch");
|
||||
if (H5Sget_simple_extent_dims(dataspace, &sdims[0], nullptr) != ndims)
|
||||
throw hdf5::exception("unable to get dimension sizes");
|
||||
for (unsigned int i=0; i<ddims.size()-2; i++)
|
||||
if (sdims[i] != ddims[i])
|
||||
throw std::runtime_error("data space dimension inconsistency");
|
||||
}
|
||||
|
||||
std::vector<hsize_t> tdims(sdims); // target dimensions
|
||||
if (kind == kind_11) {
|
||||
auto i = sdims.size() - mdims.size();
|
||||
decltype(i) j = 0;
|
||||
for (; i<sdims.size(); i++, j++)
|
||||
tdims[i] = mdims[j];
|
||||
tdims[0] = last - first;
|
||||
} else {
|
||||
auto i = sdims.size();
|
||||
auto j = mdims.size();
|
||||
while ((i > 0) && (j > 1)) { // don't copy first matrix dimension (number of files/datasets)
|
||||
i--; j--;
|
||||
tdims[i] = mdims[j];
|
||||
}
|
||||
while (i) { // check that remaining data file dimensions are 1
|
||||
i--;
|
||||
if (tdims[i] != 1)
|
||||
throw std::runtime_error("data space dimension error");
|
||||
}
|
||||
}
|
||||
DEBUG {
|
||||
OUT << "init mem space: " << vec_to_str(tdims, 'x') << " at address " << buf << std::endl;
|
||||
}
|
||||
hdf5::dataspace memspace(H5Screate_simple(tdims.size(), &tdims[0], nullptr));
|
||||
if (! memspace.valid())
|
||||
throw hdf5::exception("unable to create memory data space");
|
||||
adapt_spaces<kind>(memspace, dataspace, tdims, sdims, first, last, roi_center);
|
||||
// Read dataset
|
||||
if (H5Dread(dataset, hdf5::type_trait<f_type>::type, memspace, dataspace, H5P_DEFAULT, buf) < 0) {
|
||||
DEBUG {
|
||||
std::FILE *out = std::fopen(std::getenv("PTYCHO_READ_DEBUG"), "a+");
|
||||
if (! out) {
|
||||
OUT << "unable to open file stream: " << std::strerror(errno) << std::endl;
|
||||
} else {
|
||||
H5Eprint(H5E_DEFAULT, out);
|
||||
std::fclose(out);
|
||||
}
|
||||
}
|
||||
throw hdf5::exception("unable to read dataset");
|
||||
}
|
||||
|
||||
DEBUG {
|
||||
OUT << "read finished - [0]=" << buf[0] << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Open HDF5 file
|
||||
* \param file_path path to HDF5 file
|
||||
* \return HDF5 file object id
|
||||
*/
|
||||
hid_t open_file(const std::string &file_path)
|
||||
{
|
||||
hdf5::file file(H5Fopen(file_path.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT));
|
||||
if (! file.valid())
|
||||
throw hdf5::exception(std::string("unable to open file ") + file_path);
|
||||
return file.grab();
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Drop leading dimensions of size 1
|
||||
* \param dims vector for which leading dimensions of size 1 beyond the second dimension will be dropped
|
||||
* \tparam T type of dimension size
|
||||
*/
|
||||
template <typename T>
|
||||
void collapse_dims (std::vector<T> &dims) noexcept
|
||||
{
|
||||
int i=0;
|
||||
for (; i<dims.size()-2; i++) {
|
||||
if (dims[i] != 1)
|
||||
break;
|
||||
}
|
||||
int j=0;
|
||||
for (; i<dims.size(); i++, j++)
|
||||
dims[j] = dims[i];
|
||||
return dims.resize(j);
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Get HDF5 data size
|
||||
*
|
||||
* \param file HDF5 file
|
||||
* \param dataset_path dataset path
|
||||
* \param dims filled with dataset dimensions
|
||||
*/
|
||||
void get_data_size(hdf5::file &file, const std::string &dataset_path, std::vector<hsize_t> &dims)
|
||||
{
|
||||
hdf5::dataset dataset(H5Dopen(file, dataset_path.c_str(), H5P_DEFAULT));
|
||||
if (! dataset.valid())
|
||||
throw hdf5::exception("unable to open dataset");
|
||||
hdf5::dataspace dataspace(H5Dget_space(dataset));
|
||||
if (! dataspace.valid())
|
||||
throw hdf5::exception("unable to get data space");
|
||||
int ndims = H5Sget_simple_extent_ndims(dataspace);
|
||||
if (ndims <= 0)
|
||||
throw hdf5::exception("wrong number of dimensions in dataset");
|
||||
dims.resize(ndims);
|
||||
if (H5Sget_simple_extent_dims(dataspace, &dims[0], nullptr) != ndims)
|
||||
throw hdf5::exception("unable to get dimension sizes");
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Traits for kind of data
|
||||
* \tparam kind kind of data layout
|
||||
*/
|
||||
template<int kind>
|
||||
struct ktrait final {};
|
||||
|
||||
/*!
|
||||
* \brief Traits for new eiger data
|
||||
*/
|
||||
template<>
|
||||
struct ktrait<kind_11> final {
|
||||
/*!
|
||||
* \brief Assign matrix dimensions from data dimensions
|
||||
* \param mdims (OUT) matrix dimensions to be assigned
|
||||
* \param ddims (IN) data dimensions
|
||||
* \param data_path (IN) path of data file
|
||||
* \param data_location (IN) location of dataset
|
||||
*/
|
||||
static void assign_matrix_dims(std::vector<mwSize> &mdims, const std::vector<hsize_t> &ddims, const std::vector<std::string> &data_path, const std::vector<std::string> &data_location)
|
||||
{
|
||||
auto ndims = ddims.size();
|
||||
mdims.resize(ndims);
|
||||
for (decltype(ndims) i=0; i<ndims; i++) {
|
||||
mdims[i] = static_cast<mwSize>(ddims[i]);
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Path range start offset
|
||||
* \param first start offset in work range
|
||||
* \param last end offset in work range (exclusive)
|
||||
* \return start path start range offset
|
||||
*/
|
||||
static int start_path(int first, int last)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Path range end offset
|
||||
* \param first start offset in work range
|
||||
* \param last end offset in work range (exclusive)
|
||||
* \return path range end offset (exclusive)
|
||||
*/
|
||||
static int end_path(int first, int last)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Location range start offset
|
||||
* \param f file index
|
||||
* \param first start offset in work range
|
||||
* \param last end offset in work range (exclusive)
|
||||
* \return start location start range offset
|
||||
*/
|
||||
static int start_location(int f, int first, int last)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Location range end offset
|
||||
* \param f file index
|
||||
* \param first start offset in work range
|
||||
* \param last end offset in work range (exclusive)
|
||||
* \return start location end range offset
|
||||
*/
|
||||
static int end_location(int f, int first, int last)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Buffer chunk offset
|
||||
* \param f file index
|
||||
* \param d dataset index
|
||||
* \param first start offset in work range
|
||||
* \return offset into chunk buffer
|
||||
*/
|
||||
static std::size_t buf_offset(int f, int d, int first)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief Traits for old eiger data
|
||||
*/
|
||||
template<>
|
||||
struct ktrait<kind_N1> final {
|
||||
static void assign_matrix_dims(std::vector<mwSize> &mdims, const std::vector<hsize_t> &ddims, const std::vector<std::string> &data_path, const std::vector<std::string> &data_location)
|
||||
{
|
||||
auto ndims = ddims.size();
|
||||
mdims.resize(ndims+1);
|
||||
mdims[0] = data_path.size();
|
||||
for (decltype(ndims) i=0; i<ndims; i++) {
|
||||
mdims[i+1] = static_cast<mwSize>(ddims[i]);
|
||||
}
|
||||
}
|
||||
|
||||
static int start_path(int first, int last)
|
||||
{
|
||||
return first;
|
||||
}
|
||||
|
||||
static int end_path(int first, int last)
|
||||
{
|
||||
return last;
|
||||
}
|
||||
|
||||
static int start_location(int f, int first, int last)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int end_location(int f, int first, int last)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
static std::size_t buf_offset(int f, int d, int first)
|
||||
{
|
||||
return f - first;
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief Traits for ESRF data
|
||||
*/
|
||||
template<>
|
||||
struct ktrait<kind_1N> final {
|
||||
static void assign_matrix_dims(std::vector<mwSize> &mdims, const std::vector<hsize_t> &ddims, const std::vector<std::string> &data_path, const std::vector<std::string> &data_location)
|
||||
{
|
||||
auto ndims = ddims.size();
|
||||
mdims.resize(ndims+1);
|
||||
mdims[0] = data_location.size();
|
||||
for (decltype(ndims) i=0; i<ndims; i++) {
|
||||
mdims[i+1] = static_cast<mwSize>(ddims[i]);
|
||||
}
|
||||
}
|
||||
|
||||
static int start_path(int first, int last)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int end_path(int first, int last)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int start_location(int f, int first, int last)
|
||||
{
|
||||
return first;
|
||||
}
|
||||
|
||||
static int end_location(int f, int first, int last)
|
||||
{
|
||||
return last;
|
||||
}
|
||||
|
||||
static std::size_t buf_offset(int f, int d, int first)
|
||||
{
|
||||
return d - first;
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief Traits for N (file, dataset) pairs
|
||||
*/
|
||||
template<>
|
||||
struct ktrait<kind_NN> final {
|
||||
static void assign_matrix_dims(std::vector<mwSize> &mdims, const std::vector<hsize_t> &ddims, const std::vector<std::string> &data_path, const std::vector<std::string> &data_location)
|
||||
{
|
||||
auto ndims = ddims.size();
|
||||
mdims.resize(ndims+1);
|
||||
mdims[0] = data_path.size();
|
||||
for (decltype(ndims) i=0; i<ndims; i++) {
|
||||
mdims[i+1] = static_cast<mwSize>(ddims[i]);
|
||||
}
|
||||
}
|
||||
|
||||
static int start_path(int first, int last)
|
||||
{
|
||||
return first;
|
||||
}
|
||||
|
||||
static int end_path(int first, int last)
|
||||
{
|
||||
return last;
|
||||
}
|
||||
|
||||
static int start_location(int f, int first, int last)
|
||||
{
|
||||
return f;
|
||||
}
|
||||
|
||||
static int end_location(int f, int first, int last)
|
||||
{
|
||||
return f+1;
|
||||
}
|
||||
|
||||
static std::size_t buf_offset(int f, int d, int first)
|
||||
{
|
||||
return f - first;
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief Read in objects with multiple processes
|
||||
*
|
||||
* \param nprocs number of read processes
|
||||
* \param data_path paths to hdf5 files
|
||||
* \param data_location paths to datasets within hdf5 files
|
||||
* \param image_size two dimensional image size
|
||||
* \param roi_center center of two dimensional image
|
||||
* \tparam f_type float or double result type
|
||||
* \tparam kind kind of data layout (see above)
|
||||
* \return MATLAB result array
|
||||
*/
|
||||
template <typename f_type, int kind>
|
||||
mxArray* read_data_parallel(int nprocs,
|
||||
const std::vector<std::string> &data_path,
|
||||
const std::vector<std::string> &data_location,
|
||||
std::vector<long> &image_size, std::vector<long> &roi_center)
|
||||
{
|
||||
using kt = ktrait<kind>;
|
||||
try {
|
||||
std::vector<mwSize> mdims; // matrix dimensions
|
||||
std::vector<hsize_t> ddims; // dataset dimensions
|
||||
|
||||
// Assign matrix dimension values
|
||||
{
|
||||
hdf5::file file(open_file(data_path[0]));
|
||||
if (! file.valid())
|
||||
throw hdf5::exception("unable to open eiger data file");
|
||||
get_data_size(file, data_location[0], ddims);
|
||||
auto ndims = ddims.size();
|
||||
if (ndims < 2)
|
||||
throw std::invalid_argument("dataset must have at least two dimensions");
|
||||
DEBUG {
|
||||
OUT << "data size: " << vec_to_str(ddims, 'x') << std::endl;
|
||||
}
|
||||
|
||||
auto sz = ndims - 2;
|
||||
if (image_size.empty()) {
|
||||
image_size.resize(2);
|
||||
image_size[0] = ddims[sz];
|
||||
image_size[1] = ddims[sz+1];
|
||||
DEBUG {
|
||||
OUT << "setting image size to " << image_size[0] << 'x' << image_size[1] << std::endl;
|
||||
}
|
||||
} else if (image_size.size() != 2)
|
||||
throw std::invalid_argument("image_size must be two dimensional");
|
||||
|
||||
if (roi_center.empty()) {
|
||||
roi_center.resize(2);
|
||||
roi_center[0] = ddims[sz] / 2;
|
||||
roi_center[1] = ddims[sz+1] / 2;
|
||||
DEBUG {
|
||||
OUT << "setting roi center to " << roi_center[0] << 'x' << roi_center[1] << std::endl;
|
||||
}
|
||||
} else if (roi_center.size() != 2)
|
||||
throw std::invalid_argument("roi_center must be two dimensional");
|
||||
}
|
||||
|
||||
kt::assign_matrix_dims(mdims, ddims, data_path, data_location);
|
||||
{
|
||||
auto ndims = mdims.size() - 2;
|
||||
mdims[ndims] = image_size[0];
|
||||
mdims[ndims+1] = image_size[1];
|
||||
}
|
||||
collapse_dims(mdims);
|
||||
{
|
||||
mwSize sz = 1;
|
||||
for (const auto &d : mdims)
|
||||
sz *= d;
|
||||
if (sz <= 0)
|
||||
throw std::invalid_argument("empty result matrix");
|
||||
}
|
||||
|
||||
// Create space for MATLAB array
|
||||
DEBUG {
|
||||
OUT << "creating MATLAB array: " << vec_to_str(mdims, 'x') << std::endl;
|
||||
}
|
||||
mx_ptr<mxArray> matrix;
|
||||
{
|
||||
std::vector<mwSize> rdims(mdims);
|
||||
std::reverse(rdims.begin(), rdims.end());
|
||||
matrix.reset(mxCreateNumericArray(rdims.size(), &rdims[0], mx_trait<f_type>::class_id, mxREAL));
|
||||
}
|
||||
if (! matrix.get())
|
||||
throw std::runtime_error("matrix creation failed");
|
||||
|
||||
// Prepare (shared) memory buffer
|
||||
if (mdims[0] < nprocs)
|
||||
nprocs = mdims[0];
|
||||
mp::buf<f_type> buf(mx_trait<f_type>::get(matrix.get()), mdims, nprocs);
|
||||
|
||||
// auto r_time = std::chrono::high_resolution_clock::now();
|
||||
// Read objects data into buffer
|
||||
mp::run<f_type>(nprocs, buf, [&data_path, &data_location, &mdims, &ddims, &roi_center](int proc, mp::buf<f_type> &buf) {
|
||||
int first = buf.offset(proc);
|
||||
int last = buf.offset(proc+1);
|
||||
DEBUG {
|
||||
OUT << "Process " << proc << ": " << first << ".." << last << std::endl;
|
||||
}
|
||||
for (int f=kt::start_path(first, last); f<kt::end_path(first, last); f++) {
|
||||
for (int d=kt::start_location(f, first, last); d<kt::end_location(f, first, last); d++) {
|
||||
hdf5::file file(open_file(data_path[f]));
|
||||
f_type *data_buf = buf.get(proc) + kt::buf_offset(f, d, first) * buf.chunk_size;
|
||||
read_data_chunk<f_type, kind>(file, data_location[d], first, last, mdims, ddims, data_buf, roi_center);
|
||||
} // dataset locations
|
||||
} // file paths
|
||||
});
|
||||
// // Set statistics
|
||||
// if (r_stat) {
|
||||
// r_stat->seconds = std::chrono::duration<double>(std::chrono::high_resolution_clock::now() - r_time).count();
|
||||
// r_stat->nbytes = buf.length * sizeof(typename mx_trait<f_type>::complex_type);
|
||||
// }
|
||||
// Return result
|
||||
return matrix.release();
|
||||
} catch (std::exception &ex) {
|
||||
mexErrMsgIdAndTxt("ptycho:read:failed", "%s", ex.what());
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Read in objects with multiple processes
|
||||
*
|
||||
* \param nprocs number of read processes
|
||||
* \param data_path paths to hdf5 files
|
||||
* \param data_location paths to datasets within hdf5 files
|
||||
* \param image_size two dimensional image size
|
||||
* \param roi_center center of two dimensional image
|
||||
* \tparam f_type float or double result type
|
||||
* \return MATLAB result array
|
||||
*/
|
||||
template <typename f_type>
|
||||
mxArray* read_data_parallel(int nprocs,
|
||||
const std::vector<std::string> &data_path,
|
||||
const std::vector<std::string> &data_location,
|
||||
std::vector<long> &image_size, std::vector<long> &roi_center)
|
||||
{
|
||||
if (data_path.size() == 1) {
|
||||
if (data_location.size() == 1) {
|
||||
DEBUG {
|
||||
OUT << "11: new eiger" << std::endl;
|
||||
}
|
||||
return read_data_parallel<f_type, kind_11>(nprocs, data_path, data_location, image_size, roi_center);
|
||||
} else {
|
||||
DEBUG {
|
||||
OUT << "1N: esrf" << std::endl;
|
||||
}
|
||||
return read_data_parallel<f_type, kind_1N>(nprocs, data_path, data_location, image_size, roi_center);
|
||||
}
|
||||
} else {
|
||||
if (data_location.size() == 1) {
|
||||
DEBUG {
|
||||
OUT << "N1: old eiger" << std::endl;
|
||||
}
|
||||
return read_data_parallel<f_type, kind_N1>(nprocs, data_path, data_location, image_size, roi_center);
|
||||
} else {
|
||||
DEBUG {
|
||||
OUT << "NN: file/dataset pairs" << std::endl;
|
||||
}
|
||||
return read_data_parallel<f_type, kind_NN>(nprocs, data_path, data_location, image_size, roi_center);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace data_prep {
|
||||
|
||||
mxArray* read_eiger_data(long nprocs,
|
||||
const std::vector<std::string> &data_path,
|
||||
const std::vector<std::string> &data_location,
|
||||
std::vector<long> &image_size, std::vector<long> &roi_center,
|
||||
precision::type prec)
|
||||
{
|
||||
if (nprocs <= 0)
|
||||
nprocs = 1;
|
||||
|
||||
mx_ptr<mxArray> result;
|
||||
{
|
||||
if (prec == precision::type::Double)
|
||||
result.reset(read_data_parallel<double>(nprocs, data_path, data_location, image_size, roi_center));
|
||||
else
|
||||
result.reset(read_data_parallel<float>(nprocs, data_path, data_location, image_size, roi_center));
|
||||
}
|
||||
return result.release();
|
||||
}
|
||||
|
||||
} // namespace data_prep
|
||||
@@ -0,0 +1,33 @@
|
||||
#ifndef READ_EIGER_DATA_H
|
||||
#define READ_EIGER_DATA_H
|
||||
|
||||
/*!
|
||||
* \file
|
||||
* Header file for the read Eiger data functionality
|
||||
*/
|
||||
|
||||
/*!
|
||||
* \brief Data preparator functions
|
||||
*/
|
||||
namespace data_prep {
|
||||
|
||||
/*!
|
||||
* \brief read Eiger data
|
||||
*
|
||||
* \param nprocs number of parallel read processes
|
||||
* \param data_path paths to Eiger data files
|
||||
* \param data_location location within file, either 1 for all paths or one for each path
|
||||
* \param image_size size of returned image data
|
||||
* \param roi_center region of interest center relative to measured data
|
||||
* \param prec precision of returned data
|
||||
* \return MATLAB array with resized and recentered images
|
||||
* \pre The 1:N or N:N relationship between data_location and data_path must hold
|
||||
*/
|
||||
mxArray* read_eiger_data(long nprocs,
|
||||
const std::vector<std::string> &data_path,
|
||||
const std::vector<std::string> &data_location,
|
||||
std::vector<long> &image_size, std::vector<long> &roi_center,
|
||||
precision::type prec);
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,498 @@
|
||||
/*!
|
||||
* \file
|
||||
* Read Object Data
|
||||
*
|
||||
* This file contains the main mex code for reading object data into MATLAB memory.
|
||||
*/
|
||||
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <chrono>
|
||||
#include <cassert>
|
||||
#include <array>
|
||||
#include <vector>
|
||||
#include <cstring>
|
||||
#include <cerrno>
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
#include <exception>
|
||||
#include <stdexcept>
|
||||
#include <unistd.h>
|
||||
#include <functional>
|
||||
#include <algorithm>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/wait.h>
|
||||
#include "precision.h"
|
||||
#include "hdf5_helper.h"
|
||||
#include "mex.h"
|
||||
#include "read_object_data.h"
|
||||
#include "debug_helper.h"
|
||||
#include "multi_processing.h"
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr int min_dims = 2; //!< Minimum number of dimensions (row, col)
|
||||
constexpr int max_dims = 4; //!< Maximum number of dimensions (nslices, nmodes, row, col)
|
||||
|
||||
#include "mex_helper.h"
|
||||
|
||||
/*!
|
||||
* \brief Read error exception class
|
||||
*/
|
||||
struct read_error : public std::exception {
|
||||
bool converted = false; //!< has the error message been converted (to include 1-based file index)
|
||||
uint32_t file_index; //!< file index (starting from 0)
|
||||
std::string error_message; //!< error message (converted if converted is true)
|
||||
|
||||
/*!
|
||||
* \brief Constructor
|
||||
* \param index 0-based file index
|
||||
* \param message unconverted error message
|
||||
*/
|
||||
read_error(uint32_t index, const std::string &message)
|
||||
: file_index(index), error_message(message)
|
||||
{}
|
||||
|
||||
/*!
|
||||
* \brief Retrieve converted error message
|
||||
* This method first attempts to convert the error message by appending the 1-based file index (MATLAB indexing convention).
|
||||
* If this fails, the unconverted error message is returned.
|
||||
* \return converted error message
|
||||
*/
|
||||
const char* what() const noexcept override
|
||||
{
|
||||
try {
|
||||
if (! converted) {
|
||||
std::ostringstream oss;
|
||||
oss << error_message << " (file " << (file_index+1) << ")";
|
||||
const_cast<std::string&>(error_message) = oss.str();
|
||||
const_cast<bool&>(converted) = true;
|
||||
}
|
||||
} catch (...) {}
|
||||
return error_message.c_str();
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief Error collector class
|
||||
* The class either keeps a vector to collect file indices for files that could not be read, or it throws an exception, depending on the in_collect_mode flag.
|
||||
* The class provides a pointer to the singleton instance.
|
||||
*/
|
||||
struct error_collector {
|
||||
static error_collector *singleton; //!< static pointer to make the singleton instance available
|
||||
bool in_collect_mode = false; //!< is the error method collecting or throwing errors?
|
||||
std::vector<uint32_t> bad_index; //!< vector of bad file indices
|
||||
|
||||
/*!
|
||||
* \brief Constructor
|
||||
* \param collect_mode true if errors should be collected instead of thrown
|
||||
*/
|
||||
explicit error_collector(bool collect_mode)
|
||||
: in_collect_mode(collect_mode)
|
||||
{
|
||||
if (! singleton)
|
||||
singleton = this;
|
||||
else
|
||||
throw std::runtime_error("internal error - multiple error_collector instances!");
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Destructor
|
||||
* Zero out the singleton pointer
|
||||
*/
|
||||
~error_collector() noexcept
|
||||
{
|
||||
singleton = nullptr;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Get pointer to singleton instance
|
||||
*/
|
||||
static error_collector& get() noexcept
|
||||
{
|
||||
return *singleton;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Collect or throw error
|
||||
* \param ex read error exception that would be thrown if not in collector mode
|
||||
*/
|
||||
void error(read_error &&ex)
|
||||
{
|
||||
if (! in_collect_mode)
|
||||
throw ex;
|
||||
bad_index.push_back(ex.file_index);
|
||||
DEBUG {
|
||||
OUT << "error reading file " << ex.file_index << ": " << ex.error_message << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Add index to bad indices vector
|
||||
* \param index index of file that can not be read
|
||||
*/
|
||||
void add(uint32_t index) noexcept
|
||||
{
|
||||
bad_index.push_back(index);
|
||||
}
|
||||
};
|
||||
|
||||
error_collector* error_collector::singleton = nullptr; //!< static error_collectr singleton pointer
|
||||
|
||||
/*!
|
||||
* \brief Adapt data space layout
|
||||
*
|
||||
* Adapt the memory and file dataset layouts by selecting apropriate hyperslabs
|
||||
*
|
||||
* \param mspace memory data space layout
|
||||
* \param fspace file dataset layout
|
||||
* \param mdim memory space dimensions
|
||||
* \param fdim file dataset dimensions
|
||||
* \param doffset Start of the dimsensions present in the file data
|
||||
*/
|
||||
void adapt_spaces(hdf5::dataspace &mspace, hdf5::dataspace &fspace, const hsize_t mdim[max_dims], const hsize_t fdim[max_dims], int doffset)
|
||||
{
|
||||
hsize_t mstart[max_dims] = {0};
|
||||
hsize_t fstart[max_dims] = {0};
|
||||
hsize_t mcount[max_dims] = { mdim[0], mdim[1], mdim[2], mdim[3] };
|
||||
hsize_t fcount[max_dims] = { fdim[0], fdim[1], fdim[2], fdim[3] };
|
||||
bool mselect = false;
|
||||
bool fselect = false;
|
||||
|
||||
for (int i=2; i<=3; i++) {
|
||||
if (mdim[i] < fdim[i]) {
|
||||
fstart[i] = (fdim[i] - mdim[i]) / 2;
|
||||
fcount[i] = mdim[i];
|
||||
fselect = true;
|
||||
} else if (mdim[i] > fdim[i]) {
|
||||
mstart[i] = (mdim[i] - fdim[i]) / 2;
|
||||
mcount[i] = fdim[i];
|
||||
mselect = true;
|
||||
}
|
||||
}
|
||||
if (mselect) {
|
||||
DEBUG {
|
||||
OUT << "memory subspace start=" << mstart[0] << ',' << mstart[1] << ',' << mstart[2] << ',' << mstart[3] << '/' << doffset << " count=" << mcount[0] << ',' << mcount[1] << ',' << mcount[2] << ',' << mcount[3] << '/' << doffset << "\nfile space is " << fdim[0] << ',' << fdim[1] << ',' << fdim[2] << ',' << fdim[3] << std::endl;
|
||||
}
|
||||
if (H5Sselect_hyperslab(mspace, H5S_SELECT_SET, &mstart[doffset], nullptr, &mcount[doffset], nullptr) < 0)
|
||||
throw hdf5::exception("unable to set hdf5 memory space");
|
||||
}
|
||||
if (fselect) {
|
||||
DEBUG {
|
||||
OUT << "file subspace start=" << fstart[0] << ',' << fstart[1] << ',' << fstart[2] << ',' << fstart[3] << '/' << doffset << " count=" << fcount[0] << ',' << fcount[1] << ',' << fcount[2] << ',' << fcount[3] << '/' << doffset << "\nmem space is " << mdim[0] << ',' << mdim[1] << ',' << mdim[2] << ',' << mdim[3] << std::endl;
|
||||
}
|
||||
if (H5Sselect_hyperslab(fspace, H5S_SELECT_SET, &fstart[doffset], nullptr, &fcount[doffset], nullptr) < 0)
|
||||
throw hdf5::exception("unable to set hdf5 dataset space");
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Read object from file into buffer
|
||||
* \param file HDF5 file
|
||||
* \param object_path path to object dataset
|
||||
* \param mdims result matrix dimensions (with size 5)
|
||||
* \param buf pointer to parent acessible buffer
|
||||
* \tparam f_type buffer element type
|
||||
*/
|
||||
template <typename f_type>
|
||||
void read_complex_matrix(hdf5::file &file, const std::string &object_path,
|
||||
const std::vector<mwSize> &mdims, typename mx_trait<f_type>::complex_type *buf)
|
||||
{
|
||||
hsize_t fdims[max_dims] = { 1, 1, 0, 0 }; // (nslices, nmodes, nrows, ncols)
|
||||
hsize_t bdims[max_dims] = { 1, 1, 0, 0 }; // file and buffer dimensions
|
||||
unsigned long nelements = 0;
|
||||
int dim_offset = 0;
|
||||
int ndims = 0;
|
||||
int mdim_offset = 0;
|
||||
for (; (mdim_offset < max_dims-2) && (mdims[mdim_offset] == 1); mdim_offset++);
|
||||
|
||||
// Create complex type
|
||||
hdf5::h5type complex_type(hdf5::create_complex_type<f_type>());
|
||||
// Open dataset
|
||||
hdf5::dataset dataset(H5Dopen(file, object_path.c_str(), H5P_DEFAULT));
|
||||
if (! dataset.valid())
|
||||
throw hdf5::exception("unable to open dataset");
|
||||
hdf5::dataspace dataspace(H5Dget_space(dataset));
|
||||
if (! dataspace.valid())
|
||||
throw hdf5::exception("unable to open data space for dataset");
|
||||
ndims = H5Sget_simple_extent_ndims(dataspace);
|
||||
if (ndims < min_dims || ndims > max_dims)
|
||||
throw hdf5::exception("wrong number of dimensions in dataset");
|
||||
dim_offset = max_dims - ndims;
|
||||
if (H5Sget_simple_extent_dims(dataspace, &fdims[dim_offset], NULL) != ndims)
|
||||
throw hdf5::exception("dimension error");
|
||||
for (int i=max_dims-2; i; i--) {
|
||||
if (fdims[i-1] != mdims[i]) {
|
||||
if (i != mdim_offset)
|
||||
throw hdf5::exception("dimension mismatch");
|
||||
}
|
||||
}
|
||||
// Create memory space and adapt data space
|
||||
bdims[0] = fdims[0]; // nslices
|
||||
bdims[1] = fdims[1]; // nmodes
|
||||
bdims[2] = mdims[3]; // nrows wanted
|
||||
bdims[3] = mdims[4]; // ncols wanted
|
||||
DEBUG {
|
||||
OUT << "bdims(" << bdims[0] << ',' << bdims[1] << ',' << bdims[2] << '/' << fdims[2] << ',' << bdims[3] << '/' << fdims[3] << ')' << dim_offset << std::endl;
|
||||
}
|
||||
hdf5::dataspace memspace(H5Screate_simple(ndims, &bdims[dim_offset], nullptr));
|
||||
if (! memspace.valid())
|
||||
throw hdf5::exception("unable to create memory data space");
|
||||
if ((bdims[2] != fdims[2]) || (bdims[3] != fdims[3]))
|
||||
adapt_spaces(memspace, dataspace, bdims, fdims, dim_offset);
|
||||
// Read dataset
|
||||
if (H5Dread(dataset, complex_type, memspace, dataspace, H5P_DEFAULT, buf) < 0)
|
||||
throw hdf5::exception("unable to read dataset");
|
||||
|
||||
DEBUG {
|
||||
int last = bdims[max_dims-2] * bdims[max_dims-1] - 1;
|
||||
OUT << "read finished - [0]=(" << buf[0].real << ',' << buf[0].imag << ") [" << last << "]=(" << buf[last].real << ',' << buf[last].imag << ')' << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Open HDF5 file
|
||||
* \param file_path path to HDF5 file
|
||||
* \return HDF5 file object id
|
||||
*/
|
||||
hid_t open_file(const std::string &file_path)
|
||||
{
|
||||
hdf5::file file(H5Fopen(file_path.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT));
|
||||
if (! file.valid())
|
||||
throw hdf5::exception("unable to open file");
|
||||
return file.grab();
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Nonzero dimension size?
|
||||
* \param dims size of twodimensional object
|
||||
* \tparam T type of dimension size
|
||||
* \return are dimension sizes positive?
|
||||
*/
|
||||
template <typename T>
|
||||
bool dims_ok (const T *dims) noexcept
|
||||
{
|
||||
return dims[0] > 0 || dims[1] > 0;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Drop leading dimensions of size 1
|
||||
* \param dim dimsension sizes
|
||||
* \param max maximum dimensions
|
||||
* \tparam T type of dimension size
|
||||
* \return number of dimsension of size bigger than 1
|
||||
*/
|
||||
template <typename T>
|
||||
int collapse_dims (const std::vector<T> &dim, int max) noexcept
|
||||
{
|
||||
int i=0;
|
||||
for (; i<max-2; i++) {
|
||||
if (dim[i] != 1)
|
||||
break;
|
||||
}
|
||||
return max-i;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Get HDF5 object size
|
||||
*
|
||||
* \param file HDF5 file
|
||||
* \param dataset_path object dataset path
|
||||
* \param dims filled with object dimensions
|
||||
*/
|
||||
void get_object_size(hdf5::file &file, const std::string &dataset_path, hsize_t dims[max_dims])
|
||||
{
|
||||
int dim_offset = 0;
|
||||
int ndims;
|
||||
|
||||
hdf5::dataset dataset(H5Dopen(file, dataset_path.c_str(), H5P_DEFAULT));
|
||||
if (! dataset.valid())
|
||||
throw hdf5::exception("unable to open dataset");
|
||||
hdf5::dataspace dataspace(H5Dget_space(dataset));
|
||||
if (! dataspace.valid())
|
||||
throw hdf5::exception("unable to get data space");
|
||||
ndims = H5Sget_simple_extent_ndims(dataspace);
|
||||
if (ndims < min_dims || ndims > max_dims)
|
||||
throw hdf5::exception("wrong number of dimensions in dataset");
|
||||
dim_offset = max_dims - ndims;
|
||||
if (H5Sget_simple_extent_dims(dataspace, &dims[dim_offset], NULL) != ndims)
|
||||
throw hdf5::exception("unable to get dimension sizes");
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Read in objects with multiple processes
|
||||
*
|
||||
* \param nprocs number of read processes
|
||||
* \param dims desired result dimensions (or [0, 0] to get the default behaviour)
|
||||
* \param object_path path to hdf5 object within the files
|
||||
* \param file_paths paths to hdf5 objects
|
||||
* \param r_stat fill in if not null
|
||||
* \tparam f_type float or double result type
|
||||
* \return MATLAB result array
|
||||
*/
|
||||
template <typename f_type>
|
||||
mxArray* read_objects_parallel(int nprocs, std::array<int64_t, 2> &dims,
|
||||
const std::string &object_path, const std::vector<std::string> &file_paths,
|
||||
data_prep::read_stat *r_stat)
|
||||
{
|
||||
try {
|
||||
std::vector<mwSize> mdims({ 1, 1, 1, 0, 0 }); // extended matrix dimensions
|
||||
hsize_t odims[max_dims] = { 1, 1, 0, 0 }; // extended object dimensions (nobjects, nslices, nmodes, nrows, ncols)
|
||||
|
||||
// Assign matrix dimension values
|
||||
{
|
||||
hdf5::file file(open_file(file_paths[0]));
|
||||
get_object_size(file, object_path, odims);
|
||||
}
|
||||
for (unsigned int i=1; i<max_dims-1; i++) {
|
||||
mdims[i] = (mwSize)odims[i-1];
|
||||
}
|
||||
if (!dims_ok(&dims[0])) {
|
||||
mdims[max_dims-1] = odims[max_dims-2];
|
||||
mdims[max_dims] = odims[max_dims-1];
|
||||
} else {
|
||||
mdims[max_dims-1] = dims[0];
|
||||
mdims[max_dims] = dims[1];
|
||||
}
|
||||
int nmdims = collapse_dims(mdims, max_dims+1);
|
||||
if (file_paths.size() > 1) {
|
||||
mdims[max_dims - nmdims] = (mwSize)file_paths.size();
|
||||
nmdims++;
|
||||
}
|
||||
|
||||
std::vector<mwSize> matrix_dims(mdims); // matrix dimensions
|
||||
matrix_dims.erase(matrix_dims.begin(), matrix_dims.begin() + max_dims + 1 - nmdims); // cut away extension dimensions
|
||||
|
||||
// Create space for MATLAB array
|
||||
mx_ptr<mxArray> matrix;
|
||||
{
|
||||
std::vector<mwSize> rdims(matrix_dims);
|
||||
std::reverse(rdims.begin(), rdims.end());
|
||||
matrix.reset(mxCreateNumericArray(nmdims, &rdims[0], mx_trait<f_type>::class_id, mxCOMPLEX));
|
||||
}
|
||||
if (! matrix.get())
|
||||
throw std::runtime_error("matrix creation failed");
|
||||
|
||||
// Prepare (shared) memory buffer
|
||||
// HACK: misuse error message area as array of bad file indices
|
||||
// There should be space for the max number of handled files plus the length field
|
||||
const unsigned int message_size = error_collector::singleton->in_collect_mode ? std::max((file_paths.size() / nprocs + 2) * sizeof(uint32_t), 64ul) : 0;
|
||||
mp::buf<typename mx_trait<f_type>::complex_type> buf(mx_trait<f_type>::get_complex(matrix.get()), matrix_dims, nprocs, message_size);
|
||||
|
||||
auto r_time = std::chrono::high_resolution_clock::now();
|
||||
// Spawn processes if needed
|
||||
mp::run<typename mx_trait<f_type>::complex_type>(nprocs, buf, [&file_paths, &object_path, &mdims](int proc, mp::buf<typename mx_trait<f_type>::complex_type> &buf) {
|
||||
// Read objects data into buffer
|
||||
typename mx_trait<f_type>::complex_type *data_buf = buf.get(proc);
|
||||
int first = buf.offset(proc);
|
||||
int last = buf.offset(proc+1);
|
||||
for (int obj=first; obj<last; obj++) {
|
||||
try {
|
||||
std::size_t offset = (obj - first) * buf.chunk_size;
|
||||
DEBUG {
|
||||
OUT << "Process " << proc << ": " << file_paths[obj] << '@' << offset << std::endl;
|
||||
}
|
||||
hdf5::file file(open_file(file_paths[obj]));
|
||||
read_complex_matrix<f_type>(file, object_path, mdims, &data_buf[offset]);
|
||||
} catch (std::exception &ex) {
|
||||
error_collector::singleton->error(read_error(obj, ex.what()));
|
||||
} catch (...) {
|
||||
error_collector::singleton->error(read_error(obj, "mysterious read error"));
|
||||
}
|
||||
}
|
||||
if (proc && error_collector::singleton->in_collect_mode) {
|
||||
// HACK: misuse error message area as array of bad file indices for this process
|
||||
// Layout: [ N | elem0 | elem1 | ... | elemN-1 ]
|
||||
uint32_t *msg_area = reinterpret_cast<uint32_t*>(buf.get_msg(proc));
|
||||
auto &bad_index = error_collector::singleton->bad_index;
|
||||
msg_area[0] = bad_index.size();
|
||||
std::copy(bad_index.begin(), bad_index.end(), &msg_area[1]);
|
||||
}
|
||||
});
|
||||
// Set statistics (not correct for error collection mode)
|
||||
if (r_stat) {
|
||||
r_stat->seconds = std::chrono::duration<double>(std::chrono::high_resolution_clock::now() - r_time).count();
|
||||
r_stat->nbytes = buf.length * sizeof(typename mx_trait<f_type>::complex_type);
|
||||
}
|
||||
// Collect bad indices in error collection mode
|
||||
if (error_collector::singleton->in_collect_mode) {
|
||||
DEBUG {
|
||||
OUT << "Collecting bad indices:" << std::endl;
|
||||
}
|
||||
for (int proc=1; proc<nprocs; proc++) {
|
||||
// HACK: misuse error message area as array of bad file indices
|
||||
// Layout: [ N | elem0 | elem1 | ... | elemN-1 ]
|
||||
uint32_t *msg_area = reinterpret_cast<uint32_t*>(buf.get_msg(proc));
|
||||
uint32_t nbad = msg_area[0];
|
||||
DEBUG {
|
||||
OUT << " process " << proc << ": " << nbad << " indices" << std::endl;
|
||||
}
|
||||
if (nbad > (file_paths.size() / nprocs + 1))
|
||||
throw std::runtime_error("transfer of bad indices failed");
|
||||
for (uint32_t i=1; i<=nbad; i++)
|
||||
error_collector::singleton->add(msg_area[i]);
|
||||
}
|
||||
}
|
||||
// Return result
|
||||
return matrix.release();
|
||||
} catch (std::exception &ex) {
|
||||
mexErrMsgIdAndTxt("ptycho:read:failed", "%s", ex.what());
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Convert vector<uint32_t> to MATLAB array
|
||||
* \param vec vector to be converted
|
||||
* \return MATLAB array
|
||||
*/
|
||||
mxArray* vector_to_array(std::vector<uint32_t> &vec)
|
||||
{
|
||||
mwSize vlen[2] = {1, vec.size()};
|
||||
mx_ptr<mxArray> arr(mxCreateNumericArray(2, vlen, mxUINT32_CLASS, mxREAL));
|
||||
if (! arr.get())
|
||||
throw std::runtime_error("unable to create bad indices array");
|
||||
if (! vec.empty()) {
|
||||
auto* element = mxGetUint32s(arr.get());
|
||||
if (! element)
|
||||
throw std::runtime_error("unable to retrieve pointer for bad indices array elements");
|
||||
for (auto& e : vec)
|
||||
*element++ = e+1;
|
||||
}
|
||||
return arr.release();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace data_prep {
|
||||
|
||||
mxArray* read_object_data(int nprocs, precision::type prec, std::array<int64_t, 2> &dims,
|
||||
const std::string &object_path, const std::vector<std::string> &file_paths,
|
||||
mxArray **bad_file_idx, read_stat *r_stat)
|
||||
{
|
||||
if (! file_paths.size())
|
||||
return nullptr;
|
||||
|
||||
if (nprocs <= 0)
|
||||
nprocs = 1;
|
||||
if (nprocs > file_paths.size())
|
||||
nprocs = (int)file_paths.size();
|
||||
|
||||
error_collector errcol(bad_file_idx != nullptr);
|
||||
|
||||
mx_ptr<mxArray> result;
|
||||
{
|
||||
if (prec == precision::type::Double)
|
||||
result.reset(read_objects_parallel<double>(nprocs, dims, object_path, file_paths, r_stat));
|
||||
else
|
||||
result.reset(read_objects_parallel<float>(nprocs, dims, object_path, file_paths, r_stat));
|
||||
}
|
||||
|
||||
if (errcol.in_collect_mode)
|
||||
*bad_file_idx = vector_to_array(errcol.bad_index);
|
||||
|
||||
return result.release();
|
||||
}
|
||||
|
||||
} // namespace data_prep
|
||||
@@ -0,0 +1,43 @@
|
||||
#ifndef READ_OBJECT_DATA_H
|
||||
#define READ_OBJECT_DATA_H
|
||||
|
||||
/*!
|
||||
* \file
|
||||
* Declaration of the read_object_data function
|
||||
*/
|
||||
|
||||
/*!
|
||||
* \brief Data preparator functions
|
||||
*/
|
||||
namespace data_prep {
|
||||
|
||||
/*!
|
||||
* \brief Statistics collection
|
||||
*/
|
||||
struct read_stat final {
|
||||
double seconds; //!< read access time in seconds
|
||||
unsigned long nbytes; //!< number of bytes transferred
|
||||
};
|
||||
|
||||
/*!
|
||||
* \brief Read object data
|
||||
*
|
||||
* The function will either create the bad indices vector or return with exception and error message.
|
||||
*
|
||||
* \param nprocs Number of read processes
|
||||
* \param prec Single or Double precision
|
||||
* \param dims Object size (nrows, ncols)
|
||||
* \param object_path Path to object within HDF5 file
|
||||
* \param file_paths Paths to HDF5 files, each containing a diffraction pattern at object_path
|
||||
* \param bad_file_idx Indices of nonreadable files array, to be created if not NULL
|
||||
* \param r_stat Fill with info if not null
|
||||
* \return MATLAB array with object data
|
||||
*/
|
||||
mxArray* read_object_data(int nprocs, precision::type prec, std::array<int64_t, 2> &dims,
|
||||
const std::string &object_path, const std::vector<std::string> &file_paths,
|
||||
mxArray **bad_file_idx,
|
||||
read_stat *r_stat = nullptr);
|
||||
|
||||
} // namespace data_prep
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,8 @@
|
||||
module add gcc/6.3.0
|
||||
module add hdf5_serial/1.8.18
|
||||
|
||||
module use unstable
|
||||
module add matlab/2018a
|
||||
|
||||
module use Libraries
|
||||
module add tiff/4.0.9
|
||||
Reference in New Issue
Block a user