initial commit

+io/+CBF/cbf_uncompress.c

@@ -0,0 +1,161 @@
+/* cbf_uncompress.c: 
+ 
+   Compilation from Matlab:
+   mex cbf_uncompress.c
+   maybe a tiny bit faster code is generated by 
+   mex -O COPTIMFLAGS='-O2' LDOPTIMFLAGS='-O2' cbf_uncompress.c
+ 
+   Usage from Matlab:
+   [frame] = ...
+       cbf_uncompress(dat_in,dim1,dim2,no_of_in_bytes,compression_type);
+ 
+   history:
+   April 24th 2008: 
+   1st version based on code snippet from Eric Eikenberry
+
+*-----------------------------------------------------------------------*
+|                                                                       |
+|  Except where otherwise noted, this work is licensed under a          |
+|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+|  International (CC BY-NC-SA 4.0) license.                             |
+|                                                                       |
+|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+|                                                                       |
+|       Author: CXS group, PSI                                          |
+*-----------------------------------------------------------------------*
+ You may use this code with the following provisions:
+
+ If the code is fully or partially redistributed, or rewritten in another
+   computing language this notice should be included in the redistribution.
+
+ If this code, or subfunctions or parts of it, is used for research in a 
+   publication or if it is fully or partially rewritten for another 
+   computing language the authors and institution should be acknowledged 
+   in written form in the publication: “Data processing was carried out 
+   using the “cSAXS matlab package” developed by the CXS group,
+   Paul Scherrer Institut, Switzerland.” 
+   Variations on the latter text can be incorporated upon discussion with 
+   the CXS group if needed to more specifically reflect the use of the package 
+   for the published work.
+
+ A publication that focuses on describing features, or parameters, that
+    are already existing in the code should be first discussed with the
+    authors.
+   
+ This code and subroutines are part of a continuous development, they 
+    are provided “as they are” without guarantees or liability on part
+    of PSI or the authors. It is the user responsibility to ensure its 
+    proper use and the correctness of the results.
+*/
+
+#include "mex.h"
+
+#include <math.h> 
+#include <stdio.h>
+
+void mexFunction(int nlhs, mxArray *plhs[],
+                 int nrhs, const mxArray *prhs[])
+{
+  const mxArray *curr_arg;
+  union { 
+    unsigned char *uint8;
+             char *int8;
+	unsigned short *uint16;
+	         short *int16;
+	unsigned int *uint32;
+	         int *int32;
+  } data_in,data_in_start;
+
+  int dim1, dim2, no_of_in_bytes, compression_type;
+  int diff, val_curr;
+  double *frame_out, *frame_out_start;
+  
+    /* initialize return argument*/
+  plhs[0] = NULL;
+  
+    /* Check for proper number of arguments. */
+  if (nrhs != 5) 
+    mexErrMsgTxt("Five input arguments required: dat_in,dim1,dim2,no_of_in_bytes,compression_type.");
+  else if (nlhs != 1) 
+    mexErrMsgTxt("One output argument has to be specified.");
+
+  {
+    int ind;
+    for (ind = 0; ind < nrhs; ind++) {
+      if(mxGetNumberOfDimensions(prhs[ind]) != 2) {
+        printf("The %d. input argument must have two dimensions.",ind+1);
+        mexErrMsgTxt("wrong number of dimensions");
+      }
+    }
+  }
+  
+    /* check 1st input argument: input data */
+  curr_arg = prhs[0];
+  if (mxIsUint8(curr_arg) != 1)
+    mexErrMsgTxt("Input 1 (input data) must be of type uint8.");
+  data_in_start.uint8 = data_in.uint8 = (char *) mxGetPr(curr_arg);
+
+    /* check 2nd input argument: dim1 */
+  curr_arg = prhs[1];
+  if (mxIsDouble(curr_arg) != 1)
+    mexErrMsgTxt("Input 2 (dimension 1) must be of type double.");
+  dim1 = mxGetScalar(curr_arg);
+  if (dim1 < 1) {
+    mexErrMsgTxt("Input 2 (dimension 1) must be at least 1.");
+  }
+
+    /* check 3rd input argument: dim2 */
+  curr_arg = prhs[2];
+  if (mxIsDouble(curr_arg) != 1)
+    mexErrMsgTxt("Input 3 (dimension 2) must be of type double.");
+  dim2 = mxGetScalar(curr_arg);
+  if (dim2 < 1) {
+    mexErrMsgTxt("Input 3 (dimension 2) must be at least 1.");
+  }
+  
+    /* check 4th input argument: no_of_in_bytes */
+  curr_arg = prhs[3];
+  if (mxIsDouble(curr_arg) != 1)
+    mexErrMsgTxt("Input 4 (no. of input bytes) must be of type double.");
+  no_of_in_bytes = mxGetScalar(curr_arg);
+  
+    /* check 5th input argument: compression_type */
+  curr_arg = prhs[4];
+  if (mxIsDouble(curr_arg) != 1)
+    mexErrMsgTxt("Input 5 (compression type) must be of type double.");
+  compression_type = mxGetScalar(curr_arg);
+  if (compression_type != 1) {
+    mexErrMsgTxt("currently only compression type 1, byte-offset compression, is supported");
+  }
+  
+    /* allocate memory for the output data */
+  plhs[0] = mxCreateNumericMatrix(dim1, dim2, mxDOUBLE_CLASS, mxREAL);
+  if (plhs[0] == NULL)
+    mexErrMsgTxt("Could not allocate memory for return data.");
+  frame_out_start = frame_out = mxGetPr(plhs[0]);
+
+  val_curr = 0;
+  while (data_in.uint8-data_in_start.uint8 < no_of_in_bytes) {
+	if (*data_in.uint8 != 0x80) {
+	  diff = (int) *data_in.int8++;
+    } else {
+  	  data_in.uint8++;
+	  if (*data_in.uint16 != 0x8000) {
+		diff = (int) *data_in.int16++;
+      } else {
+        data_in.uint16++;
+		diff = (int) *data_in.int32++;
+  	  }
+    }
+	val_curr += diff;
+    *frame_out++ = (double) val_curr;
+  }
+
+  if (frame_out-frame_out_start != dim1*dim2) {
+    printf("%ld bytes after uncompression, %ld bytes expected",
+           frame_out-frame_out_start, dim1*dim2);
+    mexErrMsgTxt("mismatch in number of extracted data bytes");
+  }
+  
+  return;
+}

+io/+CBF/cbfread.m

@@ -0,0 +1,339 @@
+% Call function without arguments for a detailed explanation of its use
+
+% Filename: $RCSfile: cbfread.m,v $
+%
+% $Revision: 1.2 $  $Date: 2011/05/09 13:03:07 $
+% $Author:  $
+% $Tag: $
+%
+% Description:
+% Macro for reading Crystallographic Binary File (CBF) files written by the
+% Pilatus detector control program camserver. 
+%
+% Note:
+% Compile the C-program cbf_uncompress using mex (see header of
+% cbf_uncompress.c) to use it for uncompression instead of the slower
+% Matlab code. 
+% Currently this routine supports only the subset of CBF features needed to
+% read the Pilatus detector data. 
+% Call without arguments for a brief help text.
+%
+% Dependencies:
+% - image_read_set_default
+% - fopen_until_exists
+% - get_hdr_val
+% - compiling cbf_uncompress.c increases speed but is not mandatory
+%
+%
+% history:
+%
+% May 9th 2008: 1st version
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [frame,vararg_remain] = cbfread(filename,varargin)
+import io.*
+import io.CBF.*
+import utils.char_to_cellstr
+import utils.fopen_until_exists
+import utils.get_hdr_val
+
+% 0: no debug information
+% 1: some feedback
+% 2: a lot of information
+debug_level = 0;
+
+% initialize return argument
+frame = struct('header',[], 'data',[]);
+
+
+% check minimum number of input arguments
+if (nargin < 1)
+    image_read_sub_help(mfilename,'cbf');
+    error('At least the filename has to be specified as input parameter.');
+end
+
+% accept cell array with name/value pairs as well
+no_of_in_arg = nargin;
+if (nargin == 2)
+    if (isempty(varargin))
+        % ignore empty cell array
+        no_of_in_arg = no_of_in_arg -1;
+    else
+        if (iscell(varargin{1}))
+            % use a filled one given as first and only variable parameter
+            varargin = varargin{1};
+            no_of_in_arg = 1 + length(varargin);
+        end
+    end
+end
+
+% check number of input arguments
+if (rem(no_of_in_arg,2) ~= 1)
+    error('The optional parameters have to be specified as ''name'',value pairs');
+end
+    
+% set default values for the variable input arguments and parse the named
+% parameters: 
+vararg = cell(0,0);
+for ind = 1:2:length(varargin)
+    name = varargin{ind};
+    value = varargin{ind+1};
+    switch name
+        otherwise
+            % pass further arguments on to fopen_until_exists
+            vararg{end+1} = name;
+            vararg{end+1} = value;
+    end
+end
+
+
+% expected maximum length for the text header
+max_header_length = 4096;
+
+% end of header signature
+eoh_signature = char([ 12 26 4 213 ]);
+
+% CBF file signature
+cbf_signature = '###CBF: VERSION';
+
+% Calling an external C routine for uncompressing the data did save about
+% 30% time on a specific machine. 
+% The C-routine is used if a compiled version of it exists. 
+% See the header of cbf_uncompress.c for information on how to compile the
+% C file using mex in Matlab. 
+
+c_routine = exist('+io/+CBF/cbf_uncompress.mexa64', 'file') || ...
+            ~isempty(which('cbf_uncompress')) ; % faster than which('cbf_uncompress')
+
+
+% try to open the data file
+if (debug_level >= 1)
+    fprintf('Opening %s.\n',filename);
+end
+[fid,vararg_remain] = fopen_until_exists(filename,vararg);
+if (fid < 0)
+    return;
+end
+
+% read all data at once
+[fdat,fcount] = fread(fid,'uint8=>uint8');
+
+% close input data file
+fclose(fid);
+if (debug_level >= 2)
+    fprintf('%d data bytes read\n',fcount);
+end
+
+% search for end of header signature within the expected maximum length of
+% a header
+end_of_header_pos = ...
+    strfind( fdat(1:min(max_header_length,length(fdat)))',...
+             eoh_signature );
+if (length(end_of_header_pos) < 1)
+    cbf_error(filename,'no header end signature found');
+    return;
+end
+if (debug_level >= 2)
+    fprintf('Header length is %d bytes.\n',end_of_header_pos -1);
+end
+
+% return the complete header as lines of a cell array
+frame.header = char_to_cellstr( char(fdat(1:(end_of_header_pos-1))') );
+
+% check for CBF signature
+if (~strncmp(cbf_signature,frame.header{1},length(cbf_signature)))
+    cbf_error(filename,[ 'CBF signature ''' cbf_signature ...
+        ''' not found in first line ''' frame.header{1} '''' ]);
+end
+
+% extract the mandatory information for decompression from the header
+no_of_bin_bytes = get_hdr_val(frame.header,'X-Binary-Size:','%f',1);
+dim1 = get_hdr_val(frame.header,'X-Binary-Size-Fastest-Dimension:','%f',1);
+dim2 = get_hdr_val(frame.header,'X-Binary-Size-Second-Dimension:','%f',1);
+el_type = get_hdr_val(frame.header,'X-Binary-Element-Type: "','%[^"]',1);
+switch (el_type)
+    case 'signed 32-bit integer'
+        bytes_per_pixel = 4;
+    otherwise
+        cbf_error(filename,[ 'unknown element type ' el_type ]);
+end
+compr_type = get_hdr_val(frame.header,'conversions="','%[^"]',1);
+switch (compr_type)
+    case 'x-CBF_BYTE_OFFSET' 
+        compression_type = 1;
+    case 'x-CBF_NONE'
+        compression_type = 2;
+    otherwise
+        cbf_error(filename,[ 'unknown compression type ' compr_type ]);
+end
+if (debug_level >= 2)
+    fprintf('Frame dimensions are %d x %d.\n',dim2,dim1);
+end
+
+% uncompress the binary data
+[frame.data] = ...
+    extract_frame(fdat((end_of_header_pos+length(eoh_signature)):end),...
+        dim1,dim2,no_of_bin_bytes,compression_type,...
+        filename,...
+        c_routine,debug_level);
+    
+
+    
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+function [] = cbf_error(filename,text)
+
+fprintf('cbfread of %s:\n %s\n',filename,text);
+return;
+
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+function [frame] = ...
+    extract_frame(dat_in,...
+        dim1,dim2,no_of_in_bytes,compression_type,...
+        filename,...
+        c_routine,debug_level)
+
+import io.*
+import io.CBF.*
+import utils.char_to_cellstr
+import utils.fopen_until_exists
+import utils.get_hdr_val
+
+
+% uncompressed data are copied directly
+if (compression_type == 2)
+    % initialize return array
+    frame = zeros(dim1,dim2);
+    % copy uncompressed data
+    for (ind_out = 1:(dim1*dim2))
+        ind_in = ind_out *4 -3;
+        frame(ind_out) = double(dat_in(ind_in)) + ...
+            256 * double(dat_in(ind_in+1)) + ...
+            65536 * double(dat_in(ind_in+2)) + ...
+            16777216 * double(dat_in(ind_in+3));
+    end
+    return;
+end
+
+
+if (c_routine)    
+    if (debug_level >= 2)
+        fprintf('C routine called.\n');
+    end
+    [frame] = ...
+        cbf_uncompress(dat_in,dim1,dim2,no_of_in_bytes,compression_type);
+    return;
+end
+
+if (debug_level >= 2)
+    fprintf('Matlab routine called.\n');
+end
+
+
+% initialize return array
+frame = zeros(dim1,dim2);
+
+% only byte-offset compression is supported
+if (compression_type ~= 1)
+    cbf_error(filename,...
+        ['extract_frame does not support compression type no. ' ...
+        num2str(compression_type)]);
+end
+
+
+% In byte-offset compression the difference to the previous pixel value is
+% stored as a byte, 16-bit integer or 32-bit integer, depending on its
+% size. 
+% The sizes above one byte are indicated by the escape sequence -1 in the
+% previous data format, i.e, a 32-bit integer is preceded by the sequence %
+% 0x80 (too large for a byte)
+% 0x8000 (too large for a 16-bit integer). 
+ind_out = 1;
+ind_in = 1;
+val_curr = 0;
+val_diff = 0;
+while (ind_in <= no_of_in_bytes)
+    val_diff = double(dat_in(ind_in));
+    ind_in = ind_in +1;
+    if (val_diff ~= 128)
+        % if not escaped as -128 (0x80=128) use the current byte as
+        % difference, with manual complement to emulate the sign
+        if (val_diff >= 128)
+            val_diff = val_diff - 256;
+        end
+    else
+        % otherwise check for 16-bit integer value
+        if ((dat_in(ind_in) ~= 0) || (dat_in(ind_in+1) ~= 128))
+            % if not escaped as -32768 (0x8000) use the current 16-bit integer
+            % as difference 
+            val_diff = double(dat_in(ind_in)) + ...
+                256 * double(dat_in(ind_in+1));
+            % manual complement to emulate the sign
+            if (val_diff >= 32768)
+                val_diff = val_diff - 65536;
+            end
+            ind_in = ind_in +2;
+        else
+            ind_in = ind_in +2;
+            % if everything else failed use the current 32-bit value as
+            % difference
+            val_diff = double(dat_in(ind_in)) + ...
+                256 * double(dat_in(ind_in+1)) + ...
+                65536 * double(dat_in(ind_in+2)) + ...
+                16777216 * double(dat_in(ind_in+3));
+            % manual complement to emulate the sign
+            if (val_diff >= 2147483648)
+                val_diff = val_diff - 4294967296;
+            end
+            ind_in = ind_in +4;
+        end
+    end
+	val_curr = val_curr + val_diff;
+    frame(ind_out) = val_curr;
+    ind_out = ind_out +1;
+end
+    
+if (ind_out-1 ~= dim1*dim2)
+    cbf_error(filename,[ 'mismatch between ' num2str(ind_out-1) ...
+        ' bytes after decompression with ' num2str(dim1*dim2) ...
+        ' expected ones' ]);
+end
+
+
+% if (~original_orientation)
+%     frame.data = frame.data(end:-1:1,end:-1:1)';
+% end

+io/+CBF/cbfwrite.m

@@ -0,0 +1,258 @@
+% Call function without arguments for a detailed explanation of its use
+
+% Filename: $RCSfile: cbfwrite.m,v $
+%
+% $Revision: 1.2 $  $Date: 2014/04/17 16:49:22 $
+% $Author:  $
+% $Tag: $
+%
+% Description:
+% Macro for writing data to a Crystallographic Binary File (CBF) file. 
+% Assumes that the data have been read from such a file beforehand, i.e.,
+% that the header exists already. 
+%
+%
+% history:
+%
+% April 10th 2014: 1st version
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [fcount_total] = cbfwrite(filename,frame,varargin)
+import io.CBF.*
+import io.*
+import utils.default_parameter_value
+import utils.fopen_until_exists
+import utils.get_hdr_val
+
+% 0: no debug information
+% 1: some feedback
+% 2: a lot of information
+debug_level = 0;
+
+% determine default orientation based on the file name extension
+orient_by_extension = default_parameter_value('image_read','OrientByExtension');
+
+fcount_total = -1;
+
+% check minimum number of input arguments
+if (nargin < 2)
+    fprintf('\nUsage:\n');
+    fprintf('[bytes_written]=%s( filename, frame-structure [[,<name>,<value>]...]);\n',mfilename);
+    fprintf('Write a CBF file from data that have been read via cbfread.m before.\n');
+    fprintf('Conventions are likely to be specific for the PILATUS detector.\n')
+    fprintf('\n');
+    fprintf('The optional <name>,<value> pairs are:\n');
+    image_orient_help(mfilename,'ParametersOnly',1);
+    fprintf('\n');
+    fprintf('The file name should be the name of a single file without wildcards.\n');
+
+    error('At least the filename and the data-structure have to be specified as input parameter.');
+end
+
+% check frame structure
+if (~isstruct(frame))
+    error('The 2nd parameter needs to be a structure as read by cbfread.m');
+end
+if (~isfield(frame,'header'))
+    error('The 2nd parameter needs to be a structure with a field ''header'', as read by cbfread.m');
+end
+if (~isfield(frame,'data'))
+    error('The 2nd parameter needs to be a structure with a field ''data'', as read by cbfread.m');
+end
+if (size(frame.header,2) ~= 1)
+    error('The header has the wrong dimensions. Only single-frame structures are suppoerted.\n');
+end
+
+% check minimum number of input arguments
+if (nargin < 2)
+    error('At least the filename and the frame have to be specified as input parameter.');
+end
+
+% accept cell array with name/value pairs as well
+no_of_in_arg = nargin;
+if (nargin == 3)
+    if (isempty(varargin))
+        % ignore empty cell array
+        no_of_in_arg = no_of_in_arg -1;
+    else
+        if (iscell(varargin{1}))
+            % use a filled one given as first and only variable parameter
+            varargin = varargin{1};
+            no_of_in_arg = 2 + length(varargin);
+        end
+    end
+end
+
+% check number of input arguments
+if (rem(no_of_in_arg,2) ~= 0)
+    error('The optional parameters have to be specified as ''name'',value pairs');
+end
+    
+% set default values for the variable input arguments and parse the named
+% parameters: 
+vararg_remain = cell(4,1);
+for ind = 1:2:length(varargin)
+    name = varargin{ind};
+    value = varargin{ind+1};
+    switch name
+        case 'OrientByExtension' 
+            orient_by_extension = value;
+        otherwise
+            % pass further arguments on to fopen_until_exists
+            vararg_remain{end+1} = name; %#ok<AGROW>
+            vararg_remain{end+1} = value; %#ok<AGROW>
+    end
+end
+vararg_remain{1} = 'OrientByExtension';
+vararg_remain{2} = orient_by_extension;
+vararg_remain{3} = 'InvertOrientation';
+vararg_remain{4} = 1;
+
+% get the length of the zero-padding at the end of the file
+padding_length = get_hdr_val(frame.header{1},'X-Binary-Size-Padding:','%d',1);
+
+% end of header signature
+eoh_signature = char([ 12 26 4 213 ]);
+
+% orient image
+[frame,vararg_remain] = image_orient(frame,vararg_remain);
+
+
+% open file for write-access, overwrite in case of an existing file
+[fid] = fopen(filename,'w');
+if (fid < 0)
+    return;
+end
+
+
+% In byte-offset compression the difference to the previous pixel value is
+% stored as a byte, 16-bit integer or 32-bit integer, depending on its
+% size. 
+% The sizes above one byte are indicated by the escape sequence -1 in the
+% previous data format, i.e, a 32-bit integer is preceded by the sequence %
+% 0x80 (too large for a byte)
+% 0x8000 (too large for a 16-bit integer). 
+ind_out = 1;
+no_of_pixels = size(frame.data,1) * size(frame.data,2);
+
+% initialize output array at maximum size
+frame_out = zeros(no_of_pixels*4 + padding_length,1,'uint8');
+val_prev = 0;
+for ind_in = 1:no_of_pixels
+    val_diff = frame.data(ind_in) - val_prev;
+    if (abs(val_diff) < 128)
+        % write differences in the range from -127 to 127 directly as 8bit
+        % signed integer:
+        % manual complement to emulate the sign
+        if (val_diff < 0)
+            val_diff = 256 + val_diff;
+        end
+        frame_out(ind_out) = val_diff;
+        ind_out = ind_out +1;
+    else
+        % escape with 0x80
+        frame_out(ind_out) = 128;
+        % check for 16-bit integer value
+        if (abs(val_diff) < 32768)
+            % write signed 16bit integer value:
+            % manual complement to emulate the sign
+            if (val_diff < 0)
+                val_diff = 65536 + val_diff;
+            end
+            frame_out(ind_out+1) = bitand(val_diff,255);
+            frame_out(ind_out+2) = bitand(val_diff,65280)/256;
+            ind_out = ind_out +3;
+        else
+            % escape with 0x8000
+            frame_out(ind_out+1) = 0;
+            frame_out(ind_out+2) = 128;
+            % write signed 32bit integer value:
+            % manual complement to emulate the sign
+            if (val_diff < 0)
+                val_diff = 4294967296 + val_diff;
+            end
+            frame_out(ind_out+3) = bitand(val_diff,255);
+            frame_out(ind_out+4) = bitand(val_diff,65280)/256;
+            frame_out(ind_out+5) = bitand(val_diff,16711680)/65536;
+            frame_out(ind_out+6) = bitand(val_diff,4278190080)/16777216;
+            ind_out = ind_out +7;
+        end
+    end
+    % the current intensity value becomes the previous value
+    val_prev = frame.data(ind_in);
+end
+
+% calculate length of binary data including zero-padding at the end
+binary_length = ind_out -1 + padding_length;
+
+% update this value in the file-header
+[~, line_no] = get_hdr_val(frame.header{1},'X-Binary-Size:','%f',1);
+frame.header{1}{line_no} = sprintf('X-Binary-Size: %d',binary_length-padding_length);
+
+% write header
+[fcount_total] = fprintf(fid,'%s\r\n',frame.header{1}{:});
+if (fcount_total == 0)
+    error('Could not write any header data.\n');
+end
+% write end-of-header signature
+[fcount] = fwrite(fid,eoh_signature,'uint8');
+fcount_total = fcount_total + fcount;
+if (fcount == 0)
+    error('Could not write end-of-header signature.\n');
+end
+
+% write binary data
+[fcount] = fwrite(fid,frame_out(1:(binary_length)));
+fcount_total = fcount_total + fcount;
+if (fcount ~= binary_length)
+    error('Could not write CBF image-data.');
+end
+    
+% write binary-end signature
+[fcount] = fprintf(fid,'\r\n--CIF-BINARY-FORMAT-SECTION----\r\n;\r\n\r\n');
+fcount_total = fcount_total + fcount;
+if (fcount < 1)
+    error('Could not write CBF binary-end signature');
+end
+
+
+% close output data file
+fclose(fid);
+if (debug_level >= 2)
+    fprintf('%d data bytes written\n',fcount_total);
+end
+
+return;

+io/+HDF/add_content.m

@@ -0,0 +1,82 @@
+%ADD_CONTENT write matlab structure to H5 file
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function add_content(data, gid, plist, comp, overwrite)
+import io.HDF.*
+
+fn = fieldnames(data);
+for jj=1:length(fn)
+    if isstruct(data.(fn{jj}))
+        fn_sub = fieldnames(data.(fn{jj}));
+        if length(fn_sub) <= 2 && isfield(data.(fn{jj}), 'Value')
+            % found dataset
+            if isfield(data.(fn{jj}), 'Attributes')
+                Attributes.MATLAB_class = class(data.(fn{jj}).Value);
+                write_dataset(data.(fn{jj}).Value, gid, fn{jj}, plist, comp, overwrite, data.(fn{jj}).Attributes);
+            else
+                Attributes.MATLAB_class = class(data.(fn{jj}).Value);
+                write_dataset(data.(fn{jj}).Value, gid, fn{jj}, plist, comp, overwrite, Attributes);
+            end
+        elseif strcmpi(fn{jj}, 'Attributes')
+            % add attributes to group
+            fn_attr = fieldnames(data.Attributes);
+            for ii=1:length(fn_attr)
+                write_attribute(gid, data.Attributes.(fn_attr{ii}), fn_attr{ii});
+            end
+        else
+            % found group
+            gid_new = add_groups(gid, fn{jj}, plist, true);
+            if length(data.(fn{jj})) == 1
+                add_content(data.(fn{jj}), gid_new, plist, comp, overwrite)
+            else
+                fn_names = cell(1,length(data.(fn{jj})));
+                for ii=1:length(data.(fn{jj}))
+                    fn_names{ii} = sprintf([fn{jj} '_%d'],ii-1);
+                    gid_new_sub = add_groups(gid_new, fn_names{ii}, plist, true);
+                    add_content(data.(fn{jj})(ii), gid_new_sub, plist, comp, overwrite);
+                end
+                write_attribute(gid_new, 'structure array', 'MATLAB_class');
+
+            end
+        end
+    else
+        % append dataset
+        Attributes.MATLAB_class = class(data.(fn{jj}));
+        write_dataset(data.(fn{jj}), gid, fn{jj}, plist, comp, overwrite, Attributes);
+    end
+end
+        
+end

+io/+HDF/add_groups.m

@@ -0,0 +1,65 @@
+%ADD_GROUPS add groups or open them if they exist
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function gid = add_groups(fileID, gpath, plist, varargin)
+
+if nargin > 3
+    single_data = varargin{1};
+else
+    single_data = true;
+end
+
+if ~single_data
+    gpath_depth = length(gpath);
+    gid{1} = fileID;
+    for ii=1:gpath_depth
+        try
+            gid{end+1} = H5G.open(gid{ii}, gpath{ii}, plist);
+        catch
+            gid{end+1} = H5G.create(gid{ii},gpath{ii},plist,plist,plist);
+        end
+    end
+    
+else
+    try
+        gid = H5G.open(fileID, gpath, plist);
+    catch
+        gid = H5G.create(fileID,gpath,plist,plist,plist);
+    end
+end
+
+end
+

+io/+HDF/get_datatype.m

@@ -0,0 +1,84 @@
+% Determine datatype for HDF files
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [datatype_h5, data] = get_datatype(data)
+
+if isempty(data)
+    data = '';
+end
+
+switch class(data)
+    case 'uint32'
+        datatype_h5 = 'H5T_STD_U32LE';
+    case 'int32'
+        datatype_h5 = 'H5T_STD_I32LE';
+    case 'int64'
+        datatype_h5 = 'H5T_STD_I64LE';
+    case 'uint64'
+        datatype_h5 = 'H5T_STD_U64LE';
+    case 'double'
+        if isreal(data)
+            datatype_h5 = 'H5T_NATIVE_DOUBLE';
+        else
+            datatype_h5 = 'complex';
+        end
+    case 'single'
+        if isreal(data)
+            datatype_h5 = 'H5T_NATIVE_FLOAT';
+        else
+            datatype_h5 = 'complex';
+        end
+    case 'logical'
+        data = uint32(data);
+        datatype_h5 = 'H5T_STD_U32LE';
+    case 'char'
+        if size(data,1)>1
+            data = cellstr(data);
+            datatype_h5 = 'char_array';
+        else
+            datatype_h5 = 'H5T_UNIX_D32BE'; %'H5T_UNIX_D32LE';
+        end
+    case 'cell'
+        datatype_h5 = 'H5T_C_S1';
+    case 'struct'
+        datatype_h5 = 'struct';
+        
+    otherwise
+        error('Unknown data type %s', class(data))
+end
+
+end
+

+io/+HDF/hdf5_append_attr.m

@@ -0,0 +1,69 @@
+%HDF5_APPEND_ATTR Append an attribute to a dataset
+%
+% file...       HDF filename
+% attr...       structure of attributes
+% loc...        location of the dataset that needs to be removed
+% 
+%
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function hdf5_append_attr( file, attr, loc)
+import io.HDF.*
+plist = 'H5P_DEFAULT';
+
+fileID = H5F.open(file,'H5F_ACC_RDWR',plist);
+
+gpath = strsplit(rm_delimiter(loc), '/');
+
+gpath_depth = length(gpath);
+gid{1} = fileID;
+for ii=1:gpath_depth
+    try
+        gid{end+1} = H5G.open(gid{ii}, gpath{ii}, plist);
+    catch
+        gid{end+1} = H5D.open(gid{ii}, gpath{ii});
+    end
+end
+
+fn = fieldnames(attr);
+for ii=1:length(fn)
+    write_attribute(gid{end}, attr.(fn{ii}), fn{ii}, true);
+end
+    
+H5F.close(fileID);
+
+
+end

+io/+HDF/hdf5_attr_exists.m

@@ -0,0 +1,65 @@
+%HDF5_ATTR_EXISTS check if attribute exists in given file
+%   file...     h5 file path
+%   attr...     dataset name
+%   
+%   *optional*
+%   gpath...    path within the h5 file; default root (/)
+%
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [ out ] = hdf5_attr_exists(file, name, varargin)
+
+out = false;
+
+% load info
+if nargin > 2
+    h = h5info(file, varargin{1});
+else
+    h = h5info(file);
+end
+
+% loop through datasets and check if name exists
+for ii=1:numel(h.Attributes)
+    if strcmpi(h.Attributes(ii).Name, name)
+        out = true;
+        break
+    end
+end
+
+
+end
+
+

+io/+HDF/hdf5_cp_file.m

@@ -0,0 +1,132 @@
+%HDF5_CP_FILE copy HDF files
+%   orig_filename...        source file
+%   duplicate_filename...   target file
+%
+%   *optional*              given as name/value pair
+%   groups...               groups to copy; either string or cell of
+%                           strings; default: everything in root 
+%   copy_type...            'deep', 'normal' or 'shallow' copy; 
+%                           'shallow' creates external links in target file; 
+%                           'normal' is similar to linux 'cp' command; 
+%                           'deep' dereferences all internal and external links; 
+%                           default: 'shallow'
+%
+%   EXAMPLES:
+%       hdf5_cp_file('./test.h5', './test_new.h5')
+%       hdf5_cp_file('./test.h5', './test_new.h5', 'copy_type', 'deep');
+%   
+% 
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function hdf5_cp_file(orig_filename, duplicate_filename, varargin)
+import io.HDF.*
+% take care of input arguments
+groups = [];
+copy_type = 'shallow';
+
+% parse the variable input arguments vararg = cell(0,0);
+if ~isempty(varargin)
+    for ind = 1:2:length(varargin)
+        name = varargin{ind};
+        value = varargin{ind+1};
+        switch lower(name)
+            case 'groups'
+                groups = value;
+            case 'copy_type'
+                copy_type = value;
+        end
+    end
+end
+
+switch copy_type
+    case 'shallow'
+        if isempty(groups)
+            % if no groups are specified, use h5info to get all datasets and groups
+            % from root
+            h = h5info(orig_filename, '/');
+            lng = length(h.Groups);
+            lnd = length(h.Datasets);
+            lna = length(h.Attributes);
+            
+            groups = cell([1 lng+lnd]);
+            attributes = [];
+            
+            for ii=1:lng
+                groups{ii} = h.Groups(ii).Name;
+            end
+            for ii=1:lnd
+                groups{ii+lng} = h.Datasets(ii).Name;
+            end
+            for ii=1:lna
+                attributes.(h.Attributes(ii).Name) = h.Attributes(ii).Value;
+                if iscell(h.Attributes(ii).Value)
+                    attributes.(h.Attributes(ii).Name) = attributes.(h.Attributes(ii).Name){1};
+                end
+            end
+        else
+            attributes = [];
+        end
+        
+        
+        s = [];
+        if iscell(groups)
+            for ii=1:length(groups)
+                subgrps = strsplit(rm_delimiter(groups{ii}), '/');
+                s = setfield(s, subgrps{:}, ['ext:' orig_filename ':' groups{ii}]);
+            end
+        else
+            s.groups = ['ext:' orig_filename ':' groups];
+        end
+        
+        % append attributes
+        if ~isempty(attributes)
+            s.Attributes = attributes;
+        end
+        
+        save2hdf5(duplicate_filename, s, 'overwrite', true, 'iscopy', true);
+        
+    case 'deep'
+        s = io.HDF.hdf5_load(orig_filename, '-ca');
+        save2hdf5(duplicate_filename, s, 'overwrite', true, 'iscopy', true);
+        
+    case 'normal'
+        copyfile(orig_filename, duplicate_filename)
+        
+    otherwise
+        error('Unknown copy type!')
+end
+
+end

+io/+HDF/hdf5_dset_exists.m

@@ -0,0 +1,99 @@
+%HDF5_DSET_EXISTS check if dataset exists in given file
+%   file...         h5 file path
+%   dset...         dataset name
+%   
+%   *optional*
+%   gpath...        path within the h5 file; default root (/)
+%   check_links...  include links; default true
+%
+%   EXAMPLES:
+%       out = io.HDF.hdf5_dset_exists('./recons.h5',
+%       'object_phase_unwrapped', '/reconstruction', true);
+%
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [out] = hdf5_dset_exists(file, dset, varargin)
+
+out = false;
+
+% load info
+if nargin > 2
+    h = h5info(file, varargin{1});
+else
+    h = h5info(file);
+end
+
+if nargin > 3
+    check_links = varargin{2};
+else
+    check_links = true;
+end
+
+if nargin > 4
+    check_groups = varargin{3};
+else
+    check_groups = true;
+end
+
+% loop through datasets and check if name exists
+for ii=1:numel(h.Datasets)
+    if strcmpi(h.Datasets(ii).Name, dset)
+        out = true;
+        break
+    end
+end
+
+if check_links
+    for ii=1:numel(h.Links)
+        if strcmpi(h.Links(ii).Name, dset)
+            out = true;
+            break
+        end
+    end
+end
+
+if check_groups
+    for ii=1:numel(h.Groups)
+        [~, gname] = fileparts(h.Groups(ii).Name);
+        if strcmpi(gname, dset)
+            out = true;
+            break
+        end
+    end
+end
+
+end
+

+io/+HDF/hdf5_load.m

@@ -0,0 +1,642 @@
+% HDF5_LOAD Load an hdf5 file
+%
+% DATA = HDF5_LOAD(filename) reads a complete file hierarchy recursively, with
+% file name/path being specified by the 'filename' argument
+%
+% DATA = HDF5_LOAD(filename, '-a') reads a complete file hierarchy
+% recursively, including attributes
+%
+% DATA = HDF5_LOAD(filename, location) reads a particular group, link, or a single dataset
+% specified by the 'location' argument
+%
+% ATT = HDF5_LOAD(filename, location, '-a') reads all datasets and attributes associated
+% with a particular location in the file (group, link or dataset)
+%
+% ATT = HDF5_LOAD(filename, location, '-ca') reads all datasets and attributes associated
+% with a particular location in the file (group, link or dataset) and
+% converts datasets to a specific matlab class based on attribute 'MATLAB_class'
+%
+% SLICE = HDF5_LOAD(filename, location, {rowRange, colRange, frameRange, ...}) reads a
+% portion of a dataset along specified dimentions, where slicing ranges can be defined in
+% the following ways (negative indexes count from the end of the corresponding dimensions):
+%   range = scalar_index - reads a particular row/col/frame/... (indentical to
+%       'range = [scalar_index, scalar_index]')
+%   range = [start_index, end_index] - reads all data between start and end
+%       indexes
+%   range = [start_index, Inf] - reads all data from start_index to the last
+%       existing element in the file
+%   range = [], or range is omitted at the end - reads the full range of values for that
+%       dimention (indentical to 'range = [1, Inf]')
+%
+% Examples:
+%   hdf5_load('scan_003.hdf5')
+%   hdf5_load('scan_003.hdf5', '/entry/sample/description')
+%   hdf5_load('scan_003.hdf5', '/entry/collection/data/spec', '-a')
+%   hdf5_load('scan_003.hdf5', '/entry/instrument/Pilatus_2M/data', {5})
+%   hdf5_load('scan_003.hdf5', '/entry/instrument/Pilatus_2M/data', {[-100, Inf]})
+%   hdf5_load('scan_003.hdf5', '/entry/instrument/Pilatus_2M/data', {5, [500, Inf], [1, 100]})
+%   hdf5_load('scan_003.hdf5', '/entry/instrument/Pilatus_2M/data', {[], [], [1, 100]})
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a
+%   publication or if it is fully or partially rewritten for another
+%   computing language the authors and institution should be acknowledged
+%   in written form in the publication: “Data processing was carried out
+%   using the “cSAXS matlab package” developed by the CXS group
+%   and the Science IT group, Paul Scherrer Institut, Switzerland.”
+%   Variations on the latter text can be incorporated upon discussion with
+%   the CXS group if needed to more specifically reflect the use of the package
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%
+% This code and subroutines are part of a continuous development, they
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its
+%    proper use and the correctness of the results.
+
+function data = hdf5_load(filename, varargin)
+import io.HDF.*
+
+load_attr = false;
+convert2matlab = false;
+
+narginchk(1, 3);
+if nargin == 1
+    % Read the complete file hierarchy recursively
+    try
+        info = h5info(filename);
+        info.Name = ''; % a special case of the root group
+        
+    catch ME
+        if strcmp(ME.identifier, 'MATLAB:imagesci:h5info:fileOpenErr')
+            ME = MException('hdf5_load:h5info', ...
+                strjoin({'File', filename, 'does not exist'}));
+        end
+        
+        throwAsCaller(ME);
+    end
+    
+    [data, links] = hdf5_loadGroup(filename, info);
+    data = assign_links(data, info, links);
+    
+elseif nargin == 2
+    if any(strcmp(varargin{1}, {'-a', '-ca', '-c'}))
+        % second argument is an attribute flag
+        try
+            info = h5info(filename);
+            info.Name = ''; % a special case of the root group
+            
+        catch ME
+            if strcmp(ME.identifier, 'MATLAB:imagesci:h5info:fileOpenErr')
+                ME = MException('hdf5_load:h5info', ...
+                    strjoin({'File', filename, 'does not exist or is not valid h5 file'}));
+            end
+            
+            throwAsCaller(ME);
+        end
+        
+        if any(strcmp(varargin{1}, {'-a', '-ca'}))
+            load_attr = true;
+        end
+        
+        if any(strcmp(varargin{1}, {'-ca', '-c'}))
+            convert2matlab = true;
+        end
+        
+        [data, links] = hdf5_loadGroup(filename, info, convert2matlab, load_attr);
+        data = assign_links(data, info, links, varargin{1});
+        
+    else
+        % Read a group or a single dataset
+        location = varargin{1};
+        try
+            info = h5info(filename, location);
+            if strcmp(info.Name, '/') % a special case of the root group
+                info.Name = '';
+            end
+            
+        catch ME
+            if strcmp(ME.identifier, 'MATLAB:imagesci:h5info:fileOpenErr')
+                ME = MException('hdf5_load:h5info', ...
+                    strjoin({'File', filename, 'does not exist'}));
+                
+            elseif strcmp(ME.identifier, 'MATLAB:imagesci:h5info:libraryError')
+                ME = MException('hdf5_load:h5info', ...
+                    strjoin({'H5Location', location, 'was not found in', filename, 'file'}));
+            end
+            
+            throwAsCaller(ME);
+        end
+        
+        if isfield(info, 'Groups')
+            % Read a group with its internal hierarchy
+            [data, links] = hdf5_loadGroup(filename, info);
+            data = assign_links(data, info, links);
+            
+        elseif isfield(info, 'Datatype')
+            % Read a data set
+            type = info.Datatype.Class;
+            data = hdf5_loadDataset(filename, location, type);
+            
+        elseif isfield(info, 'Type')
+            % Read a link
+            data = hdf5_loadLink(info);
+            
+        else
+            error('hdf5_load:parse_argument', ...
+                'The 2-nd argument must be a name of a group, dataset, or link');
+        end
+    end
+    
+elseif nargin == 3
+    % Read attributes of a group or a data set, or slices of a data set
+    location = varargin{1};
+    try
+        info = h5info(filename, location);
+        if strcmp(info.Name, '/') % a special case of the root group
+            info.Name = '';
+        end
+        
+    catch ME
+        if strcmp(ME.identifier, 'MATLAB:imagesci:h5info:fileOpenErr')
+            ME = MException('hdf5_load:h5info', ...
+                strjoin({'File', filename, 'does not exist or is not HDF5 format'}));
+            
+        elseif strcmp(ME.identifier, 'MATLAB:imagesci:h5info:libraryError')
+            ME = MException('hdf5_load:h5info', ...
+                strjoin({'H5Location', location, 'was not found in', filename, 'file'}));
+        end
+        
+        throwAsCaller(ME);
+    end
+    
+    if iscell(varargin{2})
+        % Read slices of a data set
+        slices = varargin{2};
+        
+        % Check if the specified location is a data set
+        if ~isfield(info, 'Dataspace')
+            error('hdf5_load:invalid_location', ...
+                'Slicing ranges are not applicable, the location is not a data set');
+        end
+        
+        data_size = info.Dataspace.Size;
+        if length(slices) > length(data_size)
+            error('hdf5_load:invalid_slicing', ...
+                'A number of slicing ranges is larger than a dimention of a data set')
+        end
+        
+        % Parse ranges
+        startIndex = ones(1, length(data_size));
+        nElements = Inf(1, length(data_size));
+        for i = 1:length(slices)
+            [startIndex(i), nElements(i)] = parse_range(slices{i}, data_size(i));
+        end
+        
+        % Read data
+        data = h5read(filename, location, startIndex, nElements);
+        
+    elseif any(strcmp(varargin{2}, {'-a', '-ca', '-c'}))
+        % Read attributes and/or convert to matlab structures
+        if any(strcmp(varargin{2}, {'-a', '-ca'}))
+            load_attr = true;
+        end
+        if any(strcmp(varargin{2}, {'-ca', '-c'}))
+            convert2matlab = true;
+        end
+        
+        if isfield(info, 'Groups')
+            % Read a group with its internal hierarchy
+            [data, links] = hdf5_loadGroup(filename, info, convert2matlab, load_attr);
+            data = assign_links(data, info, links, varargin{2});
+            
+        elseif isfield(info, 'Datatype')
+            % Read a data set
+            type = info.Datatype.Class;
+            dset_val = hdf5_loadDataset(filename, location, type);
+            if load_attr || convert2matlab
+                [dset_attr, ml_class_dset] = hdf5_loadAttributes(info, convert2matlab, load_attr);
+            else
+                ml_class_dset = [];
+            end
+            
+            if ~isempty(ml_class_dset)
+                switch ml_class_dset
+                    case 'complex'
+                        dset_val = dset_val.r + 1i*dset_val.i;
+                        
+                    case 'cell'
+                        if ~iscell(dset_val)
+                            dset_val = {dset_val};
+                        end
+                        
+                    case 'char_array'
+                        dset_val = char(dset_val);
+                        
+                        
+                    otherwise
+                        conv2ml = str2func(ml_class_dset);
+                        dset_val = conv2ml(dset_val);
+                end
+            end
+            
+            if load_attr
+                data.Attributes = dset_attr;
+                data.Value = dset_val;
+                
+            else
+                data = dset_val;
+            end
+            
+        elseif isfield(info, 'Type')
+            % Read a link
+            data = hdf5_loadLink(info, convert2matlab, load_attr);
+            
+        end
+        
+    else
+        error('hdf5_load:parse_argument', ...
+            'Incorrect 3-rd argument');
+    end
+end
+
+function [data, links] = hdf5_loadGroup(filename, info, varargin)
+import io.HDF.*
+
+if nargin > 2
+    convert2matlab = varargin{1};
+    load_attr = varargin{2};
+else
+    convert2matlab = false;
+    load_attr = false;
+end
+
+data = [];
+
+% Collect links
+links = info.Links.'; % transform to a row for easier indexing
+if ~isempty(links)
+    for link_ind = 1:length(links)
+        links(link_ind).Name = [info.Name, '/', links(link_ind).Name];
+    end
+end
+
+% Load the datasets
+for dataset_ind = 1:length(info.Datasets)
+    dset_info = info.Datasets(dataset_ind);
+    dset_name = dset_info.Name;
+    location = [info.Name, '/', dset_name];
+    type = dset_info.Datatype.Class;
+    
+    dset_val = hdf5_loadDataset(filename, location, type);
+    
+    % Load attributes of a dataset
+    if load_attr || convert2matlab
+        [dset_attr, ml_class_dset] = hdf5_loadAttributes(dset_info, convert2matlab, load_attr);
+    else
+        ml_class_dset = [];
+    end
+    
+    if ~isempty(ml_class_dset)
+        switch ml_class_dset
+            case 'complex'
+                dset_val = dset_val.r + 1i*dset_val.i;
+                
+            case 'cell'
+                if ~iscell(dset_val)
+                    dset_val = {dset_val};
+                end
+                
+            case 'char_array'
+                dset_val = char(dset_val);
+                
+            otherwise
+                conv2ml = str2func(ml_class_dset);
+                dset_val = conv2ml(dset_val);
+        end
+    end
+    
+    if load_attr
+        data.(dset_name).Attributes = dset_attr;
+        data.(dset_name).Value = dset_val;
+        
+    else
+        data.(dset_name) = dset_val;
+    end
+end
+
+% Load attributes of a group
+if load_attr || convert2matlab
+    [group_attr, ml_class_group] = hdf5_loadAttributes(info, convert2matlab, load_attr);
+    if load_attr
+        data.Attributes = group_attr;
+    end
+else
+    ml_class_group = [];
+end
+
+% Load the internal groups recursively
+for group_ind = 1:length(info.Groups)
+    [group_data, child_links] = hdf5_loadGroup(filename, info.Groups(group_ind), convert2matlab, load_attr);
+    
+    [~, group_name] = fileparts(info.Groups(group_ind).Name);
+    data.(group_name) = group_data;
+    
+    % Aggregate links
+    links = [links, child_links]; %#ok<AGROW> There shouldn't be too many links present
+end
+
+if ~isempty(ml_class_group)
+    % convert the groups
+    data_temp = data;
+    data = [];
+    if isfield(data_temp, 'Attributes')
+        data.Attributes = data_temp.Attributes;
+        data_temp = rmfield(data_temp, 'Attributes');
+        fn = fieldnames(data_temp);
+        for group_ind  = 1:length(fn)
+            switch ml_class_group
+                case 'cell'
+                    data.Value{group_ind} = data_temp.(fn{group_ind});
+                    
+                case 'structure array'
+                    data.Value(group_ind) = data_temp.(fn{group_ind});
+                    
+                otherwise
+                    keyboard
+            end
+        end
+    else
+        fn = fieldnames(data_temp);
+        for group_ind  = 1:length(fn)
+            switch ml_class_group
+                case 'cell'
+                    data{group_ind} = data_temp.(fn{group_ind});
+                    
+                case 'structure array'
+                    data(group_ind) = data_temp.(fn{group_ind});
+                    
+                otherwise
+                    keyboard
+            end
+        end
+    end
+    
+end
+
+function [data, ml_class] = hdf5_loadAttributes(info, convert2matlab, load_attr)
+data = [];
+ml_class = [];
+if isfield(info, 'Attributes') % info structure may not contain Attributes field
+    attr_info = info.Attributes;
+    for attr_ind = 1:length(attr_info)
+        attr = attr_info(attr_ind);
+        attr_name = attr.Name;
+        if ~isvarname(attr_name)
+            if ~any(strcmpi({attr_info.Name}, ['MATLAB' attr_name])) && ~strcmpi(attr_name, '_class')
+                warning('Invalid attribute name! Added "MATLAB" prefix to %s.', attr_name)
+                attr_name = ['MATLAB' attr_name];
+            else
+                error('Invalid attribute name.')
+            end
+        end
+        if convert2matlab && strcmpi(attr_name, 'MATLAB_class')
+            ml_class = attr.Value{1};
+            
+        elseif load_attr
+            if iscell(attr.Value)
+                data.(attr_name) = attr.Value{1};
+            else
+                data.(attr_name) = attr.Value;
+            end
+        end
+    end
+end
+
+function data = hdf5_loadDataset(filename, location, type)
+if strcmp(type, 'H5T_ENUM')
+    % Workaround for a bug in h5postprocessenums (part of h5read) function
+    data = read_enum(filename, location);
+    
+else
+    data = h5read(filename, location);
+    if iscell(data) && numel(data) == 1 && ischar(data{1})
+        data = data{1}; % utility string unwrapping from a single cell
+    end
+end
+
+function data = hdf5_loadLink(link, varargin)
+
+if nargin > 2
+    convert2matlab = varargin{1};
+    load_attr = varargin{2};
+else
+    convert2matlab = false;
+    load_attr = false;
+end
+
+switch link.Type
+    case {'hard link', 'soft link'}
+        filename = link.Filename;
+        location = link.Value{1};
+        
+    case 'external link'
+        filename = absolute_path(link.Value{1}, link.Filename);
+        location = link.Value{2};
+        
+    otherwise
+        error('hdf5_load:hdf5_loadLink', ...
+            strjoin({'Unknown link type at', link.Name}));
+end
+
+link_info = h5info(filename, location);
+if strcmp(link_info.Name, '/') % a special case of the root group
+    link_info.Name = '';
+end
+
+if isfield(link_info, 'Groups')
+    [data, links] = hdf5_loadGroup(filename, link_info, convert2matlab, load_attr);
+    data = assign_links(data, link_info, links);
+    
+elseif isfield(link_info, 'Datatype')
+    type = link_info.Datatype.Class;
+    data = hdf5_loadDataset(filename, location, type);
+    
+elseif isfield(link_info, 'Type')
+    data = hdf5_loadLink(link_info, convert2matlab, load_attr);
+    
+else
+    error('hdf5_load:hdf5_loadLink', ...
+        strjoin({'A link at', link_info.Name, 'must be a name of a group, dataset, or link'}));
+end
+
+function data = assign_links(data, info, links, varargin)
+import io.HDF.*
+
+if ~isempty(varargin)
+    flag = varargin{1};
+else
+    flag = [];
+end
+if ~isempty(links)
+    cut_start = length(info.Name) + 1;
+    
+    while true
+        resolved_links = false(size(links));
+        
+        for ind = 1:length(links)
+            link = links(ind);
+            place = strrep(link.Name(cut_start:end), '/', '.');
+            target = [];
+            target_struc = [];
+            if ~isempty(flag) && contains(flag, 'a') && contains(flag, 'c')
+                target_struc = ['.Value'];
+            end
+            switch link.Type
+                case {'hard link', 'soft link'}
+                    try
+                        parent = strsplit(link.Value{1}, '/');
+                        parent = strjoin(parent(1:end-1), '/');
+                        parent_info = h5info(info.Filename, parent);
+                        if isfield(parent_info, 'Attributes') && ~isempty(parent_info.Attributes)
+                            for ii=1:numel(parent_info.Attributes)
+                                if strcmp(parent_info.Attributes(ii).Name, 'MATLAB_class') && ~isempty(flag) && contains(flag, 'c')
+                                    % get pointer index
+                                    pnt_indx = strsplit(link.Value{1}, '_');
+                                    pnt_indx = str2double(pnt_indx(end));
+                                    target_add = [];
+                                    switch parent_info.Attributes(ii).Value{1}
+                                        case 'cell'
+                                            target_add = sprintf('{%d}', pnt_indx+1);
+                                            
+                                        case 'structure array'
+                                            target_add = sprintf('(%d)', pnt_indx+1);
+                                            
+                                        otherwise
+                                            keyboard
+                                    end
+                                    target = [strrep(parent, '/', '.') target_struc target_add];
+                                    break
+                                end
+                            end
+                        end
+                        if isempty(target)
+                            target = [strrep(link.Value{1}(cut_start:end), '/', '.') target_struc];
+                        end
+                        
+                        evalc(['data', place, ' = data', target]);
+                        
+                    catch
+                        continue % postpone this link resolution
+                    end
+                    
+                case 'external link'
+                    ext_link = absolute_path(link.Value{1}, info.Filename);
+                    
+                    % make sure to reference the same variable in evalc!
+                    if ~isempty(flag)
+                        target_data = hdf5_load(ext_link, link.Value{2}, flag); %#ok<NASGU>
+                    else
+                        target_data = hdf5_load(ext_link, link.Value{2});
+                    end
+                    evalc(['data', place, ' = target_data']);
+                    
+                otherwise
+                    error('hdf5_load:assign_links', ...
+                        strjoin({'Unknown link type at', place}));
+            end
+            
+            resolved_links(ind) = true;
+        end
+        
+        if all(resolved_links)
+            % all links have been assigned
+            return
+        end
+        
+        if ~any(resolved_links)
+            % none of the links has been assigned in this iteration
+            error('hdf5_load:assign_links', ...
+                strjoin({'Cannot assign link(s) at', ''}));
+        end
+        
+        links = links(~resolved_links);
+    end
+end
+
+function filepath = absolute_path(filepath, current_filepath)
+if ~startsWith(filepath, '/')
+    path = fileparts(current_filepath);
+    filepath = fullfile(path, filepath);
+end
+
+function [startVal, nVals] = parse_range(valRange, maxVal)
+if isempty(valRange) % empty
+    startVal = 1;
+    nVals = Inf;
+    
+elseif isscalar(valRange) % single value
+    if valRange <= -1
+        valRange = maxVal + valRange + 1;
+    end
+    startVal = valRange;
+    nVals = 1;
+    
+elseif isvector(valRange) && numel(valRange) == 2 % vector with two values
+    if valRange(1) <= -1
+        if isinf(valRange(1))
+            valRange(1) = 1; % = -Inf
+        else
+            valRange(1) = maxVal + valRange(1) + 1;
+        end
+    end
+    startVal = valRange(1);
+    
+    if valRange(2) <= -1
+        if isinf(valRange(2))
+            valRange(2) = 1; % = -Inf
+        else
+            valRange(2) = maxVal + valRange(2) + 1;
+        end
+    end
+    nVals = valRange(2) - startVal + 1;
+    
+else
+    error('hdf5_load:parse_range', ...
+        'A range should be specified with <= 2 parameters');
+end
+
+if startVal < 1 || startVal > maxVal || nVals < 1 || (nVals > maxVal && ~isinf(nVals))
+    error('hdf5_load:parse_range', ...
+        'The resulting range is out of data borders');
+end
+
+function data = read_enum(filename, location)
+file_id = H5F.open(filename);
+dset_id = H5D.open(file_id, location);
+type_id = H5D.get_type(dset_id);
+
+data = H5D.read(dset_id); % numerical member of enumeration
+data = H5T.enum_nameof(type_id, data); % associated symbol name
+
+H5T.close(type_id);
+H5D.close(dset_id);
+H5F.close(file_id);
+

+io/+HDF/hdf5_mv_data.m

@@ -0,0 +1,80 @@
+%HDF5_MV_DATA Move data within an HDF5 file
+% hdf5_mv_data creates a new (UNIX-like) hard link at loc_dest to the dataset at
+% loc_origin and deletes the hard link to the dataset at loc_origin.
+%
+% file...       HDF filename
+% loc_origin... location of the data that needs to be moved
+% loc_dest...   destination and name of the new data
+%
+% EXAMPLE:
+%   % move dataset probe from root to group measurements
+%   hdf5_mv_data('./awesome_file.h5', 'probe', 'measurements/probe')
+%
+%   Please notice that all groups and datasets have to exist before running
+%   the script!
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+function hdf5_mv_data( file, loc_origin, loc_dest)
+import io.HDF.*
+plist = 'H5P_DEFAULT';
+
+fileID = H5F.open(file,'H5F_ACC_RDWR',plist);
+
+gpath1 = strsplit(rm_delimiter(loc_origin), '/');
+gid1 = add_groups(fileID, gpath1(1:end-1), plist, false);
+
+gpath2 = strsplit(rm_delimiter(loc_dest), '/');
+gid2 = add_groups(fileID, gpath2(1:end-1), plist, false);
+
+try
+    datasetID = H5D.open(gid1{end}, gpath1{end});
+    dataset = true;
+catch
+    gid1 = add_groups(fileID,gpath1,plist, false);
+    datasetID = gid1{end};
+    dataset = false;
+end
+
+H5O.link(datasetID,gid2{end},gpath2{end},plist,plist);
+if dataset
+    H5L.delete(gid1{end}, gpath1{end}, plist);
+else
+    H5L.delete(gid1{end-1}, gpath1{end}, plist);
+end
+
+H5F.close(fileID);
+
+
+end

+io/+HDF/hdf5_rm_attr.m

@@ -0,0 +1,65 @@
+%HDF5_RM_ATTR Delete attribute(s) from HDF file
+%
+%   file...         HDF filename 
+%   loc...          location within the HDF file
+%   attr_name...    string or cell of strings containing the names of the
+%                   obsolete attributes
+%
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function hdf5_rm_attr( file, loc, attr_name)
+import io.HDF.*
+
+plist = 'H5P_DEFAULT';
+% open file
+fileID = H5F.open(file,'H5F_ACC_RDWR',plist);
+
+% get group ID
+gpath = strsplit(rm_delimiter(loc), '/');
+gid = add_groups(fileID, gpath(1:end-1), plist, false);
+
+% delete attributes
+if iscell(attr_name)
+    for ii=1:length(attr_name)
+        H5A.delete(gid{end}, attr_name{ii})
+    end
+else
+    H5A.delete(gid{end}, attr_name)
+end
+
+
+end
+

+io/+HDF/hdf5_rm_data.m

@@ -0,0 +1,57 @@
+%HDF5_RM_DATA Delete a dataset within an HDF5 file
+%
+% file...       HDF filename
+% loc...        location of the dataset that needs to be removed
+% 
+% Please notice that HDF5 does not free the space after removing datasets!
+%
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function hdf5_rm_data( file, loc)
+import io.HDF.*
+plist = 'H5P_DEFAULT';
+
+fileID = H5F.open(file,'H5F_ACC_RDWR',plist);
+
+gpath1 = strsplit(rm_delimiter(loc), '/');
+gid1 = add_groups(fileID, gpath1(1:end-1), plist, false);
+
+H5L.delete(gid1{end}, gpath1{end}, plist);
+
+H5F.close(fileID);
+
+
+end

+io/+HDF/hdf5read_main.m

@@ -0,0 +1,143 @@
+% Call function without arguments for instructions on how to use it
+
+% Filename: $RCSfile: hdf5read_main.m,v $
+%
+% $Revision: 1.1 $  $Date: 2010/10/02 07:58:50 $
+% $Author:  $
+% $Tag: $
+%
+% Description:
+% Macro for reading HDF5 files written for example by the EIGER server
+% program cbd_server
+%
+% Note:
+% So far this is mainly a place holder for a thorough implementation. 
+%
+% Dependencies:
+% - image_read_set_default
+% - fopen_until_exists
+% - get_hdr_val
+%
+%
+% history:
+%
+% September 30th 2010: 1st version
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [frame,vararg_remain] = hdf5read_main(filename,varargin)
+import io.HDF.*
+import io.image_read
+import utils.fopen_until_exists
+import utils.get_hdr_val
+
+% 0: no debug information
+% 1: some feedback
+% 2: a lot of information
+debug_level = 0;
+
+% initialize return argument
+frame = struct('header',[], 'data',[]);
+
+
+% check minimum number of input arguments
+if (nargin < 1)
+    image_read_sub_help(mfilename,'h5');
+    error('At least the filename has to be specified as input parameter.');
+end
+
+% accept cell array with name/value pairs as well
+no_of_in_arg = nargin;
+if (nargin == 2)
+    if (isempty(varargin))
+        % ignore empty cell array
+        no_of_in_arg = no_of_in_arg -1;
+    else
+        if (iscell(varargin{1}))
+            % use a filled one given as first and only variable parameter
+            varargin = varargin{1};
+            no_of_in_arg = 1 + length(varargin);
+        end
+    end
+end
+
+% check number of input arguments
+if (rem(no_of_in_arg,2) ~= 1)
+    error('The optional parameters have to be specified as ''name'',value pairs');
+end
+    
+% set default values for the variable input arguments and parse the named
+% parameters: 
+vararg = cell(0,0);
+for ind = 1:2:length(varargin)
+    name = varargin{ind};
+    value = varargin{ind+1};
+    switch name
+        otherwise
+            % pass further arguments on to fopen_until_exists
+            vararg{end+1} = name;
+            vararg{end+1} = value;
+    end
+end
+
+
+% try to open the data file
+if (debug_level >= 1)
+    fprintf('Opening %s.\n',filename);
+end
+[fid,vararg_remain] = fopen_until_exists(filename,vararg);
+if (fid < 0)
+    return;
+end
+% close input data file
+fclose(fid);
+
+% get file header
+hdr = hdf5info(filename);
+
+% store part of the file header in the return argument
+frame.header = {};
+frame.header{end+1} = 'Exposure_time 1.0';
+% add the file modification date to the header
+dir_entry = dir(filename);
+frame.header{end+1} = [ 'DateTime ' dir_entry.date ];
+
+% read all data of first data set at once
+frame.data = hdf5read(hdr.GroupHierarchy(1).Groups(1).Datasets(1));
+
+if (debug_level >= 2)
+    fprintf('%dx%dx%dx%s data bytes read\n',...
+        size(fdat,1),size(fdat,2),size(fdat,3),size(fdat,4));
+end

+io/+HDF/private/rm_delimiter.m

@@ -0,0 +1,43 @@
+%RM_DELIMITER makes sure that the path does not start with /
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function path = rm_delimiter(path)
+% make sure that the path does not start with /
+if strcmp(path(1), '/')
+    path = path(2:end);
+end
+end
+

+io/+HDF/save2hdf5.m

@@ -0,0 +1,242 @@
+%SAVE2HDF5 saves matlab data to a Hierarchical Data Format file (hdf5)
+%   
+%   filename...     full path to file, including file extension
+%   data...         matlab structure or array or link
+%   data_name...    needed if input data is not a matlab structure, needs
+%                   to be given as name/value pair
+%
+%   *optional*
+%   overwrite...    replace existing file if it exists
+%   gpath...        specify the group to which you want to append the data
+%                   (only if data is an array); default root ('/')
+%   Attributes...   structure of attributes; will be appended to current
+%                   gpath
+%   comp...         compression level; default 0 (no compression)
+%   creator...      attribute in root; default 'ptycho_recons'
+%   
+%
+%   If you want to save a structure, everything declared within an 'Attributes'
+%   fieldname will be treated as an attribute to the current group.
+%   If you want to add attributes to a dataset, you have to define your
+%   data within .Value and your attributes within .Attributes.
+%
+%   A simple structure could look like:
+%       h5_struc = [];
+%       h5_struc.probe_mask = ones(256,256);
+%       h5_struc.Attributes.probe_id = 1;
+%       h5_struc.measurement.n0.diff = fmag(:,:,1);
+%       h5_struc.measurement.n0.Attributes.detector = 0;
+%       h5_struc.measurement.n1.diff.Value = fmag(:,:,2);
+%       h5_struc.measurement.n1.diff.Attributes.slice = 2;
+%
+%   fmag(:,:,1) will be written to dataset 'diff' in group '/measurement/n0' 
+%   fmag(:,:,2) with attribute 'slice' will be written to dataset 'diff' in 
+%   group '/measurement/n1' 
+%
+%
+%   EXAMPLES:
+%     -) if data is a matlab structure:
+%           save2hdf5('./awesome_file.h5', data);
+%           save2hdf5('./awesome_file.h5', data, 'overwrite', true);
+%                    
+%
+%     -) if data is a matlab array:
+%           save2hdf5('./awesome_file.h5', data, 'data_name', data_name);
+%           save2hdf5('./awesome_file.h5', data, 'data_name', 'my_dataset',...
+%                     'gpath', 'group1/group2', 'Attributes', attr_struc);
+%
+%     -) if data is a link:
+%           currently, only external links ('ext') and internal soft links
+%           ('int_soft') are supported
+%
+%           external links have to be specified by a single string with 
+%           3 sections: '<link_type>:<file_path>:<target_object>'
+%               
+%               e.g.: 'ext:./awesome_file2.h5:/data' 
+%                   save2hdf5('./awesome_file.h5',...
+%                     'ext:./awesome_file2.h5:/data', 'data_name', data_name)
+%               
+%               will create a link called $data_name to dataset (or group) '/data' 
+%               in './awesome_file2.h5'
+%
+%           internal links have to be specified by a single string with
+%           2 sections: '<link_type>:<target_object>'
+%
+%               e.g.: 'int_soft:/data' 
+%                   save2hdf5('./awesome_file.h5',...
+%                     'int_soft:/data', 'data_name', data_name, 'gpath', 'g1/g2')
+%               
+%               will create a link called $data_name to dataset (or group) '/data' 
+%               in '/g1/g2'
+%
+%
+%    Please notice that structures are not supported as attributes, i.e.
+%       h5_struc = [];
+%       h5_struc.attr.probe.probe_id = 1;
+%
+%       save2hdf5('./awesome_file.h5', h5_struc)
+%
+%    will crash! 
+%
+%
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function save2hdf5( filename, data, varargin)
+import io.HDF.*
+
+% take care of input arguments
+overwrite = false;
+gpath_full = '';
+attr = [];
+data_name = '';
+comp = 0;
+creator = 'ptycho_recons';
+iscopy = false;
+extend_dim = 0;
+extendable = false;
+extend_offset = 0;
+extend_maxdims = 0;
+
+vararg = cell(0,0);
+% parse the variable input arguments vararg = cell(0,0);
+if ~isempty(varargin)
+    for ind = 1:2:length(varargin)
+        name = varargin{ind};
+        value = varargin{ind+1};
+        switch lower(name)
+            case 'data_name'
+                data_name = value;
+            case 'overwrite'
+                overwrite = value;
+            case 'gpath'
+                gpath_full = value;
+            case 'attr'
+                attr = value;
+            case 'comp'
+                comp = value;
+            case 'creator'
+                creator = value;
+            case 'iscopy'
+                iscopy = value;
+            case 'extend_dim'
+                extend_dim = value;
+            case 'extendable'
+                extendable = value;
+            case 'extend_offset'
+                extend_offset = value;
+            case 'extend_maxdims'
+                extend_maxdims = value;
+                
+            otherwise
+                vararg{end+1} = name;
+                vararg{end+1} = value;
+        end
+    end
+end
+
+if ~isstruct(data)
+    full_data = false;
+else
+    full_data = true;
+end
+
+if ~isstruct(data) && isempty(data_name)
+    data_name = inputname(2);
+    if isempty(data_name)
+        error('Please specify the data_name.')
+    end
+end
+
+if extendable && extend_dim
+    error('Extending the dimension of an unlimited dataset is currently not supported.');
+end
+
+plist = 'H5P_DEFAULT';
+
+%%% create file if it does not exist
+if exist(filename, 'file')&&~overwrite
+    fileID = H5F.open(filename,'H5F_ACC_RDWR',plist);
+else
+    fileID = H5F.create(filename,'H5F_ACC_TRUNC','H5P_DEFAULT','H5P_DEFAULT');
+    if ~iscopy
+        write_attribute(fileID, filename, 'filename');
+        write_attribute(fileID, creator,'creator');
+        write_attribute(fileID, datestr(now),'file_time');
+    end
+end
+
+if full_data
+    
+    %%%%%%%%%%%%%%%%%%%%%%%%%
+    %%% data as structure %%%
+    %%%%%%%%%%%%%%%%%%%%%%%%%
+    
+    add_content(data, fileID, plist, comp, overwrite)
+       
+    
+else
+    
+    %%%%%%%%%%%%%%%%%%%%%
+    %%% data as array %%%
+    %%%%%%%%%%%%%%%%%%%%%
+
+    % prepare group handles
+    if ~isempty(gpath_full)
+        gpath = strsplit(rm_delimiter(gpath_full), '/');
+        gid = add_groups(fileID, gpath, plist, false);
+    else
+        gid{1} = fileID;
+    end
+    
+    % write data to file
+    write_dataset(data, gid{end}, data_name, plist, comp, overwrite, [], extend_dim, extendable, extend_offset, extend_maxdims);
+    
+    % append attributes
+    if ~isempty(attr)
+        attr_fn = fieldnames(attr);
+        for ii=1:length(attr_fn)
+            write_attribute(gid{end}, attr.(attr_fn{ii}), attr_fn{ii}, true);
+        end
+    end
+    
+end
+
+% close handles
+H5F.close(fileID);
+
+end
+
+

+io/+HDF/write_attribute.m

@@ -0,0 +1,119 @@
+%WRITE_ATTRIBUTE write attribute data_name with value data to ID gid
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function write_attribute(gid, data, data_name, varargin)
+import io.HDF.*
+
+if nargin > 3
+    safe = varargin{1};
+else
+    safe = false;
+end
+
+[datatypeID, data] = get_datatype(data);
+if ischar(data)
+    % The ptycho C++ code expects strings as H5S_SCALAR, so we have to
+    % convert it
+    data = {data};
+    filetype = H5T.copy ('H5T_FORTRAN_S1');
+    H5T.set_size (filetype,'H5T_VARIABLE');
+    memtype = H5T.copy ('H5T_C_S1');
+    H5T.set_size (memtype, 'H5T_VARIABLE');
+    space = H5S.create ('H5S_SCALAR');
+    if safe
+        try
+            attr = H5A.create (gid, data_name, filetype, space, 'H5P_DEFAULT');
+        catch
+            H5A.delete(gid, data_name);
+            attr = H5A.create (gid, data_name, filetype, space, 'H5P_DEFAULT');
+        end
+    else
+        attr = H5A.create (gid, data_name, filetype, space, 'H5P_DEFAULT');
+    end
+
+    H5A.write (attr, memtype, data);
+    
+elseif iscell(data)
+    % If it is a cell, save it as 1D dataset 
+    H5T.set_size(datatypeID,'H5T_VARIABLE');    
+    agcv = H5ML.get_constant_value('H5S_UNLIMITED');
+    dspace = H5S.create_simple(1,numel(data),agcv);
+    
+    plist = H5P.create('H5P_ATTRIBUTE_CREATE');
+    if safe
+        try
+            attr = H5A.create(gid,data_name,datatypeID,dspace,plist);
+        catch
+            H5A.delete(gid, data_name);
+            attr = H5A.create(gid,data_name,datatypeID,dspace,plist);
+        end
+    else
+        attr = H5A.create(gid,data_name,datatypeID,dspace,plist);
+    end
+    H5A.write(attr,'H5ML_DEFAULT',data);
+    
+else 
+    
+    acpl = H5P.create('H5P_ATTRIBUTE_CREATE');
+    dims = size(data);    
+    if length(dims)>1 && dims(2)~=1
+        if dims(1) == 1
+            space_id = H5S.create_simple(dims(1), dims(2), []);
+        else
+            space_id = H5S.create_simple(dims(1), dims, []);
+        end
+    else
+        space_id = H5S.create('H5S_SCALAR');
+    end    
+    if safe
+        try
+            attr = H5A.create(gid,data_name,datatypeID,space_id,acpl);
+        catch
+            H5A.delete(gid, data_name);
+            attr = H5A.create(gid,data_name,datatypeID,space_id,acpl);
+        end
+    else
+        attr = H5A.create(gid,data_name,datatypeID,space_id,acpl);
+    end
+
+
+    H5A.write(attr,'H5ML_DEFAULT',data)
+    
+
+end
+H5A.close(attr);
+end
+

+io/+HDF/write_dataset.m

@@ -0,0 +1,370 @@
+%WRITE_DATASET write dataset data_name, containing data to ID gid
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function write_dataset(data, gid, data_name, plist, varargin)
+import io.HDF.*
+
+extend_data = false;
+link = false;
+link_type = '';
+cellstrdata = false;
+write_data = true;
+
+if ~isempty(varargin)
+    comp = varargin{1};
+else
+    comp = true;
+end
+if nargin > 5
+    overwrite = varargin{2};
+else
+    overwrite = true;
+end
+
+if nargin > 6
+    data_attr = varargin{3};
+else
+    data_attr = [];
+end
+
+if nargin > 7
+    extend_dim = varargin{4};
+else
+    extend_dim = 0;
+end
+
+if nargin > 8
+    extendable = varargin{5};
+else
+    extendable = false;
+end
+
+if nargin > 9
+    extend_offset = varargin{6};
+else
+    extend_offset = 0;
+end
+
+if nargin > 10
+    extend_maxdims = varargin{7};
+else
+    extend_maxdims = 0;
+end
+
+[datatype, data] = get_datatype(data);
+filespaceID = [];
+
+    function create_dataspace()
+        
+        if extendable
+            unlimited = H5ML.get_constant_value('H5S_UNLIMITED');
+            dims_max = repmat(unlimited, 1, numel(dims));
+        else
+            dims_max = dims;
+        end
+        
+        if ~extend_dim
+            if ~extendable
+                dataspaceID = H5S.create_simple(length(dims), fliplr(dims), fliplr(dims_max));
+            else
+                try 
+                    datasetID = H5D.open(gid, data_name);
+                    filespaceID = H5D.get_space(datasetID);
+                    [~, spaceDims] = H5S.get_simple_extent_dims(filespaceID);
+%                     spaceDims = fliplr(spaceDims);
+                    
+                    start = ones(1,numel(dims))-1;
+                    count = dims;
+
+                    stride = ones(1, numel(start));
+                    boundsEnd = start + (count).*stride;
+                    new_dims = fliplr(boundsEnd);
+                    H5S.close(filespaceID);
+                    H5D.set_extent(datasetID,new_dims);
+                    filespaceID = H5D.get_space(datasetID);
+                    H5S.select_hyperslab(filespaceID, 'H5S_SELECT_SET', fliplr(start), fliplr(stride), ...
+                        fliplr(count), ones(1,length(start)));
+                    
+                    dataspaceID = H5S.create_simple(numel(count),fliplr(count),[]);
+                    
+                    extend_data = true;
+
+                catch
+                    dataspaceID = H5S.create_simple(length(dims), fliplr(dims), fliplr(dims_max));
+                end
+            end
+
+                    
+        else
+            try
+                datasetID = H5D.open(gid, data_name);
+                filespaceID = H5D.get_space(datasetID);
+                [~, spaceDims] = H5S.get_simple_extent_dims(filespaceID);
+                spaceDims = fliplr(spaceDims);
+                if extend_offset
+                    start = [ones(1,extend_dim-1) extend_offset+1]-1;
+                else
+                    start = [ones(1,extend_dim-1) spaceDims(end)+1]-1;
+                end
+
+                if numel(spaceDims) > numel(dims)
+                    count = [dims 1];
+                else
+                    count = dims;
+                end
+                stride = ones(1, numel(start));
+                boundsEnd = start + (count-1).*stride;
+                if extend_maxdims
+                    boundsStart = spaceDims;
+                    boundsStart(end) = extend_maxdims;
+                else
+                    boundsStart = spaceDims;
+                end
+                new_dims = fliplr(max(boundsStart,boundsEnd+1));
+                H5S.close(filespaceID);
+                H5D.set_extent(datasetID,new_dims);
+                filespaceID = H5D.get_space(datasetID);
+                H5S.select_hyperslab(filespaceID, 'H5S_SELECT_SET', fliplr(start), fliplr(stride), ...
+                    fliplr(count), ones(1,length(start)));
+                
+                dataspaceID = H5S.create_simple(numel(count),fliplr(count),[]);
+                
+                extend_data = true;
+                              
+                
+            catch
+                unlimited = H5ML.get_constant_value('H5S_UNLIMITED');
+                maxdims = [dims(1:extend_dim-1) unlimited];
+%                 maxdims = repmat(-1, 1, extend_dim);
+                if numel(maxdims) > numel(dims)
+                    dims = [dims 1];
+                end
+                dataspaceID = H5S.create_simple(length(dims), [fliplr(dims)], fliplr(maxdims));
+            end
+        end
+    end
+
+if strcmp(datatype, 'complex')
+    
+    %%% prepare compound dataset for complex input data
+    dims = size(data);
+
+    data_temp = data;
+    data = [];
+    data.r = real(data_temp);
+    data.i = imag(data_temp);
+    
+    create_dataspace();
+    
+    % Create the required data types
+    complexType = H5T.copy(get_datatype(data.r));
+    sz = H5T.get_size(complexType);
+    
+    % Create the compound datatype for memory.   
+    datatypeID = H5T.create ('H5T_COMPOUND', 2*sz);
+    H5T.insert (datatypeID, 'r',0, complexType);
+    H5T.insert (datatypeID, 'i',sz, complexType);
+    memtype = datatypeID;
+    
+    data_attr.MATLAB_class =  'complex';
+
+    
+elseif ischar(data)
+    % check if char is a link
+    ch_entrs = strsplit(data, ':');
+    if length(ch_entrs) >= 2
+        link = true;
+        if strcmp(ch_entrs{1}, 'ext')
+            % prepare external link
+            link_type = 'ext';
+        elseif strcmp(ch_entrs{1}, 'int_soft')
+            % prepare internal soft link
+            link_type = 'int_soft';
+        elseif strcmp(ch_entrs{1}, 'int_hard')
+            % prepare internal hard link
+            link_type = 'int_hard';
+        end
+    else
+        data = {data};
+        datatypeID = H5T.copy ('H5T_FORTRAN_S1');
+        H5T.set_size (datatypeID,'H5T_VARIABLE');
+        memtype = H5T.copy ('H5T_C_S1');
+        H5T.set_size (memtype, 'H5T_VARIABLE');
+        dataspaceID = H5S.create ('H5S_SCALAR');
+
+    end
+    
+elseif iscell(data) || strcmp(datatype, 'char_array')
+
+    
+    if iscellstr(data)
+        cellstrdata = true;
+        datatypeID = H5T.copy ('H5T_C_S1');
+        H5T.set_size (datatypeID, 'H5T_VARIABLE');
+        
+        dgcv = H5ML.get_constant_value('H5S_UNLIMITED');
+        dataspaceID    = H5S.create_simple(1,numel(data),dgcv);
+        memtype = datatypeID;
+        plist_cr = H5P.create('H5P_DATASET_CREATE');
+        H5P.set_chunk(plist_cr,1);
+        if strcmp(datatype, 'char_array')
+            data_attr.MATLAB_class =  'char_array';
+        end
+    else
+        write_data = false;
+        fn_names = cell(1,length(data));
+        cell_gid = add_groups(gid, data_name, plist, true);
+        for ii=1:length(data)
+            fn_names{ii} = sprintf([data_name '_%d'],ii-1);
+            write_dataset(data{ii}, cell_gid, fn_names{ii}, plist, comp, overwrite);
+        end
+        write_attribute(cell_gid, 'cell', 'MATLAB_class');
+
+
+    end
+    
+elseif isstruct(data)
+    
+    write_data = false;
+    struct_gid = add_groups(gid, data_name, plist, true);
+    add_content(data, struct_gid, plist, comp, overwrite);
+    
+else
+    datatypeID = H5T.copy(datatype);
+    dims = size(data);
+    if isfield(data_attr, 'save2hdf5DataShape')
+        dims = data_attr.save2hdf5DataShape;
+    end
+
+    % prepare dataspace
+    create_dataspace();
+
+    memtype = 'H5ML_DEFAULT';
+end
+
+%%% create groups and write data
+if comp && ~iscell(data) && ~ischar(data) && write_data || extend_dim || extendable
+    % define compression and chunk size
+    plist_ch = H5P.create('H5P_DATASET_CREATE');
+    if length(dims)>=3
+        chunk_dims = [dims(1) dims(2) ones(1, numel(dims)-2)];
+    else
+        chunk_dims = dims;
+    end
+
+    h5_chunk_dims = fliplr(chunk_dims);
+    H5P.set_chunk(plist_ch,h5_chunk_dims);
+    H5P.set_shuffle(plist_ch);
+    if comp
+        H5P.set_deflate(plist_ch,comp);
+    end
+    
+    % Try to create a new dataset. If it exists, try to open it.
+    try
+        if ~extend_data
+            if cellstrdata
+                datasetID = H5D.create(gid,data_name,datatypeID,dataspaceID,plist_cr);
+            else
+                datasetID = H5D.create(gid,data_name,datatypeID,dataspaceID,plist_ch);
+%                 create_dataspace();
+            end
+        end
+    catch
+        if ~overwrite
+            try
+                datasetID = H5D.open(gid, data_name);
+            catch
+                error('Could not create dataset %s! Try a different name or overwrite the already existing file.', data_name);
+            end
+        else
+            keyboard
+            error('Dataset %s already exists! Try a different name or overwrite the already existing file.', data_name);
+        end
+    end
+    
+elseif ~link && write_data
+    % Same as above but without compression:
+    % Try to create a new dataset. If it exists, try to open it.
+    try
+        if cellstrdata
+            datasetID = H5D.create(gid,data_name,datatypeID,dataspaceID,plist_cr);
+        else
+            datasetID = H5D.create(gid,data_name,datatypeID,dataspaceID,plist);
+        end
+    catch
+        if ~overwrite
+            try
+                datasetID = H5D.open(gid, data_name);
+            catch
+                error('Could not open dataset %s! Try a different name or overwrite the already existing file.', data_name);
+            end
+        else
+            error('Dataset %s already exists! Try a different name or overwrite the already existing file.', data_name);
+        end
+    end
+end
+
+if write_data
+    % write data to disk or link data
+    
+    if ~link && ~extend_data
+        H5D.write(datasetID,memtype,'H5S_ALL','H5S_ALL',plist ,data);
+        % append attributes if needed
+        if ~isempty(data_attr)
+            fn = fieldnames(data_attr);
+            for ii=1:length(fn)
+                write_attribute(datasetID, data_attr.(fn{ii}), fn{ii}, true);
+            end
+        end
+        H5D.close(datasetID);
+    elseif extend_data 
+        H5D.write(datasetID,memtype,dataspaceID, filespaceID, plist, data)
+    elseif strcmp(link_type, 'ext')
+        H5L.create_external(ch_entrs{2},ch_entrs{3},gid,data_name,plist,plist);
+    elseif strcmp(link_type, 'int_hard')
+        error('Currently not supported, sorry!')
+        %     H5L.create_hard(ch_entrs{2},'g3',gid1,'g4',plist,plist);
+    elseif strcmp(link_type, 'int_soft')
+        H5L.create_soft(ch_entrs{2},gid,data_name,plist,plist);
+    end
+    
+    
+
+
+end
+
+end
+

+io/common_header_value.m

@@ -0,0 +1,201 @@
+% Call function without arguments for a detailed explanation of its use
+
+% Filename: $RCSfile: common_header_value.m,v $
+%
+% $Revision: 1.7 $  $Date: 2013/01/25 10:22:26 $
+% $Author:  $
+% $Tag: $
+%
+% Description:
+% return header information which are common to most file formats used at
+% the cSAXS beamline
+%
+% Note:
+% Call without arguments for a brief help text.
+%
+% Dependencies:
+% - get_hdr_val
+%
+%
+% history:
+%
+% September 30th 2010:
+% add HDF5
+%
+% October 3rd 2008:
+% update FLI date-field since version 1.20 provides a time stamp string
+%
+% August 28th 2008:
+% correct error display for unknown extensions,
+% add extension .dat
+%
+% July 17th 2008: add mar extension
+%
+% May 7th 2008: 1st version
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [value] = common_header_value(header,extension,signature)
+import io.image_read
+import utils.get_hdr_val
+
+% initialize return value
+value = [];
+
+% check number of input arguments
+if (nargin ~= 3)
+    common_header_value_help();
+    error('invalid number of input arguments');
+end
+    
+switch extension
+    case 'cbf'
+        switch signature
+            case 'ExposureTime'
+                value = get_hdr_val(header,'Exposure_time',' %f',1);
+            case 'Date'
+                % The date is available in the Pilatus comments, without
+                % any signature. Searching for the 20 string will fail
+                % beyond the year 2099
+                value = [ '20' get_hdr_val(header,'# 20',' %[^\r]',1) ];
+            otherwise
+                error('unknown signature %s for extension %s',...
+                    signature,extension);
+        end
+    case {'h5', 'hdf5'}
+        switch signature
+            case 'ExposureTime'
+                value = get_hdr_val(header,'Exposure_time',' %f',1);
+            case 'Date'
+                value = get_hdr_val(header,'DateTime',' %[^\n]',1);
+            otherwise
+                error('unknown signature %s for extension %s',...
+                    signature,extension);
+        end
+    case 'dat'
+        switch signature
+            case 'ExposureTime'
+                value = get_hdr_val(header,'Exposure_time',' %f',1);
+            case 'Date'
+                value = get_hdr_val(header,'DateTime',' %[^\n]',1);
+            otherwise
+                error('unknown signature %s for extension %s',...
+                    signature,extension);
+        end        
+    case 'edf'
+        switch signature
+            case 'ExposureTime'
+                value = get_hdr_val(header,'count_time',' = %f',1);
+            case 'Date'
+                value = get_hdr_val(header,'Date',' = %[^;]',1);
+            otherwise
+                error('unknown signature %s for extension %s',...
+                    signature,extension);
+        end
+    case {'mar','mccd'}
+        switch signature
+            case 'ExposureTime'
+                value = get_hdr_val(header,'ExposureTime_ms',' %f',1)/1000;
+            case 'Date'
+                value = get_hdr_val(header,'DateTime',' %[^\n]',1);
+            otherwise
+                error('unknown signature %s for extension %s',...
+                    signature,extension);
+        end
+    case {'mat'}
+        switch signature
+            case 'ExposureTime'
+                value = get_hdr_val(header,'Exposure_time',' %f',1);
+            case 'Date'
+                value = get_hdr_val(header,'DateTime',' %[^\n]',1);
+            otherwise
+                error('unknown signature %s for extension %s',...
+                    signature,extension);
+        end
+    case 'raw'
+        switch signature
+            case 'ExposureTime'
+                value = get_hdr_val(header,'exptimesec',' %f',1);
+            case 'Date'
+                value = get_hdr_val(header,'version',' %[^\n]',1);
+                if (version < 1.20)
+                    value = get_hdr_val(header,'FileTimestamp',' %[^\n]',1);
+                else
+                    value = get_hdr_val(header,'timestamp_string',' %[^\n]',1);
+                end
+            otherwise
+                error('unknown signature %s for extension %s',...
+                    signature,extension);
+        end
+    case 'spe'
+        switch signature
+            case 'ExposureTime'
+                value = get_hdr_val(header,'exposure',' %f',1);
+            case 'Date'
+                value = [ get_hdr_val(header,'date',' %[^\n]',1)'; ' '; 
+                    get_hdr_val(header,'ExperimentTimeLocal',' %[^\n]',1)' ]';
+            otherwise
+                error('unknown signature %s for extension %s',...
+                    signature,extension);
+        end
+    case {'tif', 'tiff'}
+        switch signature
+            case 'ExposureTime'
+                value = get_hdr_val(header,'Exposure_time',' %f',1);
+            case 'Date'
+                value = get_hdr_val(header,'DateTime',' %[^\n]',1);
+            otherwise
+                error('unknown signature %s for extension %s',...
+                    signature,extension);
+        end
+    otherwise
+        common_header_value_help();
+        error('unknown extension ''%s''',extension);
+end
+
+
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+function [] = common_header_value_help()
+
+fprintf('Usage:\n');
+fprintf('[value]=%s(header,extension,signature);',mfilename);
+fprintf('header and extension are returned by image_read\n');
+fprintf('The following signatures are recognized:\n');
+fprintf('ExposureTime      exposure time in seconds\n');
+fprintf('Date              date string in detector specific format\n');
+fprintf('Example:\n');
+fprintf('exp_time_sec=common_header_value(frame.header{1},frame.extension{1},''ExposureTime'');\n');

+io/convert_radial_2_dat.m

@@ -0,0 +1,86 @@
+% convert_radial_2_dat converts all radial integration mat files in 
+%    readpahtmask into dat files, pauses 10 minutes and repeats
+% 
+% Inputs:
+%    **readpathmask     A cell containing the input file string masks
+%    **outpathmask      A cel containint the corresponding output
+%                       directories
+%
+% Example:
+% readpathmask{1} = '~/Data10/analysis/radial_integration/*.mat';
+% outpathmask{1}  = '~/Data10/analysis/radial_integration_dat/';
+% readpathmask{2} = '~/Data10/analysis/radial_integration_waxs/*.mat';
+% outpathmask{2}  = '~/Data10/analysis/radial_integration_waxs_dat/';
+% convert_radial_2_dat(readpathmask, outpathmask)
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2019 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+
+% clear
+
+% readpathmask{1} = '~/Data10/analysis/radial_integration/*.mat';
+% outpathmask{1}  = '~/Data10/analysis/radial_integration_dat/';
+% 
+% readpathmask{2} = '~/Data10/analysis/radial_integration_waxs/*.mat';
+% outpathmask{2}  = '~/Data10/analysis/radial_integration_waxs_dat/';
+
+function convert_radial_2_dat(readpathmask, outpathmask)
+
+while 1==1
+for ii = 1:numel(readpathmask)
+    if ~exist(outpathmask{ii},'dir')
+        mkdir(outpathmask{ii})
+    end
+    files = dir(readpathmask{ii});
+    for jj = 1:numel(files)
+        currentradial = fullfile(files(jj).folder,files(jj).name);
+        [auxpath, auxname, auxext] = fileparts(currentradial);
+        outputradial  = fullfile(outpathmask{ii},[auxname '.dat']);
+        s = load(currentradial);
+        
+        save_data = [s.q.', ...
+            reshape(s.I_all     , [size(s.I_all,1) size(s.I_all,2)*size(s.I_all,3) ]) , ...
+            reshape(s.I_std     , [size(s.I_std,1) size(s.I_std,2)*size(s.I_std,3) ]) , ...
+            reshape(s.norm_sum  , [size(s.norm_sum,1) size(s.norm_sum,2)*size(s.norm_sum,3) ]) , ...
+            ];
+        
+        fprintf('Saving %s\n',outputradial);
+        save( outputradial , 'save_data', '-ascii','-double');
+    end
+    clear files
+end
+fprintf('Pausing 10 minutes\n')
+pause(60*10)
+end

+io/convert_to_rgb.m

@@ -0,0 +1,32 @@
+function [rgb_data] = convert_to_rgb(data)
+%Convert complex data into rgb image showing both magnitude and phase
+%   Detailed explanation goes here
+
+import math.sp_quantile
+
+[W,H] = size(data);
+adata = abs(data);
+
+alpha  = 1e-3;
+tmp= sort(adata(:));
+MAX = tmp(ceil(end*(1-alpha)));
+ind = adata > MAX;
+data(ind) = MAX * data(ind) ./ abs(data(ind));
+adata = abs(data);
+range = sp_quantile(adata(:), [1e-2, 1-1e-2],10);
+adata = (adata  - range(1) ) ./ ( range(2) - range(1) );
+
+ang_data = angle(data); 
+hue =  mod(ang_data+2.5*pi, 2*pi)/(2*pi);
+hsv_data = [ hue(:) , ones(W*H,1), adata(:) ];
+
+hsv_data = min(max(0, hsv_data),1);
+
+
+rgb_data = hsv2rgb(hsv_data);
+
+rgb_data = reshape(rgb_data, W,H,3);      
+rgb_data = min(1,rgb_data);
+
+end
+

+io/datread.m

@@ -0,0 +1,201 @@
+% Call function without arguments for a detailed explanation of its use
+
+% Filename: $RCSfile: datread.m,v $
+%
+% $Revision: 1.2 $  $Date: 2009/02/20 19:33:01 $
+% $Author:  $
+% $Tag: $
+%
+% Description:
+% Macro for reading .dat files in self-defined data formats
+%
+% Note:
+% Call without arguments for a brief help text.
+%
+% Dependencies:
+% - image_read_set_default
+% - fopen_until_exists
+% - get_hdr_val
+% - compiling cbf_uncompress.c increases speed but is not mandatory
+%
+%
+% history:
+%
+% February 18th 2009: 1st version
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [frame,vararg_remain] = datread(filename,varargin)
+import io.*
+import utils.fopen_until_exists
+import utils.get_hdr_val
+import utils.char_to_cellstr
+
+% 0: no debug information
+% 1: some feedback
+% 2: a lot of information
+debug_level = 0;
+
+% initialize return argument
+frame = struct('header',[], 'data',[]);
+
+
+% check minimum number of input arguments
+if (nargin < 1)
+    image_read_sub_help(mfilename,'cbf');
+    error('At least the filename has to be specified as input parameter.');
+end
+
+% accept cell array with name/value pairs as well
+no_of_in_arg = nargin;
+if (nargin == 2)
+    if (isempty(varargin))
+        % ignore empty cell array
+        no_of_in_arg = no_of_in_arg -1;
+    else
+        if (iscell(varargin{1}))
+            % use a filled one given as first and only variable parameter
+            varargin = varargin{1};
+            no_of_in_arg = 1 + length(varargin);
+        end
+    end
+end
+
+% check number of input arguments
+if (rem(no_of_in_arg,2) ~= 1)
+    error('The optional parameters have to be specified as ''name'',value pairs');
+end
+    
+% set default values for the variable input arguments and parse the named
+% parameters: 
+vararg = cell(0,0);
+for ind = 1:2:length(varargin)
+    name = varargin{ind};
+    value = varargin{ind+1};
+    switch name
+        otherwise
+            % pass further arguments on to fopen_until_exists
+            vararg{end+1} = name;
+            vararg{end+1} = value;
+    end
+end
+
+
+% expected maximum length for the text header
+max_header_length = 4096;
+
+% end-of-header signature
+eoh_signature = [ '# end-of-header' char(10) ];
+
+% try to open the data file
+if (debug_level >= 1)
+    fprintf('Opening %s.\n',filename);
+end
+[fid,vararg_remain] = fopen_until_exists(filename,vararg);
+if (fid < 0)
+    return;
+end
+
+% read all data at once
+[fdat,fcount] = fread(fid,'uint8=>uint8');
+
+% close input data file
+fclose(fid);
+if (debug_level >= 2)
+    fprintf('%d data bytes read\n',fcount);
+end
+
+% files with header start with a has sign, otherwise just a series of
+% numbers is expected
+if (~strcmp(fdat(1),'#'))
+    % convert the array to cell strings
+    cell_values = char_to_cellstr( char(fdat)' );
+    % convert the strings to double precision values
+    frame.data = zeros(length(cell_values),1);
+    for (ind = 1:length(cell_values))
+        frame.data(ind) = str2double(cell_values{ind});
+    end
+    
+    % No header information are available. 
+    % Fake exposure time information to avoid problems in other
+    % macros. 
+    frame.header{end+1} = 'Exposure_time 1.0';
+    % add the file modification date to the header
+    dir_entry = dir(filename);
+    frame.header{end+1} = [ 'DateTime ' dir_entry.date ];
+end
+
+
+% search for end of header signature within the expected maximum length of
+% a header
+end_of_header_pos = ...
+    strfind( fdat(1:min(max_header_length,length(fdat)))',...
+             eoh_signature );
+if (length(end_of_header_pos) < 1)
+    error( [ filename,': no header end signature found' ] );
+    return;
+end
+if (debug_level >= 2)
+    fprintf('Header length is %d bytes.\n',end_of_header_pos -1);
+end
+
+% return the complete header as lines of a cell array
+frame.header = char_to_cellstr( char(fdat(1:(end_of_header_pos-1))') );
+
+% increase the index to the first data byte
+end_of_header_pos = end_of_header_pos + length(eoh_signature);
+
+% check for information on the various dimensions in ascending speed order
+dim1 = get_hdr_val(frame.header,'dim2','%d',1);
+dim2 = get_hdr_val(frame.header,'dim1','%d',1);
+dim3 = get_hdr_val(frame.header,'number-of-exposures','%d',1);
+dim4 = get_hdr_val(frame.header,'channels','%d',1);
+
+if (debug_level >= 2)
+    fprintf('Frame dimensions are %d x %d % %d x %d.\n', ...
+            dim4,dim3,dim2,dim1);
+end
+
+% store the numbers in the array, fastest axis first
+frame.data = zeros(dim4,dim3,dim2,dim1);
+
+% convert the strings to double precision values
+data_1d = sscanf(char(fdat(end_of_header_pos:end))','%f');
+frame.no_of_el_read = length(data_1d);
+if (length(data_1d) > numel(frame.data))
+    frame.data = zeros(dim4,dim3,dim2,ceil(length(data_1d)/(dim4*dim3*dim2)));
+end
+frame.data(1:frame.no_of_el_read) = data_1d;
+

+io/edfread.m

@@ -0,0 +1,234 @@
+% Call function without arguments for a detailed explanation of its use
+
+% Filename: $RCSfile: edfread.m,v $
+%
+% $Revision: 1.1 $  $Date: 2008/06/10 17:05:14 $
+% $Author:  $
+% $Tag: $
+%
+% Description:
+% Macro for reading ESRF Data Format (EDF) files written by the
+% Pilatus detector control program camserver. 
+%
+% Note:
+% Currently this routine supports only the subset of EDF features needed to
+% read the Pilatus detector data. 
+% Call without arguments for a brief help text.
+%
+% Dependencies:
+% - fopen_until_exists
+% - get_hdr_val
+% - image_read_set_default
+%
+%
+% history:
+%
+% May 9th 2008: 1st version after redesign
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [frame,vararg_remain] = edfread(filename,varargin)
+import io.*
+import utils.char_to_cellstr
+import utils.fopen_until_exists
+import utils.get_hdr_val
+
+% 0: no debug information
+% 1: some feedback
+% 2: a lot of information
+debug_level = 0;
+
+% initialize return argument
+frame = struct('header',[], 'data',[]);
+
+% check minimum number of input arguments
+if (nargin < 1)
+    image_read_sub_help(mfilename,'edf');
+    error('At least the filename has to be specified as input parameter.');
+end
+
+% accept cell array with name/value pairs as well
+no_of_in_arg = nargin;
+if (nargin == 2)
+    if (isempty(varargin))
+        % ignore empty cell array
+        no_of_in_arg = no_of_in_arg -1;
+    else
+        if (iscell(varargin{1}))
+            % use a filled one given as first and only variable parameter
+            varargin = varargin{1};
+            no_of_in_arg = 1 + length(varargin);
+        end
+    end
+end
+
+% check number of input arguments
+if (rem(no_of_in_arg,2) ~= 1)
+    error('The optional parameters have to be specified as ''name'',''value'' pairs');
+end
+
+    
+% set default values for the variable input arguments and parse the named
+% parameters: 
+vararg = cell(0,0);
+for ind = 1:2:length(varargin)
+    name = varargin{ind};
+    value = varargin{ind+1};
+    switch name
+        otherwise
+            % pass further arguments on to fopen_until_exists
+            vararg{end+1} = name;
+            vararg{end+1} = value;
+    end
+end
+
+
+% expected maximum length for the text header
+max_header_length = 4096;
+
+
+% try to open the data file
+if (debug_level >= 1)
+    fprintf('Opening %s.\n',filename);
+end
+[fid,vararg_remain] = fopen_until_exists(filename,vararg);
+if (fid < 0)
+    return;
+end
+
+% read all data at once
+[fdat,fcount] = fread(fid,'uint8=>uint8');
+
+% close input data file
+fclose(fid);
+if (debug_level >= 2)
+    fprintf('%d data bytes read\n',fcount);
+end
+
+% search for end of header signature within the expected maximum length of
+% a header
+end_of_header_pos = 1024;
+max_pos = min( max_header_length,length(fdat) );
+
+while ((end_of_header_pos < max_pos) && (fdat(end_of_header_pos-1) ~= '}'))
+    end_of_header_pos = end_of_header_pos +1024;
+end
+if (end_of_header_pos >= max_pos)
+    error('no header end signature found');
+end
+if (debug_level >= 2)
+    fprintf('Header length is %d bytes.\n',end_of_header_pos);
+end
+data_length = fcount - end_of_header_pos;
+
+% convert the header to lines of a cell array
+frame.header = char_to_cellstr( char(fdat(1:(end_of_header_pos-1))') );
+
+% check for opening parenthesis
+if (frame.header{1} ~= '{')
+    error([filename ': EDF start ,''{'' not found in first line ''' ...
+        frame.header{1} '''' ]);
+end
+
+
+% extract the mandatory information for data extraction from the header:
+byte_order = get_hdr_val(frame.header,'ByteOrder',' = %s',1);
+dim1 = get_hdr_val(frame.header,'Dim_1',' = %d',1);
+dim2 = get_hdr_val(frame.header,'Dim_2',' = %d',1);
+data_type = get_hdr_val(frame.header,'DataType',' = %s',1);
+if (debug_level >= 2)
+    fprintf('Byte order is %s\n',byte_order);
+    fprintf('Frame dimensions are %d x %d.\n',dim2,dim1);
+    fprintf('Data type is %s\n',data_type);
+end
+
+% determine number of bytes per pixel
+switch data_type
+    case 'UnsignedByte',
+        bytes_per_pixel = 1;
+        data_class = 'uint8';
+    case 'UnsignedShort',
+        bytes_per_pixel = 2;
+        data_class = 'uint16';
+    case {'SignedInteger','UnsignedInteger','UnsignedInt','UnsignedLong'}
+        bytes_per_pixel = 4;
+        data_class = 'uint32';
+    case {'Float','FloatValue','Real'}
+        bytes_per_pixel = 4;
+        data_class = 'single';
+    case 'DoubleValue'
+        bytes_per_pixel = 8;
+        data_class = 'double';
+    otherwise
+        error('unsupported data type %s',data_type);
+end
+no_of_bytes = bytes_per_pixel * dim1 * dim2;
+if (debug_level >= 2)
+    fprintf('%d bytes per pixel, %d in total expected, %d available\n',...
+        bytes_per_pixel,no_of_bytes,data_length);
+end
+
+% check length of available data
+if (no_of_bytes > data_length)
+    error('%d data bytes expected, %d are available',...
+        no_of_bytes,data_length);
+end
+
+% compare file with machine byte order, swap if necessary
+[str,maxsize,endian] = computer;
+if (((strcmp(byte_order,'HighByteFirst')) && (endian == 'L')) || ...
+    ((strcmp(byte_order,'LowByteFirst')) && (endian == 'H')))
+    if (debug_level >= 2)
+        fprintf('Machine byte order is %s: swapping data bytes\n',...
+            endian,bytes_per_pixel);
+    end
+    dat = fdat(end_of_header_pos+1:end_of_header_pos+no_of_bytes);
+    dat = reshape(dat,bytes_per_pixel,[]);
+    dat = flipud(dat);
+    fdat(end_of_header_pos+1:end_of_header_pos+no_of_bytes) = dat(:);
+end
+
+% extract the frame from the binary data
+[frame.data] = ...
+    double(reshape(typecast(fdat(end_of_header_pos+1:end_of_header_pos+no_of_bytes),...
+                            data_class),...
+                   dim1,dim2));
+
+    
+% conversion to standard view on Pilatus 2M data at the SLS/cSAXS beamline
+% if (~original_orientation)
+%     % this is slow, even slower is fliplr(flipud(frame.'))
+%     frame.data = frame.data(end:-1:1,end:-1:1)';
+% end

+io/export_for_SASfit.m

@@ -0,0 +1,298 @@
+% This script is to plot, correct and export solution SAXS data to SASfit
+% accounts for transmission, time and thickness correction
+% scales the data to a calibration factor 
+% background correction, removal of bad pixels
+% not suitable for anisotropic data
+% saves the output to be used in SASfit
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%  EDIT HERE
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% give the calibration factor for absolute intensity, calculated previously
+cal_factor_SAXS = 2.91e-4;
+cal_factor_WAXS = 2.01e-5 ;
+
+% where the data is saved (Data10, afs, p-account)
+base_dir = '~/Data10/';%'/sls/X12SA/Data20/e16598/';
+save_dir = '~/Data10/';%'/mnt/das-gpfs/work/p16598/';
+eaccount = beamline.identify_eaccount; % 'e16598';
+
+% samples and thicknesses
+Air = 15; % scan used for transmission calculation
+sample_scan = [34:38]; % should be given
+sample_thickness = 0.15; % in cm: important for absolute scattering
+back_scan = []; % used as background, leave it empty [] for no subtraction !!NOT TESTED!!
+back_thickness = 0.01; % in cm: important for absolute scattering
+
+% export data for SASfit?
+export_sasfit = 1;
+
+%plot curves?
+plot_curves = 0;
+
+% save figures?
+save_fig = 0;
+
+% average the scan points? 1 = yes, 0 = no
+average_scan = 1;
+% scale also the waxs data? yes = 1; no = 0;
+use_waxs = 0;
+
+% which bad pixels should be removed
+bad_pixel = []; %given as a vector [811, 825]
+
+% use all data measurement points or skip some (faster)
+skip_measurements = [100]; % use 1 to show all
+
+% used to reduce noise at the beginning and end of scattering curve
+skip_first_points = 55;
+skip_last_points = 150;
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+%% load the diode value for the air
+S_air = io.spec_read(base_dir,'ScanNr',Air);
+%scale in case the exposure times are different
+exp_time = S_air.sec(1,1);
+scale_air = 1/exp_time;
+Air_data = load(sprintf('%s/analysis/radial_integration/%s_1_%05d_00000_00000_integ.mat', base_dir, eaccount, Air));
+I_air = mean(Air_data.I_all, 3);
+% average over the segments when needed
+if size(I_air, 2) > 1
+    I_air = (I_air .* Air_data.norm_sum)./sum(Air_data.norm_sum, 2);
+    I_air = sum(I_air, 2);
+end
+
+if use_waxs
+    Air_waxs = load(sprintf('%s/analysis/radial_integration_waxs/%s_2_%05d_00000_00000_integ.mat', base_dir, eaccount, Air));
+    I_air_waxs = mean(squeeze(Air_waxs.I_all), 2);
+end
+%% background correction
+if ~isempty(back_scan)
+    for b = 1:length(back_scan)
+        bgr = load(sprintf('%s/analysis/radial_integration/%s_1_%05d_00000_00000_integ.mat', base_dir, eaccount, back_scan(b)));
+        S_back = spec_read(base_dir,'ScanNr',back_scan(b));
+        % in case the burst scan takes place, the transmission is
+        % calculated differently
+        if ~isempty(findstr(S_back.S, 'burst_scan'))
+            delimiter = ' ';
+            formatSpec = '%*s%*s%s%[^\n\r]';
+            fileID = fopen(sprintf('%smcs/S00000-00999/S%05d/%s_%05d.dat', base_dir,back_scan(b), eaccount, back_scan(b)), 'r');
+            dataArray = textscan(fileID, formatSpec, 'Delimiter', delimiter, 'MultipleDelimsAsOne', true,  'ReturnOnError', false);
+            exp_time = dataArray{1,1}{7,1};
+            scale_back = 1/str2num(exp_time);
+            diode = mean(str2num(dataArray{1,1}{9,end}));
+            transm_back = ((diode*scale_back)/(mean(S_air.diode)*scale_air));
+            fclose(fileID);
+        else
+            %scale in case the exposure times are different
+            exp_time = S_back.sec(1,1);
+            scale_back = 1/exp_time;
+            transm_back = (mean(S_back.diode)/mean(S_back.bpm4i))/(mean(S_air.diode)/mean(S_air.bpm4i));
+        end
+        %average background
+        I_bgr = mean(bgr.I_all, 3);
+        if size(I_bgr, 2) > 1
+            I_bgr = (I_bgr .* bgr.norm_sum)./sum(bgr.norm_sum, 2);
+            I_bgr = sum(I_bgr, 2);
+        end
+        I_bgr = ((((I_bgr*scale_back)*1/transm_back)-(I_air*scale_air))*1/back_thickness);
+        I_bgr = I_bgr * cal_factor_SAXS;
+        if use_waxs
+            %average background_WAXS
+            bgr_waxs = importdata(sprintf('%s/analysis/radial_integration_waxs/%s_2_%05d_00000_00000_integ.mat', base_dir, eaccount, back_scan(b)));
+            I_bgr_waxs = mean(squeeze(bgr_waxs.I_all), 2);
+            I_bgr_waxs = ((((I_bgr_waxs*scale_back)*1/transm_back)-(I_air_waxs*scale_air))*1/back_thickness);
+            I_bgr_waxs = I_bgr_waxs * cal_factor_WAXS;
+        end
+    end
+else
+    I_bgr_waxs = 0;
+    I_bgr = 0;
+end
+
+%% load and correct the sample
+for s = 1:length(sample_scan)
+    sample_filename=sprintf('%s/analysis/radial_integration/%s_1_%05d_00000_00000_integ.mat', base_dir, eaccount, sample_scan(s));
+    if exist(sample_filename) == 2
+        display(['reading file ',sample_filename])
+        sample = load(sample_filename);
+    else
+        continue
+    end
+    S_s = io.spec_read(base_dir,'ScanNr',sample_scan(s));
+    if ~isempty(findstr(S_s.S, 'burst_scan'))
+        delimiter = ' ';
+        formatSpec = '%*s%*s%s%[^\n\r]';
+        fileID = fopen(sprintf('%smcs/S00000-00999/S%05d/%s_%05d.dat', base_dir,sample_scan(s), eaccount,sample_scan(s)), 'r');
+        dataArray = textscan(fileID, formatSpec, 'Delimiter', delimiter, 'MultipleDelimsAsOne', true,  'ReturnOnError', false);
+        exp_time = dataArray{1,1}{7,1};
+        scale_s = 1/str2num(exp_time);
+        diode = mean(str2num(dataArray{1,1}{9,end}));
+        transm_sample = ((diode*scale_s)/(mean(S_air.diode)*scale_air));
+        fclose(fileID);
+    else
+        %scale in case the exposure times are different
+        exp_time = S_s.sec(1,1);
+        scale_s = 1/exp_time;
+        transm_sample = (mean(S_s.diode)/mean(S_s.bpm4i))/(mean(S_air.diode)/mean(S_air.bpm4i));
+    end
+    %load the sample
+    I_sample = squeeze(sample.I_all);
+    q_sample = sample.q';
+    
+    if use_waxs
+        %average background_WAXS
+        sample_waxs = load(sprintf('%s/analysis/radial_integration_waxs/%s_2_%05d_00000_00000_integ.mat', base_dir, eaccount, sample_scan(s)));
+        I_sample_waxs = (sample_waxs.I_all);
+        q_sample_waxs = sample_waxs.q';
+    else
+        q_sample_waxs = [];
+        I_sample_waxs = [];
+    end
+    if average_scan
+        I_sample = mean(I_sample, 3);
+        I_std = mean(sample.I_std, 3);
+        if size(I_sample, 2) > 1
+            I_sample = (I_sample .* sample.norm_sum)./sum(sample.norm_sum, 2);
+            I_std = (I_std .* sample.norm_sum)./sum(sample.norm_sum, 2);
+            I_std = sum(I_std, 2).*cal_factor_SAXS;
+            I_sample = sum(I_sample, 2);
+        end
+        I_std = I_std(skip_first_points:end-skip_last_points,:);
+        I_sample = ((((I_sample*scale_s)*1/transm_sample)-(I_air*scale_air))*1/sample_thickness);
+        if ~isempty(bad_pixel)
+            I_sample(bad_pixel,1) = (I_sample(bad_pixel-1,1)+I_sample(bad_pixel+1,1))/2;
+        end
+        I_sample = I_sample * cal_factor_SAXS;
+        I_cor = (I_sample-I_bgr);
+        I_cor = I_cor(skip_first_points:end-skip_last_points,:);
+        
+        if use_waxs
+            hold on
+            I_sample_waxs = median(I_sample_waxs,3);
+            I_sample_waxs = ((((I_sample_waxs.*scale_s).*1/transm_sample)-(I_air_waxs.*scale_air)).*1/sample_thickness);
+            I_sample_waxs = I_sample_waxs * cal_factor_WAXS;
+            I_cor_waxs = (I_sample_waxs-I_bgr_waxs);
+            
+        else
+            I_cor_waxs = [];
+        end
+        I_total = [I_cor; I_cor_waxs];
+        
+        q_total = [q_sample(skip_first_points: end-skip_last_points,:); q_sample_waxs];
+        [q_total, index] = sort(q_total);
+        I_total = I_total(index);
+        
+        if plot_curves
+            figure
+            plot(q_total*10, I_total);
+            set(gca,'XScale','log', 'YScale','log');
+            grid on;
+            box on;
+            xlabel('scattering vector q (nm^{-1})');
+            ylabel('differential scattering cross-section (cm^{-1})');
+            hold on
+        end
+        if export_sasfit
+            save_data = [q_total*10, I_total, I_std];
+            filename = sprintf('scan_%05d_avg', sample_scan);
+            save(sprintf('%sanalysis/dat_files/%s.dat', save_dir , filename) , 'save_data', '-ascii');
+        end
+    else
+        if plot_curves
+            figure
+            hold on
+        end
+        for i = 1:skip_measurements:size(sample.I_all, 3)
+            I_point = sample.I_all(:,:,i);
+            I_point_std = sample.I_std(:,:, i);
+            if size(I_point, 2) > 1
+                I_point = (I_point .* sample.norm_sum)./sum(sample.norm_sum, 2);
+                I_point = sum(I_point, 2);
+                I_point_std = (I_point_std .* sample.norm_sum)./sum(sample.norm_sum, 2);
+                I_point_std = sum(I_point_std, 2).*cal_factor_SAXS;
+            end
+            I_point_std = I_point_std(skip_first_points:end-skip_last_points,:);
+            if ~isempty(bad_pixel)
+                I_point(bad_pixel,1) = (I_point(bad_pixel-1,1) + I_point(bad_pixel+1,1))/2;
+            end
+            I_point = ((((I_point*scale_s)*1/transm_sample)-(I_air*scale_air))*1/sample_thickness);
+            I_point = I_point * cal_factor_SAXS;
+            I_cor = (I_point-I_bgr);
+            I_cor = I_cor(skip_first_points: end-skip_last_points,:);
+            
+            if use_waxs
+                I_point_waxs = I_sample_waxs(:,i);
+                I_point_waxs = ((((I_point_waxs*scale_s)*1/transm_sample)-(I_air_waxs*scale_air))*1/sample_thickness);
+                I_point_waxs = I_point_waxs * cal_factor_WAXS;
+                I_cor_waxs = (I_point_waxs-I_bgr_waxs);
+                I_point_std_WAXS = I_sample_waxs(:,:, i);
+                I_point_std_WAXS = I_point_std_WAXS.*cal_factor_WAXS;
+                I_point_std_WAXS = sum(I_point_std_WAXS, 2).*cal_factor_SAXS;
+            else
+                I_cor_waxs = [];
+            end
+            I_total = [I_cor; I_cor_waxs];
+            q_total = [q_sample(skip_first_points: end-skip_last_points,:); q_sample_waxs];
+            [q_total, index] = sort(q_total);
+            I_total = I_total(index);
+            I_point_std_total= [I_point_std; I_point_std_WAXS];
+            if plot_curves
+                plot(q_total*10, I_total);
+                grid on;
+                box on;
+                set(gca,'XScale','log', 'YScale','log');
+                xlabel('scattering vector q (nm^{-1})');
+                ylabel('differential scattering cross-section (cm^{-1})');
+                axis tight
+                hold on
+                drawnow
+            end
+            if export_sasfit
+                save_data = [q_total*10, I_total, I_point_std_total];
+                filename = sprintf('scan_%05d_pt_%05d', sample_scan(s), i);
+                save(sprintf('%sanalysis/dat-files/%s.dat', save_dir , filename) , 'save_data', '-ascii');
+            end
+        end
+        
+    end
+    
+end
+if save_fig
+    %save the results
+    saveas(gcf, sprintf('%sanalysis/scanNr_%05d.jpg', save_dir , sample_scan))
+end
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.

+io/falcon_read.m

@@ -0,0 +1,67 @@
+ 
+% Read data from Falcon readout electronics
+% Input is the filename with path
+% Output is a structure containing fields:
+%    Data contains one spectrum per measurement point
+%    Metadata contains some other information like the number of points per
+%    data transfer and the total number of points
+% 12 March 2019
+%
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function dataout = falcon_read(filename)
+
+data1 = io.HDF.hdf5_load(filename);
+
+spectra_per_transfer    = data1.entry.instrument.FalconX1.PixelsPerBuffer(end);%meta.spectra_per_transfer; % aka pixels per buffer in the MEDM
+numberofpositions       = data1.entry.instrument.FalconX1.CurrentPixel(end);%data1.entry.instrument.NDAttributes.CurrentPixel(end);%meta.numberofpositions;
+data = data1.entry.data.data;
+
+data = data(1+256:end,1,:);
+
+N = size(data);
+if size(data,3) > 1
+    data = reshape(data,N(1)/spectra_per_transfer,spectra_per_transfer*N(3));
+else
+    data = reshape(data,N(1)/spectra_per_transfer,spectra_per_transfer);
+end
+
+dataout.data = data(1:2:end,1:numberofpositions);
+
+end
+
+
+
+

+io/find_base_package.m

@@ -0,0 +1,69 @@
+%FIND_BASE_PACKAGE
+% finds the path to the cSAXS base package by looking for a specific file (+math)
+% the code goes up to 3 levels down in the folder structure and it tries to find any folder matching ./*/+math/
+%
+% returns:
+% ++ base_package_path              path to the cSAXS base package
+%
+% Example how to use it: addpath(find_base_package())
+
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+
+
+function base_package_path = find_base_package()
+    maxdepth = 3;
+	test_path = '+math'; % one file to find them all
+    
+    lvl = 1;
+    cpath = '';
+    ret = '';
+    while isempty(ret) && ~contains(strtrim(ret), test_path)
+        [~, ret] = system(sprintf('find -L %s -maxdepth 2 -type d -name "%s"', cpath, test_path));
+        if lvl > maxdepth
+            break
+        end
+        lvl = lvl + 1;
+        cpath = [cpath '../'];
+    end
+    ret = split(ret); 
+    base_package_path = strtrim(ret{1}); 
+    base_package_path = base_package_path(1:end-length(test_path));
+
+    if isempty(base_package_path)
+        error('cSAXS base package was not found')
+    end
+end

+io/fliread.m

@@ -0,0 +1,202 @@
+% Call function without arguments for a detailed explanation of its use
+
+% Filename: $RCSfile: fliread.m,v $
+%
+% $Revision: 1.4 $  $Date: 2008/10/03 13:50:04 $
+% $Author:  $
+% $Tag: $
+%
+% Description:
+% read a data file in the format stored by the program ccdfli.c
+%
+% Note:
+% The image files have the extension raw and the file format is home
+% defined. 
+% Call without arguments for a brief help text.
+%
+% Dependencies:
+% - image_read_set_default
+% - fopen_until_exists
+% - get_hdr_val
+%
+%
+% history:
+%
+% Pctober 3rd 2008: 
+% add new end-of-header signature search for version 1.20 files
+%
+% October 1st 2008: Exchange width and height in reshape command
+%
+% May 9th 2008: adapt to call from image_read
+%
+% November 3, 2005: include new fields of data format 1.1: 
+% exposure time Spec, exposure time measured, monitor counts
+%
+% October 2005: include optional from-to line reading
+%
+% March 2005: 1st version
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [frame,vararg_remain] = fliread(filename, varargin)
+import io.*
+import utils.char_to_cellstr
+import utils.fopen_until_exists
+import utils.get_hdr_val
+
+% 0: no debug information
+% 1: some feedback
+% 2: a lot of information
+debug_level = 0;
+
+% initialize return argument
+frame = struct('header',[], 'data',[]);
+
+% check minimum number of input arguments
+if (nargin < 1)
+    image_read_sub_help(mfilename,'raw');
+    error('At least the filename has to be specified as input parameter.');
+end
+
+% accept cell array with name/value pairs as well
+no_of_in_arg = nargin;
+if (nargin == 2)
+    if (isempty(varargin))
+        % ignore empty cell array
+        no_of_in_arg = no_of_in_arg -1;
+    else
+        if (iscell(varargin{1}))
+            % use a filled one given as first and only variable parameter
+            varargin = varargin{1};
+            no_of_in_arg = 1 + length(varargin);
+        end
+    end
+end
+
+% check number of input arguments
+if (rem(no_of_in_arg,2) ~= 1)
+    error('The optional parameters have to be specified as ''name'',''value'' pairs');
+end
+
+% set default values for the variable input arguments and parse the named
+% parameters: 
+vararg = cell(0,0);
+for ind = 1:2:length(varargin)
+    name = varargin{ind};
+    value = varargin{ind+1};
+    switch name
+        otherwise
+            % pass further arguments on to fopen_until_exists
+            vararg{end+1} = name;
+            vararg{end+1} = value;
+    end
+end
+
+% expected maximum length for the text header
+max_header_length = 1024;
+
+% try to open the data file
+if (debug_level >= 1)
+    fprintf('Opening %s.\n',filename);
+end
+[fid,vararg_remain] = fopen_until_exists(filename,vararg);
+if (fid < 0)
+    return;
+end
+
+% read all data at once
+[fdat,fcount] = fread(fid,'uint8=>uint8');
+
+% close input data file
+fclose(fid);
+if (debug_level >= 2)
+    fprintf('%d data bytes read\n',fcount);
+end
+
+% return the complete header as lines of a cell array
+frame.header = char_to_cellstr( char(fdat(1:max_header_length)'),1 );
+
+version_no = get_hdr_val(frame.header,'% version','%f',1);
+if ((version_no ~= 1.00) && (version_no ~= 1.10) && (version_no ~= 1.20))
+     fprintf('%s: File version number %.2f may not be supported\n',...
+	     mfilename,version_no);
+end
+
+[frameHeight,line_number] = get_hdr_val(frame.header,'% rows','%d',1);
+[frameWidth,line_number]  = get_hdr_val(frame.header,'% columns','%d',1);
+
+if (version_no < 1.20)
+    if (version_no == 1.10)
+        [monCounts,line_number]  = get_hdr_val(frame.header,'% monitorcounts','%d',1);
+    end
+
+    % cut off non-header lines
+    frame.header = frame.header(1:line_number);
+
+    % find start of data
+    eol_ind = regexp(char(fdat(1:max_header_length)'),'\n');
+    data_start = eol_ind(line_number) +1;
+else
+    eoh_signature = sprintf('%% EOH%c%c',10,26);
+    end_of_header_pos = ...
+        strfind( fdat(1:min(max_header_length,length(fdat)))',...
+             eoh_signature );
+    data_start = end_of_header_pos + length(eoh_signature);
+end
+
+
+% calculate end of data (should be end of file)
+data_end = data_start + frameWidth * frameHeight *2 -1;
+if (data_end > fcount)
+    error('%d bytes read but %d are needed',fcount,data_end);
+end
+if (data_end ~= fcount)
+     fprintf('%s warning: %d bytes read vs. %d needed\n',mfilename,...
+             fcount,data_end);
+end
+% cut out frame data
+frame.data = double( reshape(typecast(fdat(data_start:data_end),'uint16'), ...
+    frameWidth,frameHeight) );
+
+% conversion to standard view on FLI-CCD data at the SLS/cSAXS beamline
+% (to be determined)
+% if (~original_orientation)
+% %     frame = flipud(frame');
+%     frame.data = frame.data';
+% end
+
+% add the file modification date to the header
+dir_entry = dir(filename);
+frame.header{end+1} = [ 'FileTimestamp ' dir_entry.date ];

+io/get_host_name.m

@@ -0,0 +1,11 @@
+function [name] = get_host_name()
+%Return host name
+%   Written by YJ for I/O
+if isunix() 
+    name = getenv('HOSTNAME');
+else 
+    name = getenv('hostname'); 
+end
+
+end
+

+io/get_user_name.m

@@ -0,0 +1,11 @@
+function [name] = get_user_name()
+%Return account username
+%   Written by YJ for I/O
+if isunix() 
+    name = getenv('USER');
+else 
+    name = getenv('username'); 
+end
+
+end
+

+io/imExportTiff.m

@@ -0,0 +1,78 @@
+function [ ] = imExportTiff( matrix_input,name,section,varargin)
+%Wirte a image stack as a 'Tiff' file
+%imExportTiff( matrix,name,section )
+%   matrix: 3D matrix(y,x,z)
+%   name: output file name
+%   section: choose corss sections
+%           'XY' matrix(:,:,i)
+%           'YZ' matrix(:,i,:)
+%           'ZX' matrix(i,:,:)
+%       
+
+matrix = matrix_input;
+matrix = matrix-min(min(min(matrix)));
+s = size(matrix);
+m = double(max(max(max(matrix))));
+
+if isempty(varargin)  %gray 
+    switch section
+        case 'XY'
+            imwrite(double(matrix(:,:,1))./m, name,'tiff')
+            for i=2:s(3)
+                imwrite(double(matrix(:,:,i))./m, name,'tiff', 'WriteMode','append')
+            end
+        case 'ZY'
+            imwrite(double(squeeze(matrix(:,1,:))./m), name,'tiff')
+            for i=2:s(2)
+                imwrite(double(squeeze(matrix(:,i,:))./m), name,'tiff', 'WriteMode','append')
+            end
+        case 'ZYs'
+            imwrite(mat2gray(double(squeeze(matrix(:,1,:)))), name,'tiff')
+            for i=2:s(2)
+                imwrite(mat2gray(double(squeeze(matrix(:,i,:)))), name,'tiff', 'WriteMode','append')
+            end
+        case 'ZX'
+            imwrite(double(squeeze(matrix(1,:,:)))./m, name,'tiff')
+            for i=2:s(1)
+                imwrite(double(squeeze(matrix(i,:,:)))./m, name,'tiff', 'WriteMode','append')
+            end
+        case 'XYs'
+            imwrite(mat2gray(matrix(:,:,1)), name,'tiff')
+            for i=2:s(3)
+                imwrite(mat2gray(matrix(:,:,i)), name,'tiff', 'WriteMode','append')
+            end
+        case 'ZXs'
+            imwrite(squeeze(matrix(1,:,:))./m, name,'tiff')
+            for i=2:s(1)
+                imwrite(mat2gray(squeeze(matrix(i,:,:))), name,'tiff', 'WriteMode','append')
+            end
+    end
+else
+    a = varargin{1}; %color
+    switch section
+        case 'XY'
+            imwrite(double(matrix(:,:,1))*a*64/m,jet, name,'tiff')
+            for i=2:s(3)
+                imwrite(double(matrix(:,:,i))*a*64/m,jet, name,'tiff', 'WriteMode','append')
+            end
+        case 'ZY'
+            imwrite(squeeze(matrix(:,1,:))*a*64/m,jet, name,'tiff')
+            for i=2:s(2)
+            imwrite(squeeze(matrix(:,i,:))*a*64/m,jet, name,'tiff', 'WriteMode','append')
+            end
+
+        case 'ZX'
+            imwrite(squeeze(matrix(1,:,:))*a*64/m,jet, name,'tiff')
+            for i=2:s(1)
+            imwrite(squeeze(matrix(i,:,:))*a*64/m,jet, name,'tiff', 'WriteMode','append')
+            end
+        case 'XYs'
+            imwrite(mat2gray(matrix(:,:,1))*a*64,parula, name,'tiff')
+            for i=2:s(3)
+                imwrite(mat2gray(matrix(:,:,i))*a*64,parula, name,'tiff', 'WriteMode','append')
+            end    
+    end
+end
+
+end
+

+io/image_default_orientation.m

@@ -0,0 +1,169 @@
+% [orient_vec] = image_default_orientation(header, extension, varargin)
+% Determine default orientation for an image_orient.m call based on the
+% file extension
+
+% Filename: $RCSfile: image_default_orientation.m,v $
+%
+% $Revision: 1.11 $  $Date: 2013/03/23 15:01:09 $
+% $Author:  $
+% $Tag: $
+%
+% Description:
+% Determine default orientation for an image_orient.m call based on the
+% file extension
+%
+% Note:
+% Call without arguments for a brief help text.
+%
+% Dependencies: 
+% ---
+%
+% history:
+%
+% September 30th 2010:
+% add orientation for HDF5 files
+%
+% November 19th 2008:
+% add .mat files
+%
+% August 28th 2008:
+% add orientation for extension .dat
+%
+% July 17th 2008:
+% add mar extension, raw default orientation changed before
+%
+% June 19th 2008:
+% add header to call parameters
+%
+% June 10th 2008: 
+% 1st version
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [orient_vec] = ...
+   image_default_orientation(header, extension, varargin)
+import io.*
+
+% check minimum number of input arguments
+if (nargin < 1)
+    fprintf('Usage:\n')
+    fprintf('[orientation_vector]=%s(extension);\n',...
+        m_file_name);
+    fprintf('The vector contains three values which can be 0 or 1 for transpose, flip-left-right, flip-up-down\n');
+    error('At least one input parameter has to be specified.');
+end
+
+
+% accept cell array with name/value pairs as well
+no_of_in_arg = nargin;
+if (nargin == 3)
+    if (isempty(varargin))
+        % ignore empty cell array
+        no_of_in_arg = no_of_in_arg -1;
+    else
+        if (iscell(varargin{1}))
+            % use a filled one given as first and only variable parameter
+            varargin = varargin{1};
+            no_of_in_arg = 2 + length(varargin);
+        end
+    end
+end
+
+
+% check number of input arguments
+if (rem(no_of_in_arg,2) ~= 0)
+    error('The optional parameters have to be specified as ''name'',''value'' pairs');
+end
+
+
+
+% parse the variable input arguments
+% vararg_remain = cell(0,0);
+for ind = 1:2:length(varargin)
+    name = varargin{ind};
+%     value = varargin{ind+1};
+    switch name
+        otherwise
+            error('Do not know how to handle parameter %s\n',name);
+%             vararg_remain{end+1} = name;
+%             vararg_remain{end+1} = value;
+    end
+end
+
+
+% image_read calls this function prior to converting the single frame
+% to a cell array of frames
+if (~iscell(header))
+    fprintf('Warning (%s): header is not a cell array\n',mfilename);
+    fprintf('If this is an image_spec call then please report to Oliver:\n')
+    whos header
+    header
+else
+    if ((~isempty(header)) && (iscell(header{1})))
+        header = header{1};
+    end
+end
+
+
+% set the default orientation as a function of the filename extension
+switch extension
+    case 'dat'
+        orient_vec = [ 0 0 0 ];
+    case 'edf'
+        orient_vec = [ 1 1 1 ];
+    case 'cbf'
+        orient_vec = [ 1 1 1 ];
+    case {'h5', 'hdf5', 'nxs', 'cxs'}
+        orient_vec = [ 0 0 1 ];        
+    case {'tif', 'tiff'}
+        if (strcmp(header{1}(1:5),'Andor'))
+            orient_vec = [ 1 0 0 ];
+        else
+            orient_vec = [ 0 0 0 ];
+        end
+    case {'mar','mccd'}
+        orient_vec = [ 1 0 1 ];
+    case 'mat'
+        orient_vec = [ 0 0 0 ];
+    case 'raw'
+        % FLI CCD at ICON
+%        orient_vec = [ 0 1 0 ];
+        % FLI CCD at laser setup
+        orient_vec = [ 1 0 1 ];
+    case 'spe'
+        orient_vec = [ 0 0 0 ];
+    otherwise
+        error([ 'unknown extension ''' extension '''' ]);
+end

+io/image_info.m

@@ -0,0 +1,214 @@
+% [im_info,vararg_remain] = image_info(filenames,varargin)
+% Get information like the dimensions of the data stored in an image file
+
+% Filename: $RCSfile: image_info.m,v $
+%
+% $Revision: 1.3 $  $Date: 2013/01/25 10:23:07 $
+% $Author:  $
+% $Tag: $
+%
+% Description:
+% Get information like the dimensions of the data stored in an image file
+%
+% Note:
+% Call without arguments for a brief help text.
+%
+% Dependencies: 
+%
+% history:
+%
+% November 11th 2010:
+% 1st version
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [im_info,vararg_remain] = image_info(filenames,varargin)
+import io.*
+import utils.default_parameter_value
+import utils.find_files
+import utils.fopen_until_exists
+
+% initialize return arguments
+im_info = struct('no_of_frames',[]);
+
+% check minimum number of input arguments
+if (nargin < 1)
+%    image_read_help('ext',mfilename);
+    error('At least the filename has to be specified as input parameter.');
+end
+
+% accept cell array with name/value pairs as well
+no_of_in_arg = nargin;
+if (nargin == 2)
+    if (isempty(varargin))
+        % ignore empty cell array
+        no_of_in_arg = no_of_in_arg -1;
+    else
+        if (iscell(varargin{1}))
+            % use a filled one given as first and only variable parameter
+            varargin = varargin{1};
+            no_of_in_arg = 1 + length(varargin);
+        end
+    end
+end
+
+% check number of input arguments
+if (rem(no_of_in_arg,2) ~= 1)
+    error('The optional parameters have to be specified as ''name'',''value'' pairs');
+end
+
+
+% % set default values for the variable input arguments:
+% % default data type for the returned frames
+% data_type = default_parameter_value(mfilename,'DataType');
+% % recognize file type by file name extension
+% force_file_type = default_parameter_value(mfilename,'ForceFileType');
+% % determine default orientation based on the file name extension
+% orient_by_extension = default_parameter_value(mfilename,'OrientByExtension');
+% % filename is actually a mask that may include wildcards
+% filename_is_fmask = default_parameter_value(mfilename,'IsFmask');
+% % display file name of the file to be loaded
+% display_filename = default_parameter_value(mfilename,'DisplayFilename');
+
+% exit with an error message if unhandled named parameters are left at the
+% end of this macro
+unhandled_par_error = 1;
+filename_is_fmask = 0;
+force_file_type = [];
+display_filename = 0;
+
+% parse the variable input arguments
+vararg = cell(0,0);
+for ind = 1:2:length(varargin)
+    name = varargin{ind};
+    value = varargin{ind+1};
+    switch name
+        case 'ForceFileType'
+            force_file_type = lower(value);
+        case 'OrientByExtension' 
+            orient_by_extension = value;
+        case 'UnhandledParError'
+            unhandled_par_error = value;
+        case 'IsFmask' 
+            filename_is_fmask = value;
+        case 'DisplayFilename' 
+            display_filename = value;
+        otherwise
+            vararg{end+1} = name; %#ok<AGROW>
+            vararg{end+1} = value; %#ok<AGROW>
+    end
+end
+
+
+% initialize the list of unhandled parameters
+vararg_remain = cell(0,0);
+
+% convert the filename to a cell array to use the same loop for single and
+% multiple file names
+if (~iscell(filenames))
+    filenames = { filenames };
+end
+
+% loop over all specified file names
+file_ind_max = length(filenames);
+for (file_ind=1:file_ind_max)
+    filename = filenames{file_ind};
+    vararg_remain = vararg;
+
+    % in case of file name mask get a list of all matching file names
+    data_dir = '';
+    if (filename_is_fmask)
+        % sub macros must not complain about unknown arguments
+        vararg_remain{end+1} = 'UnhandledParError';
+        vararg_remain{end+1} = 0;
+        
+        [data_dir, fnames, vararg_remain] = ...
+            find_files( filename, vararg_remain );
+    else
+        fnames = struct('name',filename);
+    end
+    
+    for (sub_file_ind = 1:length(fnames))
+        % pick out the current filename
+        filename = [ data_dir fnames(sub_file_ind).name ];
+        
+        % check for minimum filename length
+        if (length(filename) < 5)
+            error([ mfilename ': invalid filename ' filename ]);
+        end
+
+        if (isempty(force_file_type))
+            % get the extension from the last three to four characters
+            extension = lower(filename((end-4):end));
+            pos = strfind(extension,'.');
+            if (length(pos) < 1)
+                error([ mfilename ': invalid extension in ' filename ]);
+            end
+            extension = extension(pos(end)+1:end);
+        else
+            % the file name extension is ignored since the file type is
+            % forced to a specific one
+            extension = force_file_type;
+        end
+
+        if (display_filename)
+            fprintf('file information on %s\n',filename);
+        end
+        
+        if ((strcmp(extension,'dat')) || ...
+            (strcmp(extension,'tif')) || (strcmp(extension,'tiff')) || ...
+            (strcmp(extension,'mat')))
+            % open the file to support functionality like 
+            % wait-until-exists
+            [fid,vararg_remain] = fopen_until_exists(filename,vararg);
+            if (fid >= 0)
+              fclose(fid);
+            end        
+        end
+        
+        % interprete file in the format indicated by the filename extension
+        switch extension
+            case {'h5', 'hdf5'}
+                fi = hdf5info(filename);
+                im_info.no_of_frames = fi.GroupHierarchy.Groups.Datasets.Dims(3);
+            case {'cbf', 'dat', 'edf', 'mar', 'mccd', 'mat', 'raw', 'spe', 'tif', 'tiff'}
+                [frame,vararg_remain] = image_read(filename,vararg_remain);
+                im_info.no_of_frames = size(frame.data,3);
+            otherwise
+                error([ 'unknown extension of ' filename ]);
+        end
+    end
+end
+

+io/image_orient.m

@@ -0,0 +1,176 @@
+% Call function without arguments for instructions on how to use it
+
+% Filename: $RCSfile: image_orient.m,v $
+%
+% $Revision: 1.5 $  $Date: 2014/04/11 12:47:09 $
+% $Author:  $
+% $Tag: $
+%
+% Description:
+% Macro for mirroring or rotating images or stacks of images. 
+%
+% Note:
+% Call without arguments for a brief help text.
+% In case of image stacks the first two dimensions are treated as the image
+% dimensions. 
+%
+% Dependencies: 
+% ---
+%
+% history:
+%
+% January 16th 2009:
+% return the complete structure rather than just the data array
+%
+% June 19th 2008:
+% change call to complete frame rather than data only
+%
+% May 27th 2008: 
+% 1st version based on orientm.m by Tilman Donath
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [frame_out,vararg_remain] = image_orient(frame, varargin)
+import io.*
+import utils.default_parameter_value
+
+% check minimum number of input arguments
+if (nargin < 1)
+    image_orient_help(mfilename);
+    error('At least one input parameter has to be specified.');
+end
+
+if (ndims(frame.data) < 2)
+    error('The input data array must have at least two dimensions.');
+end
+
+
+% accept cell array with name/value pairs as well
+no_of_in_arg = nargin;
+if (nargin == 2)
+    if (isempty(varargin))
+        % ignore empty cell array
+        no_of_in_arg = no_of_in_arg -1;
+    else
+        if (iscell(varargin{1}))
+            % use a filled one given as first and only variable parameter
+            varargin = varargin{1};
+            no_of_in_arg = 1 + length(varargin);
+        end
+    end
+end
+
+
+% check number of input arguments
+if (rem(no_of_in_arg,2) ~= 1)
+    error('The optional parameters have to be specified as ''name'',''value'' pairs');
+end
+
+
+% set default values for the variable input arguments:
+orient_extension = frame.extension{1};
+do_transpose = default_parameter_value(mfilename,'Transpose');
+do_fliplr = default_parameter_value(mfilename,'FlipLR');
+do_flipud = default_parameter_value(mfilename,'FlipUD');
+invert_orientation = 0;
+
+% parse the variable input arguments
+vararg_remain = cell(0,0);
+for ind = 1:2:length(varargin)
+    name = varargin{ind};
+    value = varargin{ind+1};
+    switch name
+        case 'Transpose' 
+            do_transpose = value;
+        case 'FlipLR' 
+            do_fliplr = value;
+        case 'FlipUD' 
+            do_flipud = value;
+        case 'Orientation' 
+            if (length(value) ~= 3)
+                error('Invalid Orientation parameter of length %d',length(value));
+            end
+            do_transpose = value(1);
+            do_fliplr = value(2);
+            do_flipud = value(3);
+        case 'OrientExtension' 
+            orient_extension = value;
+        case 'OrientByExtension' 
+            if (value)
+                orient_vec = image_default_orientation(frame.header,orient_extension);
+                do_transpose = orient_vec(1);
+                do_fliplr = orient_vec(2);
+                do_flipud = orient_vec(3);
+            end
+        case 'InvertOrientation'
+            invert_orientation = value;
+        otherwise
+            vararg_remain{end+1} = name;
+            vararg_remain{end+1} = value;
+    end
+end
+
+
+% initialize return arguments
+frame_out = frame;
+
+if (invert_orientation)
+    % apply orientation modifications in inverse order, e.g., to revert the
+    % original orientation prior to writing a file
+    if (do_flipud)
+        frame_out.data = flip(frame_out.data,1);
+    end
+    
+    if (do_fliplr)
+        frame_out.data = flip(frame_out.data,2);
+    end
+end
+
+if (do_transpose)
+    dim_order = 1:ndims(frame_out.data);
+    dim_order(1) = 2;
+    dim_order(2) = 1;
+    frame_out.data = permute(frame_out.data,dim_order);
+end
+
+if (~invert_orientation)
+    if (do_fliplr)
+        frame_out.data = flip(frame_out.data,2);
+    end
+
+    if (do_flipud)
+        frame_out.data = flip(frame_out.data,1);
+    end
+end

+io/image_read.m

@@ -0,0 +1,728 @@
+% Call function without arguments for instructions on how to use it
+
+% Filename: $RCSfile: image_read.m,v $
+%
+% $Revision: 1.17 $  $Date: 2013/01/25 10:23:23 $
+% $Author: $
+% $Tag: $
+%
+% Description:
+% Macro for reading image data formats used at the SLS / cSAXS beamline. 
+% The data are returned in double precision floating point format. 
+%
+% Note:
+% Call without arguments for a brief help text.
+%
+% Dependencies: 
+% - edfread
+% - cbfread
+% - hdf5read
+% - fliread
+% - speread
+% - char_to_cellstr
+%
+% history:
+%
+% September 30th 2010:
+% add a call to hdf5read_main
+%
+% June 5th 2009:
+% disable UhandledParError before calling sub-macros
+%
+% January 16th 2009:
+% adapt to image_orient returning the complete structure rather than just
+% the data array
+%
+% November 19th 2008:
+% add reading of Matlab files
+%
+% September 5th 2008:
+% skip further processing for a frame if it was not possible to read it
+%
+% September 4th 2008:
+% add the rowcol-from field to the frames structure as origin information
+%
+% June 19th 2008: adapt call to image_orient
+% 
+% May 16th 2008: send variable arguments through find files
+%
+% May 9th 2008: 1st version
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group
+%   and Computing Department, Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [frames,vararg_remain] = image_read(filenames,varargin)
+import io.*
+import io.HDF.*
+import io.CBF.*
+import plotting.*
+import utils.char_to_cellstr
+import utils.default_parameter_value
+import utils.find_files
+import utils.fopen_until_exists
+
+% initialize return arguments
+frames = struct('data',[], ...
+    'img_full_size',[], 'rowcol_from',[], ...
+    'no_of_el_read', [], ...
+    'header',[], 'filename',[], 'extension', []);
+
+% check minimum number of input arguments
+if (nargin < 1)
+    image_read_help('ext',mfilename);
+    error('At least the filename has to be specified as input parameter.');
+end
+
+% accept cell array with name/value pairs as well
+no_of_in_arg = nargin;
+if (nargin == 2)
+    if (isempty(varargin))
+        % ignore empty cell array
+        no_of_in_arg = no_of_in_arg -1;
+    else
+        if (iscell(varargin{1}))
+            % use a filled one given as first and only variable parameter
+            varargin = varargin{1};
+            no_of_in_arg = 1 + length(varargin);
+        end
+    end
+end
+
+% check number of input arguments
+if (rem(no_of_in_arg,2) ~= 1)
+    error('The optional parameters have to be specified as ''name'',''value'' pairs');
+end
+
+% convert the filename to a cell array to use the same loop for single and
+% multiple file names
+if (~iscell(filenames))
+    filenames = { filenames };
+end
+
+% hdf5 files are read via a separate sub-routine
+% if length(filenames) == 1 % accept only one file name
+    filename = filenames{1};
+    [~, ~, ext] = fileparts(filename);
+    
+    if any(strcmp(ext, {'.h5', '.hdf5', '.nxs', '.cxs'}))
+        if length(filenames) == 1
+            frames = hdf5read(filename, varargin);
+        else
+            frames = hdf5read(filenames, varargin);
+        end
+        vararg_remain = [];
+        
+        return % image_read ends here for hdf5 image files
+    end
+% end
+
+% set default values for the variable input arguments:
+% default data type for the returned frames
+data_type = default_parameter_value(mfilename,'DataType');
+% recognize file type by file name extension
+force_file_type = default_parameter_value(mfilename,'ForceFileType');
+% from/to row 0 means all rows
+row_from = default_parameter_value(mfilename,'RowFrom');
+row_to = default_parameter_value(mfilename,'RowTo');
+% from/to column 0 means all lines
+column_from = default_parameter_value(mfilename,'ColumnFrom');
+column_to = default_parameter_value(mfilename,'ColumnTo');
+% determine default orientation based on the file name extension
+orient_by_extension = default_parameter_value(mfilename,'OrientByExtension');
+% filename is actually a mask that may include wildcards
+filename_is_fmask = default_parameter_value(mfilename,'IsFmask');
+% display file name of the file to be loaded
+display_filename = default_parameter_value(mfilename,'DisplayFilename');
+% variable to load from Matlab files
+matlab_var = default_parameter_value(mfilename,'MatlabVar');
+
+% exit with an error message if unhandled named parameters are left at the
+% end of this macro
+unhandled_par_error = 1;
+
+% parse the variable input arguments
+vararg = cell(0,0);
+for ind = 1:2:length(varargin)
+    name = varargin{ind};
+    value = varargin{ind+1};
+    switch name
+        case 'DataType'
+            if (~ischar(value))
+                error('The DataType must be string defining a valid Matlab data type.');
+            end
+            data_type = value;
+        case 'ForceFileType'
+            force_file_type = lower(value);
+        case 'MatlabVar' 
+            matlab_var = value;
+        case 'RowFrom' 
+            row_from = value;
+        case 'ROI'
+            if (length(value) ~= 4)
+                error('The ROI parameter needs a vector of length four as argument.');
+            end
+            column_from = value(1);
+            row_from = value(2);
+            column_to = value(3);
+            row_to = value(4);
+        case 'RowTo' 
+            row_to = value;
+        case 'ColumnFrom' 
+            column_from = value;
+        case 'ColumnTo' 
+            column_to = value;
+        case 'OrientByExtension' 
+            orient_by_extension = value;
+        case 'UnhandledParError'
+            unhandled_par_error = value;
+        case 'IsFmask' 
+            filename_is_fmask = value;
+        case 'DisplayFilename' 
+            display_filename = value;
+        otherwise
+            vararg{end+1} = name; %#ok<AGROW>
+            vararg{end+1} = value; %#ok<AGROW>
+    end
+end
+
+
+% initialize the list of unhandled parameters
+vararg_remain = cell(0,0);
+
+% loop over all specified file names
+file_ind_max = length(filenames);
+store_ind = 1;
+for (file_ind=1:file_ind_max)
+    filename = filenames{file_ind};
+    vararg_remain = vararg;
+
+    % in case of file name mask get a list of all matching file names
+    data_dir = '';
+    if (filename_is_fmask)
+        % sub macros must not complain about unknown arguments
+        vararg_remain{end+1} = 'UnhandledParError';
+        vararg_remain{end+1} = 0;
+        
+        [data_dir, fnames, vararg_remain] = ...
+            find_files( filename, vararg_remain );
+    else
+        fnames = struct('name',filename);
+    end
+    
+    for (sub_file_ind = 1:length(fnames))
+        % pick out the current filename
+        filename = [ data_dir fnames(sub_file_ind).name ];
+        
+        % check for minimum filename length
+        if (length(filename) < 5)
+            error([ mfilename ': invalid filename ' filename ]);
+        end
+
+        if (isempty(force_file_type))
+            % get the extension from the last three to four characters
+            extension = lower(filename((end-4):end));
+            pos = strfind(extension,'.');
+            if (length(pos) < 1)
+                error([ mfilename ': invalid extension in ' filename ]);
+            end
+            extension = extension(pos(end)+1:end);
+        else
+            % the file name extension is ignored since the file type is
+            % forced to a specific one
+            extension = force_file_type;
+        end
+
+        if (display_filename)
+            fprintf('loading %s\n',filename);
+        end
+        
+        if ((strcmp(extension,'dat')) || ...
+            (strcmp(extension,'tif')) || (strcmp(extension,'tiff')) || ...
+            (strcmp(extension,'mat')))
+            % open the file to support functionality like 
+            % wait-until-exists
+            [fid,vararg_remain] = fopen_until_exists(filename,vararg);
+            if (fid >= 0)
+              fclose(fid);
+            end        
+        end
+        
+        % interprete file in the format indicated by the filename extension
+        switch extension
+            case 'cbf'
+                [frame,vararg_remain] = cbfread(filename,vararg_remain);
+            case {'hdf5', 'h5', 'nxs', 'cxs'}
+                [frame,vararg_remain] = hdf5read_main(filename,vararg_remain);
+            case 'dat' 
+                [frame,vararg_remain] = datread(filename,vararg_remain);
+            case 'edf'
+                [frame,vararg_remain] = edfread(filename,vararg_remain);
+            case {'mar', 'mccd'}
+                [frame,vararg_remain] = marread(filename,vararg_remain);
+            case 'mat' 
+                tmp_data = load(filename);
+                frame.data = tmp_data.(matlab_var);
+                frame.header = {};
+                % No header information are available. 
+                % Fake exposure time information to avoid problems in other
+                % macros. 
+                frame.header{end+1} = 'Exposure_time 1.0';
+                % add the file modification date to the header
+                dir_entry = dir(filename);
+                frame.header{end+1} = [ 'DateTime ' dir_entry.date ];
+            case 'raw'
+                [frame,vararg_remain] = fliread(filename,vararg_remain);
+            case 'spe'
+                [frame,vararg_remain] = speread(filename,vararg_remain);
+            case {'tif', 'tiff'}
+                % reading higher bit depths than 16bit needs a sufficiently
+                % up-to-date Matlab version
+                frame.data = imread(filename,'tif');
+                hdr = imfinfo(filename);
+                frame.header = {};
+                if (isfield(hdr,'ImageDescription'))
+                    frame.header = char_to_cellstr(hdr.ImageDescription);
+                end
+                if ((isfield(hdr,'Model')) && ...
+                    (strcmp(hdr.Model(1:7),'PILATUS')))
+                    frame.header{end+1} = [ 'DateTime ' hdr.DateTime ];
+                    frame.header{end+1} = [ 'Software ' hdr.Software ];
+                    frame.header{end+1} = [ 'Model ' hdr.Model ];
+                else
+                    % the exposure time is not available
+                    if (isfield(hdr,'exptimesec'))
+                        frame.header{end+1} = [ 'Exposure_time' hdr.exptimesec ];
+                    else
+                        frame.header{end+1} = 'Exposure_time 1.0';
+                    end
+                    if (isfield(hdr,'DateTime'))
+                        frame.header{end+1} = [ 'DateTime ' hdr.DateTime ];
+                    else
+                        if (isfield(hdr,'FileModDate'))
+                            frame.header{end+1} = [ 'DateTime ' hdr.FileModDate ];
+                        else
+                            % add the file modification date to the header
+                            dir_entry = dir(filename);
+                            frame.header{end+1} = ...
+                              [ 'DateTime ' dir_entry.date ];
+                        end
+                    end
+                end
+            otherwise
+                error([ 'unknown extension of ' filename ]);
+        end
+
+        % the remaining code is not needed if no file was read
+        if (isempty(frame.data))
+            continue;
+        end
+        % determine the orientation from the filename extension
+        vararg_remain_prev = vararg_remain;
+        vararg_remain = cell(1,length(vararg_remain)+2);
+        vararg_remain(3:end) = vararg_remain_prev;
+        vararg_remain{1} = 'OrientByExtension';
+        vararg_remain{2} = orient_by_extension;
+        % set the extension since image_orient is called prior
+        % to defining the return variables frames 
+        frame.extension = cell(1,1);
+        frame.extension{1} = extension;
+        
+        % orient image
+        [frame,vararg_remain] = image_orient(frame,vararg_remain);
+        
+        % cut out region of interest
+        full_size = size(frame.data);
+        if ((row_from > 0) || (row_to > 0) ||... 
+            (column_from > 0) || (column_to > 0))
+            if (row_from <= 0)
+                row_from = 1;
+            end
+            if (row_from > size(frame.data,1))
+                error('The RowFrom specification is beyond the maximum value of %d',...
+                    size(frame.data,1));
+            end
+            if (row_to <= row_from)
+                row_to = full_size(1);
+            end
+            if (row_to > full_size(1))
+                error('The RowTo specification is beyond the maximum value of %d',...
+                    full_size(1));
+            end
+
+            if (column_from <= 0)
+                column_from = 1;
+            end
+            if (column_from > full_size(2))
+                error('The ColumnFrom specification is beyond the maximum value of %d',...
+                    full_size(2));
+            end
+            if (column_to <= column_from)
+                column_to = full_size(2);
+            end
+            if (column_to > size(frame.data,2))
+                error('The ColumnTo specification is beyond the maximum value of %d',...
+                    full_size(2));
+            end
+            frame.data = frame.data(row_from:row_to,column_from:column_to,:);
+        end
+
+        % initialize the return array with the now known dimensions
+        if (store_ind == 1)
+            % in case of file name masks or multiple images in one data
+            % file the final array dimensions can only be estimated
+            init_guess = file_ind_max -1 + length(fnames);
+            frames.data = zeros( [ size(frame.data) init_guess ], data_type );
+            frames.rowcol_from = cell(1,init_guess);
+            frames.no_of_el_read = cell(1,init_guess);
+            frames.img_full_size = cell(1,init_guess);
+            frames.filename = cell(1,init_guess);
+            frames.extension = cell(1,init_guess);
+            frames.header = cell(1,init_guess);
+        end
+
+
+        % store the frame(s)
+        if ((size(frame.data,1) ~= size(frames.data,1)) || ...
+            (size(frame.data,2) ~= size(frames.data,2)))
+            error('Expected frame dimension is %d x %d, frame read has %d x %d',...
+                size(frames.data,2),size(frames.data,1),...
+                size(frame.data,2),size(frame.data,1));
+        end
+
+        % in some cases multiple frames are stored in a single file
+        if (ndims(frame.data) == 4)
+            store_ind_to = (store_ind+size(frame.data,4)-1);
+            frames.data(:,:,:,store_ind:store_ind_to) = cast(frame.data,data_type);
+        else
+            store_ind_to = (store_ind+size(frame.data,3)-1);
+            frames.data(:,:,store_ind:store_ind_to) = cast(frame.data,data_type);
+        end
+        
+        % store the filename, extension and the header in the return argument
+        for (ind=store_ind:store_ind_to)
+            frames.img_full_size{ind} = full_size;
+            frames.rowcol_from{ind} = [ row_from column_from ];
+            if (isfield(frame,'no_of_el_read'))
+                frames.no_of_el_read{ind} = frame.no_of_el_read;
+            else
+                frames.no_of_el_read{ind} = size(frame.data,3);
+            end
+            % zero means no ROI, i.e., starting at point (1,1)
+            if (frames.rowcol_from{ind}(1) < 1)
+                frames.rowcol_from{ind}(1) = 1;
+            end
+            if (frames.rowcol_from{ind}(2) < 1)
+                frames.rowcol_from{ind}(2) = 1;
+            end
+            frames.filename{ind} = filename;
+            frames.extension{ind} = extension;
+            frames.header{ind} = frame.header;
+        end
+        
+        % update index to free space in the output arrays
+        store_ind = store_ind_to + 1;
+
+        % exit in case of unhandled named parameters, if this has not been switched
+        % off
+        if ((unhandled_par_error) && (~isempty(vararg_remain)))
+            vararg_remain
+            error('Not all named parameters have been handled.');
+        end
+        % restore parameters for next iteration
+        vararg_remain = vararg_remain_prev;        
+    end
+end
+
+% resize the output arrays in case the initial size is too large
+store_ind = store_ind -1;
+if (size(frames.data,3) > store_ind)
+    frames.data = frames.data(:,:,1:store_ind);
+    frames.img_full_size = frames.img_full_size(1:store_ind);
+    frames.rowcol_from = frames.rowcol_from(1:store_ind);
+    frames.filename = frames.filename(1:store_ind);
+    frames.extension = frames.extension(1:store_ind);
+    frames.header = frames.header(1:store_ind);
+end
+
+function frames = hdf5read(filename, params)
+import utils.fopen_until_exists
+import io.image_default_orientation
+import io.HDF.hdf5_load
+import io.image_orient
+import utils.find_files
+
+frames = struct('data', [], 'img_full_size', [], 'rowcol_from', [], ...
+    'no_of_el_read', [], 'header',[], 'filename',[], 'extension', []);
+
+
+
+p = inputParser;
+p.KeepUnmatched = true;
+p.FunctionName = 'image_read';
+
+addParameter(p, 'H5Location', '/');
+addParameter(p, 'ReadAttr', false);
+
+addParameter(p, 'FrameRange', [1, Inf], @(x) isvector(x) && numel(x) <= 2 && isnumeric(x));
+addParameter(p, 'RowFrom', 1, @(x) isscalar(x) && isnumeric(x));
+addParameter(p, 'RowTo', Inf, @(x) isscalar(x) && isnumeric(x));
+addParameter(p, 'ColumnFrom', 1, @(x) isscalar(x) && isnumeric(x));
+addParameter(p, 'ColumnTo', Inf, @(x) isscalar(x) && isnumeric(x));
+
+addParameter(p, 'OrientByExtension', 1, @(x) isscalar(x) && (isnumeric(x) || islogical(x)));
+addParameter(p, 'Orientation', [0, 0, 0], @(x) isvector(x) && numel(x) == 3 && isnumeric(x));
+addParameter(p, 'InvertOrientation', 0, @(x) isscalar(x) && isnumeric(x));
+addParameter(p, 'Transpose', 0, @(x) isscalar(x) && isnumeric(x));
+addParameter(p, 'FlipLR', 0, @(x) isscalar(x) && isnumeric(x));
+addParameter(p, 'FlipUD', 0, @(x) isscalar(x) && isnumeric(x));
+addParameter(p, 'CatDim', -1, @(x) isscalar(x) && isnumeric(x));
+
+addParameter(p, 'DisplayFilename', 1, @(x) isscalar(x) && isnumeric(x));
+
+% No support for filename wildcards, i.e. ('IsMask', 1)
+addParameter(p, 'IsFmask', true, ...
+    @islogical);%@(x) assert(~x, 'Filename wildcards, i.e. (''IsFmask'', 1), are not supported for hdf5 files'));
+
+parse(p, params{:});
+r = p.Results;
+vararg_remain = [fieldnames(p.Unmatched)'; struct2cell(p.Unmatched)'];
+
+if (r.IsFmask)    
+    [data_dir, fnames, vararg_remain] = ...
+        find_files( filename, vararg_remain );
+    filename = [];
+    for ii=1:length(fnames)
+        filename{ii} = fullfile(data_dir, fnames(ii).name);
+    end
+end
+
+
+if iscell(filename)
+    dir_entry = dir(filename{1});
+    frames.header{1} = {['DateTime ' dir_entry.date], 'Exposure_time 0.0'};
+    [~, ~, frames.extension{1}] = fileparts(filename{1});
+    frames.extension{1} = frames.extension{1}(2:end); % truncate a leading dot
+    frames.filename{1} = '*multiple_frames*';
+else
+    dir_entry = dir(filename);
+    frames.header{1} = {['DateTime ' dir_entry.date], 'Exposure_time 0.0'};
+    [~, frames.filename{1}, frames.extension{1}] = fileparts(filename);
+    frames.extension{1} = frames.extension{1}(2:end); % truncate a leading dot
+end
+
+
+inputs_orient = {'OrientByExtension', r.OrientByExtension, 'InvertOrientation', r.InvertOrientation};
+
+if r.OrientByExtension
+    orient_vec = image_default_orientation(frames.header{1}, frames.extension{1});
+    do_transpose = orient_vec(1);
+    do_fliplr = orient_vec(2);
+    do_flipud = orient_vec(3);
+    
+    % Warn user if they also provided either of 'Orientation', 'Transpose', 'FlipLR' or 'FlipUD'
+    if ~all(ismember({'Orientation', 'Transpose', 'FlipLR', 'FlipUD'}, p.UsingDefaults))
+        warning(['Default hdf5 image orientation is potentially modified by either ' ...
+            '''Orientation'', or any of ''Transpose'', ''FlipLR'', or ''FlipUD'' parameters. ' ...
+            'To supress this warning set ''OrientByExtension'' to 0.']);
+    end
+    
+else
+    do_transpose = 0;
+    do_fliplr = 0;
+    do_flipud = 0;
+end
+
+if ~ismember({'Orientation'}, p.UsingDefaults)
+    do_transpose = r.Orientation(1);
+    do_fliplr = r.Orientation(2);
+    do_flipud = r.Orientation(3);
+    
+    inputs_orient = [inputs_orient, {'Orientation', r.Orientation}];
+    
+    % Warn user if they also provided either of 'Transpose', 'FlipLR' or 'FlipUD'
+    if ~all(ismember({'Transpose', 'FlipLR', 'FlipUD'}, p.UsingDefaults))
+        warning(['Image orientation specified via ''Orientation'' parameter is potentially ' ...
+            'modified by either ''Transpose'', ''FlipLR'', and/or ''FlipUD''. ' ...
+            'To supress this warning use either ''Orientation'' or a combination of ' ...
+            '''Transpose'', ''FlipLR'', and/or ''FlipUD'' parameters.']);
+    end
+end
+
+if r.Transpose || r.FlipLR || r.FlipUD
+    do_transpose = r.Transpose;
+    do_fliplr = r.FlipLR;
+    do_flipud = r.FlipUD;
+    inputs_orient = [inputs_orient, {'Transpose', r.Transpose}];
+    inputs_orient = [inputs_orient, {'FlipLR', r.FlipLR}];
+    inputs_orient = [inputs_orient, {'FlipUD', r.FlipUD}];
+end
+
+inputs = {};
+if ~isempty(r.H5Location) && ischar(r.H5Location)
+    inputs{end+1} = r.H5Location;
+end
+
+if r.ReadAttr
+    inputs{end+1} = '-sa';
+end
+
+% Add slicing indexes if a user specified any of them
+if ~all(ismember({'FrameRange', 'RowFrom', 'RowTo', 'ColumnFrom', 'ColumnTo'}, ...
+        p.UsingDefaults))
+    
+    % Support 0's as start/end index -> full left/right range
+    if numel(r.FrameRange) == 1
+        r.FrameRange(2) = r.FrameRange(1);
+    end
+    if r.FrameRange(1) == 0; r.FrameRange(1) = 1; end
+    if r.FrameRange(2) == 0; r.FrameRange(2) = Inf; end
+    if r.RowFrom == 0; r.RowFrom = 1; end
+    if r.RowTo == 0; r.RowTo = Inf; end
+    if r.ColumnFrom == 0; r.ColumnFrom = 1; end
+    if r.ColumnTo == 0; r.ColumnTo = Inf; end
+    
+    % Adjust range values according to the consequent image orientation procedure
+    if r.InvertOrientation % Transpose -> FlipLR/FlipUD
+        if do_transpose; [r.ColumnTo, r.ColumnFrom, r.RowTo, r.RowFrom] = ...
+                deal(r.RowTo, r.RowFrom, r.ColumnTo, r.ColumnFrom); end
+        if do_fliplr; [r.ColumnTo, r.ColumnFrom] = deal(-r.ColumnFrom, -r.ColumnTo); end
+        if do_flipud; [r.RowTo, r.RowFrom] = deal(-r.RowFrom, -r.RowTo); end
+        
+    else % FlipLR/FlipUD -> Transpose
+        if do_fliplr; [r.ColumnTo, r.ColumnFrom] = deal(-r.ColumnFrom, -r.ColumnTo); end
+        if do_flipud; [r.RowTo, r.RowFrom] = deal(-r.RowFrom, -r.RowTo); end
+        if do_transpose; [r.ColumnTo, r.ColumnFrom, r.RowTo, r.RowFrom] = ...
+                deal(r.RowTo, r.RowFrom, r.ColumnTo, r.ColumnFrom); end
+    end
+    
+    % Form the input
+    inputs{end+1} = {[r.RowFrom, r.RowTo],[r.ColumnFrom, r.ColumnTo], r.FrameRange};
+end
+if iscell(filename)
+    
+    if (r.DisplayFilename)
+        fprintf('loading %s\n', filename{1});
+    end
+    tmp = frames;
+    tmp.data = hdf5_load(filename{1}, inputs{:});
+    % Orient image frame(s)
+    [tmp, vararg_remain] = image_orient(tmp, [inputs_orient, vararg_remain]);
+    
+    % let's handle the 2D (or 3d with singleton) case first
+    if isnumeric(tmp.data) && ndims(tmp.data==3) && size(tmp.data,3)==1
+        frames.data = zeros([size(tmp.data(:,:,1)) length(filename)*size(tmp.data,3)]);
+        frames.data(:,:,1:size(tmp.data,3)) = tmp.data;
+        if length(filename)>1
+            for frame=2:length(filename)
+                if (r.DisplayFilename)
+                    fprintf('loading %s\n', filename{frame});
+                end
+                tmp.data = hdf5_load(filename{frame}, inputs{:});
+                
+                % Orient image frame(s)
+                [tmp, vararg_remain] = image_orient(tmp, [inputs_orient, vararg_remain]);
+                frames.data(:,:,frame) = tmp.data;
+            end
+        end
+    else
+        % in case of more than 2 dimensions, concatenate along the
+        % specified dimension, or return a cell array
+        if length(filename)>1
+            frames.data{1} = tmp.data;
+            framedim = ndims(frames.data{1});
+            for frameID=2:length(filename)
+                if (r.DisplayFilename)
+                    fprintf('loading %s\n', filename{frameID});
+                end
+                frames.data{frameID} = hdf5_load(filename{frameID}, inputs{:});
+                if ndims(frames.data{frameID})~=framedim
+                    framedim = -1;
+                end
+                % Orient image frame(s)
+                [frames, vararg_remain] = image_orient(frames, [inputs_orient, vararg_remain]);
+            end
+            if framedim > 0
+                try
+                    if r.CatDim == -1
+                        frames.data = cat(ndims(frames.data{1}),frames.data{:});
+                    else
+                        frames.data = cat(r.CatDim, frames.data{:});
+                    end
+                catch
+                    warning('Failed to concatenate frames.')
+                end
+            end
+                    
+        else
+            frames.data = tmp.data;
+        end
+    end
+else
+    % Support wait-until-exist functionality
+    [fid, vararg_remain] = fopen_until_exists(filename, vararg_remain(:));
+    if fid >= 0
+        fclose(fid);
+    else
+        % Silently exit if a file was not found and ('ErrorIfNotFound', 0)
+        return;
+    end
+    
+    
+    if (r.DisplayFilename)
+        fprintf('loading %s\n', filename);
+    end
+    frames.data = hdf5_load(filename, inputs{:});
+    
+    % Orient image frame(s)
+    [frames, vararg_remain] = image_orient(frames, [inputs_orient, vararg_remain]);
+    
+end
+
+% If its a dataset then fill these fields for further showing with
+% image_show.m or image_spec.m
+if isnumeric(frames.data)
+    frames.img_full_size = {[size(frames.data, 1), size(frames.data, 2)]};
+    frames.rowcol_from = {[r.RowFrom, r.ColumnFrom]};
+    frames.no_of_el_read = {size(frames.data, 3)};
+elseif (do_fliplr||do_flipud||do_transpose)
+    warning(['H5Location points to a group, not a dataset. Orientation/OrientByExtension/Transpose/FlipUD/FlipLR will be ignored. \n '...
+    'To remove this warning, set ''OrientByExtension'' to 0 and ''Orientation'' to [0 0 0]'])
+end
+
+% Show a warning message for unsupported input parameters
+unmatched = vararg_remain(1:2:end);
+if ~isempty(unmatched)
+    warning('These input parameters are not supported for hdf5 files and will be ignored: %s', ...
+        strjoin(unmatched, ', '));
+end

+io/image_read_help.m

@@ -0,0 +1,161 @@
+% image_read_help(extension,m_file_name,varargin)
+% parameter help for image_read
+
+% Filename: $RCSfile: image_read_help.m,v $
+%
+% $Revision: 1.3 $  $Date: 2013/01/25 10:23:47 $
+% $Author:  $
+% $Tag: $
+%
+% Description:
+% parameter help for image_read
+%
+% Note:
+% none
+%
+% Dependencies:
+% none
+%
+%
+% history:
+%
+% July 17th 2008: 
+% add ForceFileType parameter and support for MAR CCD TIFF
+%
+% May 9th 2008: 1st version
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [] = image_read_help(extension,m_file_name,varargin)
+import io.*
+
+% check minimum number of input arguments
+if (nargin < 2)
+    error('At least the extension and m-file name have to be specified as input parameter.');
+end
+
+% accept cell array with name/value pairs as well
+no_of_in_arg = nargin;
+if (nargin == 3)
+    if (isempty(varargin))
+        % ignore empty cell array
+        no_of_in_arg = no_of_in_arg -1;
+    else
+        if (iscell(varargin{1}))
+            % use a filled one given as first and only variable parameter
+            varargin = varargin{1};
+            no_of_in_arg = 2 + length(varargin);
+        end
+    end
+end
+
+% check number of input arguments
+if (rem(no_of_in_arg,2) ~= 0)
+    error('The optional parameters have to be specified as ''name'',''value'' pairs');
+end
+
+% parse the variable input arguments
+examples = 1;
+vararg = cell(0,0);
+for ind = 1:2:length(varargin)
+    name = varargin{ind};
+    value = varargin{ind+1};
+    switch name
+        case 'Examples' 
+            examples = value;
+        otherwise
+            % pass unknown parameters to image_read_sub_help
+            vararg{end+1} = name;
+            vararg{end+1} = value;
+    end
+end
+
+% do not display examples from image_read_sub_help
+vararg{end+1} = 'Examples';
+vararg{end+1} = 0;
+
+image_read_sub_help(m_file_name,extension,vararg)
+fprintf('''DataType'',<Matlab class>            default is ''double'', other possibilities are ''single'', ''uint16'', ''int16'', ''uint32'', etc.\n');
+fprintf('                                     The conversion is done using ''cast'', i.e, out-of-range values are mapped to the minimum or maximum value\n');
+fprintf('''ForceFileType'',<''extension''>        force the file types to be recognized by the here specified extension,\n');
+fprintf('                                     useful in case of no or other types of extensions, used by default as OrientExtension as well.\n');
+fprintf('                                     The extension ''mar'' and ''mccd'' can be used to read MAR CCD TIFF data.\n');
+fprintf('''RowFrom'',<0-max>                    region of interest definition, 0 or 1 for full frame\n');
+fprintf('''RowTo'',<0-max>                      region of interest definition, 0 for full frame\n');
+fprintf('''ColumnFrom'',<0-max>                 region of interest definition, 0 or 1 for full frame\n');
+fprintf('''ColumnTo'',<0-max>                   region of interest definition, 0 for full frame\n');
+image_orient_help(m_file_name,'ParametersOnly',1);
+fprintf('''IsFmask'',<0-no,1-yes>               interprete the filename(s) as search mask that may include wildcards, default true\n');
+fprintf('''DisplayFilename'',<0-no,1-yes>       display filename of a file before loading it, default yes\n');
+fprintf('''UnhandledParError'',<0-no,1-yes>     exit in case not all named parameters are used/known, default is yes\n');
+fprintf('\n');
+fprintf('HDF5, H5 or NeXus specifics           These files contain data and metadata hierarchically organized in groups and datasets,\n');
+fprintf('                                      each group or dataset can also have attributes. Such files are thus here treated in a special way.\n');
+fprintf('                                      If you provide only filename then the file contents, including links but excluding attributes,\n');
+fprintf('                                      will be recursively read and returned as a Matlab structure. See also hdf5_load.m\n');
+fprintf('''H5Location'',<location>               If <location> is a group then it will be read recursively and returned as a Matlab structure.\n');
+fprintf('                                      If <location> is a dataset, the dataset will be read and returned within the field ''data'',\n');
+fprintf('                                      this is done in an effort to be compatible with the output of image_read for other file extensions. \n');
+fprintf('                                      Only in this case the data region options will be used, e.g ''RowFrom'', ''RowTo'', etc. \n');
+fprintf('''FrameRange'',<[first_fr last_fr]>     Read only a subset of the frames available in the HDF5 file dataset specifed with ''H5Location''\n');
+fprintf('                                      This will only have an effect if ''H5Location'' is a dataset and not a group \n');
+fprintf('''ReadAttr'',<0-no,1-yes>               Read the attributes of a dataset or group (default 0). The Name and Value of the attributes are \n');
+fprintf('                                      returned in a structure. Note with this option only the attributes (and not the dataset) are read\n');
+
+if (examples)
+    fprintf('\n');
+    fprintf('\n');
+    fprintf('Examples:\n');
+    fprintf('[frame]=%s(''~/Data10/pilatus/image_1_ct.cbf'');\n',...
+        m_file_name);
+    fprintf('[frame]=%s({''~/Data10/pilatus/image_1_ct1.cbf'',''~/Data10/pilatus/image_1_ct2.cbf''});\n',...
+        m_file_name);
+    fprintf('[frame]=%s(''~/Data10/pilatus/S00010/*.cbf'',''IsFmask'',1);\n',...
+        m_file_name);
+    fprintf('[frame]=%s(''~/Data10/pilatus/image_1_ct.cbf'',''RowFrom'',500,''RowTo'',600);\n',...
+        m_file_name);
+    fprintf('\n');
+    fprintf('The returned structure has the fields data, header, filename and extension.\n');
+    fprintf('\n');
+    fprintf('\n');
+    fprintf('Examples for HDF5:\n');
+    fprintf('[data] = image_read(''scan_00300.hdf5'')                                               Read all data in the file.\n')
+    fprintf('[data] = image_read(''scan_00300.hdf5'',''H5Location'',''/entry/instrument'')              Reads NeXus instrument group.\n')
+    fprintf('[data] = image_read(''scan_00300.hdf5'',''H5Location'',''/entry/collection/data/spec'')    Reads spec data which includes counters and motors that change during a scan.\n')   
+    fprintf('[data] = image_read(''scan_00300.hdf5'',''H5Location'',''/entry/collection/data/spec'',''ReadAttr'',1)      Reads spec data that did not change during the scan, e.g. static motors.\n')   
+    fprintf('[data] = image_read(''scan_00300.hdf5'',''H5Location'',''/entry/instrument/Pilatus_2M/data'')             Reads all Pilatus frames from the scan.\n')   
+    fprintf('[data] = image_read(''scan_00300.hdf5'',''H5Location'',''/entry/instrument/Pilatus_2M/data'',''FrameRange'',[5 10], ''RowFrom'',500,''RowTo'',Inf,''ColumnFrom'',200,''ColumnTo'',800 )\n')   
+    fprintf('                                      Reads the specified frame range and region of interest of the pilatus frames.\n')   
+end

+io/json2mat.m

@@ -0,0 +1,159 @@
+function M=json2mat(J)
+import io.*
+%JSON2MAT converts a javscript data object (JSON) into a Matlab structure
+%         using s recursive approach. J can also be a file name.
+%
+%Example: lala=json2mat('{lele:2,lili:4,lolo:[1,2,{lulu:5,bubu:[[1,2],[3,4],[5,6]]}]}') 
+%         notice lala.lolo{3}.bubu is read as a 2D matrix.
+%
+% Jonas Almeida, March 2010
+
+% Copyright (c) 2010, Jonas Almeida
+% All rights reserved.
+% 
+% Redistribution and use in source and binary forms, with or without
+% modification, are permitted provided that the following conditions are
+% met:
+% 
+% * Redistributions of source code must retain the above copyright
+% notice, this list of conditions and the following disclaimer.
+% * Redistributions in binary form must reproduce the above copyright
+% notice, this list of conditions and the following disclaimer in
+% the documentation and/or other materials provided with the distribution
+% 
+% THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+% AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+% IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+% ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+% LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+% CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+% SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+% INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+% CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+% ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+% POSSIBILITY OF SUCH DAMAGE.
+
+if exist(J)==2 % if J is a filename
+    fid=fopen(J,'r');
+    J='';
+    while ~feof(fid)
+        J=[J,fgetl(fid)];
+    end
+    fclose(fid);
+    M=json2mat(J);  
+else
+    J1=regexprep(J(1:min([5,length(J)])),'\s',''); %despaced start of J string
+    if J1(1)=='{' %extract structures
+        JJ=regexp(J,'\{(.*)\}','tokens');
+        M=extract_struct(JJ{1}{1});
+    elseif J1(1)=='[' %extract cells
+        JJ=regexp(J,'\[(.*)\]','tokens');
+        M=extract_cell(JJ{1}{1});
+    elseif J1(1)=='"' %literal string
+        JJ=regexp(J,'\"(.*)\"','tokens');
+        M=JJ{1}{1};
+    else %numeric value
+        M=str2num(J); % is number
+    end
+end
+
+function y=extract_struct(x)
+import io.*
+
+%detag arrays first
+indOC=extract_embed(x,'[',']');
+n=size(indOC,1);
+for i=n:-1:1
+    tag{i}=json2mat(x(indOC(i,1):indOC(i,2)));
+    x=[x(1:indOC(i,1)-1),'tag{',num2str(i),'}',x(indOC(i,2)+1:end)];
+end
+
+
+a=regexp(x,'[^:,]+:[^,]+');
+n=length(a);
+a=[a,length(x)+2];
+for i=1:n
+    s=x(a(i):a(i+1)-2);
+    t=regexp(s,'([^:]+):(.+)','tokens');
+    %t{1}{1}(t{1}{1}==32)=[]; % remove blanks, maybe later do something fancier like replace with underscores
+    t{1}{1}=strrep(t{1}{1},' ','_');
+    t{1}{1}=strrep(t{1}{1},'"','');
+    if t{1}{1}(1)=='_' %JSON allows for fieldnames starting with "_"
+        t{1}{1}(1)=''; % this line will cause hard to track problems if the same object has 2 attributes with the same name but one of them starting with "_"
+    end        
+    if regexp(t{1}{2},'tag{\d+}')
+        y.(t{1}{1})=eval(t{1}{2});
+    else
+        y.(t{1}{1})=json2mat(t{1}{2});
+    end
+    %y.(t{1}{1})=json2mat(t{1}{2});
+end
+
+function y=extract_cell(x)
+import io.*
+
+indOC=extract_embed(x,'{','}');
+n=size(indOC,1);
+for i=n:-1:1
+    tag{i}=json2mat(x(indOC(i,1):indOC(i,2)));
+    x=[x(1:indOC(i,1)-1),'tag~<',num2str(i),'>~',x(indOC(i,2)+1:end)];
+end
+indOC=extract_embed(x,'[',']');
+m=size(indOC,1);
+for j=m:-1:1
+    i=n+j;
+    tag{i}=json2mat(x(indOC(i,1):indOC(i,2)));
+    try;tag{i}=cell2mat(tag{i});end
+    x=[x(1:indOC(i,1)-1),'tag{',num2str(i),'}',x(indOC(i,2)+1:end)];
+end
+x=strrep(x,'~<','{');
+x=strrep(x,'>~','}');
+if exist('tag') %catching numeric content
+    if isnumeric([tag{:}])
+        try
+            y=eval(['[',strrep(x,'},','};'),']']);
+        end
+    end
+end
+
+if exist('y')~=1
+    y=eval(['{',strrep(x,'"',''''),'}']);
+end
+
+%look for embeded objects and arrays
+
+function y=extract_embed(x,tagOpen,tagClose)
+import io.*
+
+%EXTRACT_EMBED identifies embeded tagged segments
+%Example y=extract_embed(str,'[',']')
+
+indOpen=strfind(x,tagOpen)';
+indOpen=[indOpen,ones(length(indOpen),1)];
+indClose=strfind(x,tagClose)';
+indClose=[indClose,-ones(length(indClose),1)];
+indOpenClose=[indOpen;indClose];
+[~,Ind]=sort(indOpenClose(:,1));
+indOpenClose=indOpenClose(Ind,:);
+n=size(indOpenClose,1);
+for i=2:n % add one for open, take one for close
+    indOpenClose(i,2)=indOpenClose(i-1,2)+indOpenClose(i,2);
+end
+i=0;
+op=0; %open
+while i<n
+    i=i+1;
+    if (indOpenClose(i,2)==1)*(op==0)
+        op=1;
+    elseif indOpenClose(i,2)==0
+        op=0;
+    else
+        indOpenClose(i,2)=-1;
+    end
+end
+if isempty(indOpenClose)
+    y=[];
+else
+    indOpenClose(indOpenClose(:,2)<0,:)=[];
+    y=[indOpenClose(1:2:end,1),indOpenClose(2:2:end,1)];% Open/Close Indexes
+end

+io/load_prepared_data.m

@@ -0,0 +1,321 @@
+%LOAD_PREPARED_DATA Load prepared data file and convert it into the default Matlab
+%structure
+%
+%   filename             path and filename of the h5 file
+%
+%   *optional*
+%   return_intensity     return intensity or magnitude; default false (= return magnitude)
+%   scan                 select scan, either integer or array
+%   enum                 return only selected frames 
+%   return_fftshifted    return results fftshifted, default == true 
+%
+%   *returns*: 
+%   fmag                fourier magnitudes of the measured data, ie fftshift(sqrt(data))
+%   fmask               mask of the fourier magnitudes, 1 for bad pixels, 0 for other
+%   pos                 scanning positions (Npos x 2 array)
+%   max_power           maximal intesity (max(sum(sum(fmag,1),2),[],3) / numel(fmag(:,:,1));)
+%   scanindexrange      indices corresponding to each of the scans 
+%   max_sum             something stored in h5_data.measurement.(['n' num2str(ii-1)]).Attributes.max_sum;
+%
+%   Examples:
+%       [fmag, fmask, ~] = load_prepared_data('~/Data10/analysis/S00668/S00668_S00669_data_400x400.h5');
+%       [fmag, fmask, pos] = load_prepared_data('~/Data10/analysis/S00668/S00668_S00669_data_400x400.h5');
+%
+%       % load intensities
+%       [I, ~, ~] = load_prepared_data('~/Data10/analysis/S00668/S00668_S00669_data_400x400.h5', true);
+%   
+%       % load data from second scan
+%       [fmag, fmask, pos] = load_prepared_data('~/Data10/analysis/S00668/S00668_S00669_data_400x400.h5', false, 2);
+%
+%       % load data from scan 1 and 3
+%       [fmag, fmask, pos] = load_prepared_data('~/Data10/analysis/S00668/S00668_S00669_data_400x400.h5', false, [1 3]);
+
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [ fmag, fmask, pos, max_power, scanindexrange, max_sum ] = load_prepared_data( filename, return_intensity, enum, return_fftshifted )
+import io.HDF.hdf5_load
+
+if nargin < 2
+    return_intensity = false;  % return intensity as measured by detector 
+end
+if nargin < 3
+    enum = [];                 % return only selected frames 
+end
+if nargin < 4
+    return_fftshifted = true;  % return data fftshifted, !! DEFAULT == true !! 
+end
+
+    
+if ~exist(filename, 'file'); error('Cannot load %s', filename); end
+
+%% compatility to load also old matlab datasets 
+[~,~,ext]=fileparts(filename); 
+if strcmpi(ext, '.mat')
+    d = load(filename); 
+    fmag = d.data; 
+    fmask = d.fmask; 
+    if ~return_intensity
+        % normalize to provide similar data as in h5-mex
+        max_power = max(sum(sum(fmag,1),2),[],3) / numel(fmag(:,:,1));
+        renorm = sqrt(1/max_power);
+        fmag = sqrt(fmag)*renorm;
+    end
+    if return_fftshifted
+        fmag = math.fftshift_2D(fmag); 
+        fmask = math.fftshift_2D(fmask); 
+    end
+    return
+end
+    
+    
+%% load h5 file
+recon  = false;
+inf = h5info(filename);
+for ii=1:numel(inf.Groups)
+    if strcmpi(inf.Groups(ii).Name, '/reconstruction')
+        recon = true;
+    end
+end
+
+if recon
+    h5_data = hdf5_load(filename, '/measurement/data', '-a');
+else
+    h5_data = hdf5_load(filename, '-a');
+end
+
+if isfield(h5_data, 'reconstruction')
+    h5_data = h5_data.measurement.data;
+end
+
+%% check hdf5 data verion (python or mex data prep?)
+if isfield(h5_data, 'measurements')
+    h5_version = 'mex';
+else
+    h5_version = 'LibDetXR';
+end
+
+switch h5_version
+    case 'mex'
+        warning('Outdated data format.')
+        asize = size(h5_data.measurements.measurement_0.diff_pat.Value);
+        fn = fieldnames(h5_data.measurements);
+        numpts = length(fn)-1;
+        fmag = zeros(asize(1), asize(2), numpts);
+        fmask = ones(asize(1), asize(2), numpts);
+        pos = zeros(numpts, 2);
+        
+        if isempty(enum) 
+            enum = 1:length(fieldnames(h5_data.detectors))-1;
+        end
+        
+        for ii=enum 
+            
+            fmaskdet{ii} = zeros(asize(1),asize(2));
+            modules = transpose(h5_data.detectors.(['detector_' num2str(ii-1)]).modules.Value);
+            
+            numrows = modules(:,1);
+            numcols = modules(:,2);
+            indbeginmody = modules(:,3)+1;
+            indbeginmodx = modules(:,4)+1;
+            indendmody = numrows + indbeginmody -1;
+            indendmodx = numcols + indbeginmodx -1;
+            nummody = length(indbeginmody);
+            nummodx = length(indbeginmodx);
+            
+            for kk = 1:nummody
+                for jj = 1:nummodx
+                    fmaskdet{ii}(indbeginmody(kk):indendmody(kk),indbeginmodx(jj):indendmodx(jj))=1;
+                end
+            end
+        end
+        scanindexrange = ones([2 length(enum)]);
+        cid=1;
+        for ii=1:numpts
+            cf =['measurement_' num2str(ii-1)];
+            det = h5_data.measurements.(cf).Attributes.detector;
+            if any(enum==det+1)
+                if ii>scanindexrange(2,det+1)
+                    scanindexrange(2,det+1) = ii;
+                end
+
+                fmag(:,:,cid) = transpose(h5_data.measurements.(cf).diff_pat.Value);
+                if isfield(h5_data.measurements.(cf), 'bad_pixels')
+                    bp_temp = h5_data.measurements.(cf).bad_pixels.Value;
+                else
+                    bp_temp = [];
+                end
+                pos(cid, :) = h5_data.measurements.(cf).Attributes.position;
+                fmask(:,:,cid) = fmaskdet{det+1};
+                if ~isempty(bp_temp)
+                    for kk=1:size(bp_temp,2)
+                        fmask(bp_temp(1,kk)+1,bp_temp(2,kk)+1,cid) = 0;
+                    end
+                end
+                cid = cid + 1;
+            end
+            
+        end
+        fmag = fmag(:,:,1:cid-1);
+        fmask = fmask(:,:,1:cid-1);
+        pos = pos(1:cid-1,:);
+%         attr = hdf5_load(filename, '/measurements/','-sa');
+        max_power = h5_data.measurements.Attributes.max_power;
+        fmask = logical(fmask);
+        if return_intensity
+            renorm = sqrt(1/max_power);
+            fmag = (fmag/renorm).^2;
+        end
+        
+        scanindexrange = scanindexrange';
+        for ii=2:length(enum)
+            scanindexrange(ii,1) = scanindexrange(ii-1,2)+1;
+        end
+
+
+
+            
+        
+        
+    case 'LibDetXR'
+        
+        %% get data dims
+        fmag_dim(1) = 0;
+        fmag_dim(2) = 1;
+        if isempty(enum) 
+            enum = 1:length(fieldnames(h5_data.measurement))-1;
+        end
+        
+        for ii=enum
+            fmag_temp{ii} = h5_data.measurement.(['n' num2str(ii-1)]).data.Value;
+            fmag_dim(ii+2) = size(fmag_temp{ii},3);
+        end
+        
+        asize = size(fmag_temp{enum(1)});
+        if asize(1) ~= asize(2)
+            error('Loading of asymmetric prepated datasets not supported, use p.force_preparation_data=true')
+        end
+        
+        fmag = zeros(asize(1), asize(2), sum(fmag_dim)-1, 'single');
+        fmask = ones(asize(1), asize(2), sum(fmag_dim)-1, 'logical');
+        pos = zeros(sum(fmag_dim)-1, 2);
+        max_sum = zeros(length(enum), 1);
+        
+        %% load modules to prepare fmask and load everything into containers
+        if isempty(enum) 
+            enum = 1:length(fieldnames(h5_data.detector));
+        end
+        
+        for ii=enum        
+            % get modules for mask
+            fmaskdet{ii} = zeros(asize(1),asize(2), 'logical');
+            modules = transpose(h5_data.detector.(['n' num2str(ii-1)]).modules.Value);
+            
+            numrows = modules(:,1);
+            numcols = modules(:,2);
+            indbeginmody = modules(:,3)+1;
+            indbeginmodx = modules(:,4)+1;
+            indendmody = numrows + indbeginmody -1;
+            indendmodx = numcols + indbeginmodx -1;
+            nummody = length(indbeginmody);
+            nummodx = length(indbeginmodx);
+            
+            for kk = 1:nummody
+                for jj = 1:nummodx
+                    fmaskdet{ii}(indbeginmody(kk):indendmody(kk),indbeginmodx(jj):indendmodx(jj))=1;
+                end
+            end
+            temp_range = sum(fmag_dim(1:ii+1)):sum(fmag_dim(1:ii+2))-1;
+            fmask(:,:,temp_range) = repmat(fmaskdet{ii},[1,1,fmag_dim(ii+2)]);
+            
+            % get bad pixels
+            if isfield(h5_data.detector.(['n' num2str(ii-1)]), 'bad_pixels')
+                bp = h5_data.detector.(['n' num2str(ii-1)]).bad_pixels.Value;
+                for kk=1:size(bp,2)
+                    fmask(bp(1,kk)+1,bp(2,kk)+1,temp_range) = 0;
+                end
+            else
+                if isfield(h5_data.measurement.(['n' num2str(ii-1)]), 'bad_pixels')
+                    bp = h5_data.measurement.(['n' num2str(ii-1)]).bad_pixels.Value;
+                    bpi = h5_data.measurement.(['n' num2str(ii-1)]).bad_pixels_index.Value;
+                    assert(length(bpi)==length(temp_range), 'Number of frames does not match the number of bad pixel datasets.')
+                    offset = 1;
+                    for kk=1:length(bpi)
+                        for jj=offset:bpi(kk)
+                            fmask(bp(1,jj)+1, bp(2,jj)+1, temp_range(kk)) = 0;
+                        end
+                        offset = bpi(kk);
+                    end
+                end
+            end
+            fmag(:,:,temp_range) = permute(fmag_temp{ii}, [2 1 3]);
+            pos_temp = h5_data.measurement.(['n' num2str(ii-1)]).positions.Value;
+            pos(temp_range,1) = pos_temp(1,:);
+            pos(temp_range,2) = pos_temp(2,:);
+            
+            max_sum(enum) = h5_data.measurement.(['n' num2str(ii-1)]).Attributes.max_sum;
+
+            
+        end
+        
+        
+        % normalize to provide similar data as in h5-mex
+        max_power = max(sum(sum(fmag,1),2),[],3) / numel(fmag(:,:,1));
+        renorm = sqrt(1/max_power);
+        if ~return_intensity
+            fmag = sqrt(fmag)*renorm;
+        end
+        fmask = logical(fmask);
+        
+        scanindexrange = zeros(numel(enum),2);
+        scanindexrange(1,:) = fmag_dim(2:3);
+        for ii=2:numel(enum)
+            scanindexrange(ii,1) = scanindexrange(ii-1,2)+1;
+            scanindexrange(ii,2) = scanindexrange(ii-1,2)+fmag_dim(ii+2);
+        end
+        
+    otherwise
+        error('Unknown HDF5 data structure!')
+end
+
+if ~return_fftshifted
+    % return data as seen by detector, 
+    fmag = math.ifftshift_2D(fmag); 
+    fmask = math.ifftshift_2D(fmask);
+end
+
+
+end
+

+io/load_ptycho_recons.m

@@ -0,0 +1,270 @@
+%LOAD_PTYCHO_RECONS Load data from cxs/h5 or mat file and return it as 
+% structure, single dataset or directly into the workspace.
+% An additional argument can be passed to select subsections of the data.
+% Loading single datasets is only supported for at least 2 output
+% arguments.
+%
+%   file...     path to cxs/h5 or mat file
+%
+%   *optional*
+%   section...  'full', 'probe', 'object', 'recon' or 'p' to select 
+%               subsections of the data; default: 'full'
+%
+%   EXAMPLES:
+%     %% recommended usage %%
+%       % load into a structure
+%       S = load_ptycho_recons('./recon.h5');
+%
+%       % load a subset
+%       S = load_ptycho_recons('./recon.h5', 'probe');
+%
+%       % load into single datasets
+%       [object, probe, p] = load_ptycho_recons('./recon.h5');
+%
+%     %% not recommended, only works in 'base' workspace %%
+%       % load directly into workspace
+%       load_ptycho_recons('./recon.h5');
+%
+%
+%   full = object, probe (current scan) and p
+%   recon = object and probe (current scan)
+%   probe = probe (current scan)
+%   object = object (current scan)
+%
+
+function varargout  = load_ptycho_recons( filename_with_path, varargin )
+
+import io.HDF.hdf5_load
+
+varargout = {};
+
+if ~ischar(filename_with_path)
+   error('First argument has to be string') 
+end
+
+filename_with_path = utils.abspath(filename_with_path); 
+
+if ~exist(filename_with_path, 'file')
+    error('Could not find reconstruction file %s', filename_with_path)
+end
+
+if nargin > 1
+    switch varargin{1}
+        case {'pr'; 'probe'; 'probes'}
+            section = 'probe';
+        case {'ob'; 'obj'; 'objects'}
+            section = 'object';
+        otherwise
+            section = varargin{1};
+    end
+else
+    section = 'full';
+end
+
+if ~nargout
+    output = 0;
+elseif nargout >=2
+    output = 2;
+else
+    output = 1;
+end
+
+    function assign_struct(val, val_name)
+        switch output
+            case 1
+                varargout{1}.(val_name) = val;
+            case 2
+                varargout{end+1} = val;
+            otherwise 
+                assignin('base', val_name, val);
+        end
+    end
+
+    function assign_val(struc)
+        switch output
+            case 1
+                varargout{1} = struc;
+                
+            case 2
+                if isfield(struc, 'object')
+                    varargout{end+1} = struc.object;
+                end
+                if isfield(struc, 'probe')
+                    varargout{end+1} = struc.probe;
+                end
+                if isfield(struc, 'p')
+                    varargout{end+1} = struc.p;
+                end
+                    
+            otherwise
+                fn = fieldnames(struc);
+                for ii=1:length(fn)
+                    assignin('base', fn{ii}, struc.(fn{ii}))
+                end
+        end
+    end
+
+
+% check if it is a .mat file or a .cxs file
+[~, ~, ext] = fileparts(filename_with_path);
+switch ext
+    case '.mat'
+        switch section
+            case 'recon'
+                S = load(filename_with_path, 'object', 'probe');
+                assign_val(S);
+
+            case 'full'
+                S = load(filename_with_path);
+                assign_val(S);
+
+            case 'object'
+                S = load(filename_with_path, 'object');
+                assign_val(S);
+
+            case 'probe'
+                S = load(filename_with_path, 'probe');
+                assign_val(S);
+
+            case 'p'
+                S = load(filename_with_path, 'p');
+                assign_val(S);
+
+            otherwise
+                error('Unknown data section %s', section);
+        end
+        
+    case {'.cxs','.h5'}
+        if io.HDF.hdf5_dset_exists(filename_with_path, 'object', '/reconstruction', true)
+            h5_path = '/reconstruction';
+        else
+            h5_path = '';
+        end
+        
+        % reconstruction
+        switch section
+            case 'recon'
+                % load object
+                h = hdf5_load(filename_with_path, [h5_path '/object']);
+                assign_struct(load_data_cell(h), 'object');
+                % load probe
+                h = hdf5_load(filename_with_path, [h5_path '/probes']);
+                assign_struct(load_data_cell(h), 'probe');
+            case 'full'
+                % load object
+                h = hdf5_load(filename_with_path, [h5_path '/object']);
+                assign_struct(load_data_cell(h), 'object');
+                % load probe
+                h = hdf5_load(filename_with_path, [h5_path '/probes']);
+                assign_struct(load_data_cell(h), 'probe');
+                % load p
+                p = convert2p(hdf5_load(filename_with_path, '/reconstruction/p', '-c'));
+                if io.HDF.hdf5_dset_exists(filename_with_path, 'meta_all', '/measurement', true)
+                    p.meta = hdf5_load(filename_with_path, '/measurement/meta_all', '-c');
+                elseif io.HDF.hdf5_dset_exists(filename_with_path, 'spec_all', '/measurement', true)
+                    p.meta = hdf5_load(filename_with_path, '/measurement/spec_all', '-c');
+                end
+                assign_struct(p, 'p');                
+            case 'object'
+                % load object
+                h = hdf5_load(filename_with_path, [h5_path '/object']);
+                assign_struct(load_data_cell(h), 'object');
+            case 'probe'
+                % load probe
+                h = hdf5_load(filename_with_path, [h5_path '/probes']);
+                assign_struct(load_data_cell(h), 'probe');
+            case 'p'
+                % load p
+                p = convert2p(hdf5_load(filename_with_path, '/reconstruction/p', '-c'));
+                if io.HDF.hdf5_dset_exists(filename_with_path, 'meta_all', '/measurement', true)
+                    p.meta = hdf5_load(filename_with_path, '/measurement/meta_all', '-c');
+                elseif io.HDF.hdf5_dset_exists(filename_with_path, 'spec_all', '/measurement', true)
+                    p.meta = hdf5_load(filename_with_path, '/measurement/spec_all', '-c');
+                end
+                assign_struct(p, 'p');
+            otherwise
+                error('Unknown data section %s', section);
+        end
+                
+        
+        
+    otherwise
+        error('Unknown ptycho datatype %s.', ext)
+end
+
+
+
+end
+
+function tmp = load_data_cell(h)
+
+    fn = fieldnames(h);
+    num_end = str2double(subsref(strsplit(fn{1}, '_'), struct('type', '{}', 'subs',{{length(strsplit(fn{1},'_'))}})));
+    if length(fn)==2 && (strcmpi(fn{1}, 'i') || strcmpi(fn{1}, 'r'))
+        tmp = permute(h.r + 1i*h.i, [2,1,3,4]);
+    elseif isnumeric(num_end) && ~isnan(num_end)
+        for ii=1:length(fn)
+            if isstruct(h.(fn{ii}))
+                tmp{ii} = load_data_cell(h.(fn{ii}));
+            else
+                if isnumeric(h.(fn{ii}))
+                    tmp{ii} = double(h.(fn{ii}));
+                else
+                    tmp{ii} = h.(fn{ii});
+                end
+            end
+        end
+%         tmp = h;
+    else
+        for ii=1:length(fn)
+            if isstruct(h.(fn{ii}))
+                tmp.(fn{ii}) = load_data_cell(h.(fn{ii}));
+            else
+                if isnumeric(h.(fn{ii}))
+                    tmp.(fn{ii}) = double(h.(fn{ii}));
+                else
+                    tmp.(fn{ii}) = h.(fn{ii});
+                end
+            end
+        end
+    end
+
+end
+
+function tmp = convert2p(h)
+    
+    fn = fieldnames(h);
+    for ii=1:length(fn)
+        if isstruct(h.(fn{ii}))
+            h.(fn{ii}) = load_data_cell(h.(fn{ii}));
+        elseif isnumeric(h.(fn{ii}))
+            h.(fn{ii}) = double(h.(fn{ii}));                
+        else
+            continue;
+        end
+    end
+    tmp = h;
+    
+    % object
+    for ii=1:length(h.objects)
+        tmp.object{ii} = permute(h.objects{ii}, [2 1 3 4]);
+    end
+    tmp = rmfield(tmp, 'objects');
+    
+    % probes
+    pr = tmp.probes;
+    tmp.probes = [];
+    for ii=1:length(pr)
+        tmp.probes(:,:,ii,:) = permute(pr{ii}, [2 1 3 4]);
+    end
+    
+    % positions
+    tmp.positions = transpose(tmp.positions);
+    tmp.positions_real = transpose(tmp.positions_real);
+    tmp.positions_orig = transpose(tmp.positions_orig);
+    
+    % ctr
+    tmp.ctr = transpose(tmp.ctr);
+            
+end
+

+io/marread.m

@@ -0,0 +1,194 @@
+% Call function without arguments for instructions on how to use it
+
+% Filename: $RCSfile: marread.m,v $
+%
+% $Revision: 1.1 $  $Date: 2008/07/17 16:55:40 $
+% $Author:  $
+% $Tag: $
+%
+% Description:
+% Macro for reading TIFF files written by a MAR CCD
+%
+% Note:
+% MAR data are TIFF and can be read by forcing the type to tif. 
+% The advantage of forcing the type to mar is, that additional header 
+% fields like the exposure time are read. 
+% This follows the MarCCD header documentaion by Blum and Doyle
+% marccd v0.17.1 
+% Call without arguments for a brief help text.
+%
+% Dependencies:
+% - fopen_until_exists
+% - get_hdr_val
+% - compiling cbf_uncompress.c increases speed but is not mandatory
+%
+%
+% history:
+%
+% July 17th 2008: 1st version
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [frame,vararg_remain] = marread(filename,varargin)
+import io.*
+import utils.fopen_until_exists
+import utils.get_hdr_val
+
+% 0: no debug information
+% 1: some feedback
+% 2: a lot of information
+debug_level = 0;
+
+% initialize return argument
+frame = struct('header',[], 'data',[]);
+
+
+% check minimum number of input arguments
+if (nargin < 1)
+    image_read_sub_help(mfilename,'mar');
+    error('At least the filename has to be specified as input parameter.');
+end
+
+% accept cell array with name/value pairs as well
+no_of_in_arg = nargin;
+if (nargin == 2)
+    if (isempty(varargin))
+        % ignore empty cell array
+        no_of_in_arg = no_of_in_arg -1;
+    else
+        if (iscell(varargin{1}))
+            % use a filled one given as first and only variable parameter
+            varargin = varargin{1};
+            no_of_in_arg = 1 + length(varargin);
+        end
+    end
+end
+
+% check number of input arguments
+if (rem(no_of_in_arg,2) ~= 1)
+    error('The optional parameters have to be specified as ''name'',value pairs');
+end
+    
+% set default values for the variable input arguments and parse the named
+% parameters: 
+vararg = cell(0,0);
+for ind = 1:2:length(varargin)
+    name = varargin{ind};
+    value = varargin{ind+1};
+    switch name
+        otherwise
+            % pass further arguments on to fopen_until_exists
+            vararg{end+1} = name;
+            vararg{end+1} = value;
+    end
+end
+
+
+% try to open the data file
+if (debug_level >= 1)
+    fprintf('Opening %s.\n',filename);
+end
+[fid,vararg_remain] = fopen_until_exists(filename,vararg);
+if (fid < 0)
+    return;
+end
+
+% read all data at once
+[fdat,fcount] = fread(fid,'uint8=>uint8');
+
+% close input data file
+fclose(fid);
+if (debug_level >= 2)
+    fprintf('%d data bytes read\n',fcount);
+end
+
+% the MAR header has a fixed length of 1024 bytes for the TIFF header plus
+% 3072 bytes for the MAR specific part
+end_of_header_pos = 4096;
+
+if (length(fdat) < end_of_header_pos)
+    error([ num2str(length(fdat)) ' bytes read, which is less than the constant header length' ]);
+end
+
+% check little/big endian, also to recognize MAR files
+if (typecast(fdat(1025+32:1025+35),'uint32') ~= 1234)
+    error([ filename ' is not a MAR file or has big endian byte order' ]);
+end
+
+% get image dimensions
+nfast = typecast(fdat(1025+80:1025+83),'uint32');
+nslow = typecast(fdat(1025+84:1025+87),'uint32');
+bytes_per_pixel = typecast(fdat(1025+88:1025+91),'uint32');
+
+if ((bytes_per_pixel ~= 2) && (bytes_per_pixel ~= 4))
+    error( [ 'unforseen no. of bytes per pixel of ' num2str(bytes_per_pixel) ] );
+end
+bytes_expected = end_of_header_pos + nfast*nslow*bytes_per_pixel;
+if (bytes_expected ~= length(fdat))
+    error([ num2str(bytes_expected) ' bytes expected, ' num2str(length(fdat)) ' read' ]);
+end
+
+% return some selected header fields as lines of a cell array
+frame.header = cell(7,1);
+frame.header{1} = sprintf('IntegrationTime_ms %d',...
+    typecast(fdat(1025+640+12:1025+640+15),'uint32'));
+frame.header{2} = sprintf('ExposureTime_ms %d',...
+    typecast(fdat(1025+640+16:1025+640+19),'uint32'));
+frame.header{3} = sprintf('ReadoutTime_ms %d',...
+    typecast(fdat(1025+640+20:1025+640+23),'uint32'));
+frame.header{4} = sprintf('nReads %d',...
+    typecast(fdat(1025+640+24:1025+640+27),'uint32'));
+frame.header{5} = sprintf('DateTime %s %s %s:%s%s %s',...
+    char(fdat(2369:2370)'),...
+    char(fdat(2371:2372)'),...
+    char(fdat(2373:2374)'),...
+    char(fdat(2375:2376)'),...
+    char(fdat(2381:2383)'),...
+    char(fdat(2377:2380)'));
+frame.header{6} = sprintf('PixelSizeX_nm %d',...
+    typecast(fdat(1025+768+4:1025+768+7),'uint32'));
+frame.header{7} = sprintf('PixelSizeY_nm %d',...
+    typecast(fdat(1025+768+8:1025+768+11),'uint32'));
+
+% store data
+switch bytes_per_pixel
+    case 2
+        frame.data = typecast(fdat(4097:end),'uint16');
+    case 4
+        frame.data = typecast(fdat(4097:end),'uint32');
+    otherwise
+        error( [ 'unforseen no. of bytes per pixel of ' num2str(bytes_per_pixel) ] );
+end
+frame.data = reshape(frame.data,nfast,nslow);

+io/mat2json.m

@@ -0,0 +1,90 @@
+function J=mat2json(M,F)
+import io.*
+%JSON2MAT converts a Matlab structure into a javscript data object (JSON).
+%         M can also be a file name. In teh spirit of fast prototyping 
+%         this function takes a very loose approach to data types and 
+%         dimensionality - neither is explicitly retained.
+%
+%         The second input argument is optional and when used it indicates
+%         the name of teh file where J is to be stored.
+%
+%Example: mat2json(json2mat('{lala:2,lele:4,lili:[1,2,{bubu:5}]}')) 
+%
+% Jonas Almeida, March 2010
+
+% Copyright (c) 2010, Jonas Almeida
+% All rights reserved.
+% 
+% Redistribution and use in source and binary forms, with or without
+% modification, are permitted provided that the following conditions are
+% met:
+% 
+% * Redistributions of source code must retain the above copyright
+% notice, this list of conditions and the following disclaimer.
+% * Redistributions in binary form must reproduce the above copyright
+% notice, this list of conditions and the following disclaimer in
+% the documentation and/or other materials provided with the distribution
+% 
+% THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+% AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+% IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+% ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+% LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+% CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+% SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+% INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+% CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+% ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+% POSSIBILITY OF SUCH DAMAGE.
+
+
+switch class(M)
+    case 'struct'
+        J='{';
+        f=fieldnames(M);
+        for i=1:length(f)
+            fld=f{i};
+            id=regexp(fld,'^id(\d*)$','tokens');
+            if ~isempty(id)
+              fld=id{1}{1};
+            end
+            J=[J,'"',fld,'":',mat2json(M.(f{i})),','];
+        end
+        J(end)='}';
+        
+    case 'cell'
+        J='[';
+        for i=1:length(M)
+            J=[J,mat2json(M{i}),','];
+        end
+        J(end)=']';
+    otherwise
+        if isnumeric(M) % notice looseness in not converting single numbers into arrays
+            if length(M(:))==1
+                J=num2str(M);
+            else
+                s=size(M);
+                if (length(s)==2)&(s(1)<2) % horizontal or null vector
+                    J=['[',num2str(M),']']; % and of destroying dimensionality
+                    J=regexprep(J,'\s+',',');
+                elseif length(s)==2 %2D solution
+                    J='[';
+                    for i=1:s(1)
+                        J=[J,mat2json(M(i,:)),','];
+                    end
+                    J(end)=']';
+                elseif length(s)>2 % for now treat higher dimensions as linear vectors
+                    J=['[',num2str(M(:)'),']']; % and of destroying dimensionality
+                    J=regexprep(J,'\s+',',');
+                end
+            end
+        else
+            J=['"',M,'"']; % otherwise it is treated as a string
+        end
+end
+
+if nargin>1 %save JSON result in file
+    fid=fopen(F,'w');
+    fprintf(fid,'%s',J);
+    fclose(fid);
+end

+io/mcs_mesh.m

@@ -0,0 +1,429 @@
+% Call function without arguments for instructions on how to use it
+
+% Filename: $RCSfile: mcs_mesh.m,v $
+%
+% $Revision: 1.7 $  $Date: 2016/08/03 08:38:32 $
+% $Author:  $
+% $Tag: $
+%
+% Description:
+% Macro for reading .dat files in self-defined data formats
+%
+% Note:
+% Call without arguments for a brief help text.
+%
+% Dependencies:
+% - image_read_set_default
+% - fopen_until_exists
+% - get_hdr_val
+% - compiling cbf_uncompress.c increases speed but is not mandatory
+%
+%
+% history:
+%
+% February 18th 2009: 1st version
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [mcs_data, data_adjusted, pos_data] = mcs_mesh(first_scan_no,no_of_intervals,varargin)
+import io.*
+import utils.adjust_projection
+import utils.fopen_until_exists
+import utils.get_hdr_val
+
+% initialize return arguments
+mcs_data = [];
+% legacy_sgalil = false;  % A flag that keeps track of different commands needed if the older legacy file of reading positions is used - 2019.04
+
+% set default parameter
+% plot this MCS channel
+ch_to_plot = 4;
+snake_scan=0;
+fast_axis_x = 1;
+% create the plot in this figure
+fig_no = 123;
+% exit with an error message if unhandled named parameters are left at the
+% end of this macro
+unhandled_par_error = 1;
+% file name base
+fname_base = '';
+% first part of directory path
+dir_base = '~/Data10/mcs/'; 
+% scaling factors for the axes
+x_scale = 1.0;
+y_scale = 1.0;
+%
+axis_minmax = [];
+% save resulting figure
+figure_dir = '~/Data10/analysis/online/stxm/figures/';      
+% save the resulting data
+data_dir =  '~/Data10/analysis/online/stxm/data/';      
+pos_file = '~/Data10/sgalil/S%05d.dat';
+positions_only = false;
+
+% check minimum number of input arguments
+if (nargin < 2)
+    fprintf('[mcs_data data_adjusted pos_data]=%s(<first scan no.>, <no. of line intervals> [[,<name>,<value>] ...]);\n',...
+        mfilename);
+    fprintf('The optional <name>,<value> pairs are:\n');
+    fprintf('''ChToPlot'',<channel no.>              if greater than zero than this channel is plotted, default is %d\n',ch_to_plot);
+    fprintf('''SnakeScan'',<0-no, 1-yes>             scan mode is a snake pattern, default is 0\n');
+    fprintf('''FastAxisX'',<0-no, 1-yes>             fast scan axis is along x, default is 1\n');
+    fprintf('''FigNo'',<figure number>               plot the data in this figure, 0 for no figure, default is %d\n',fig_no);
+    fprintf('''XScale'',<scale factor>               scale the x-axis with this factor\n');
+    fprintf('''YScale'',<scale factor>               scale the y-axis with this factor\n');
+    fprintf('''AxisMinMax'',<[ min max]>             specify both min and max value\n');    
+    fprintf('''DirBase'',<first part of data path>   including an ending slash, default is ''%s''\n',dir_base);
+    fprintf('''FnameBase'',<first part of file name> in case of an empty string the current Unix user name is used followed by an underscore, default if ''%s''\n',fname_base);
+    fprintf('''FigureDir'',''directory''               save the resulting plot in eps, jpeg and Matlab fig format, '''' for no saving, default is %s\n',figure_dir);
+    fprintf('''DataDir'',''directory''                 save the resulting data as Matlab file, '''' for no saving, default is %s\n',data_dir);
+    fprintf('''Pos_file'',''path and filename structure''   checks the flipping of images using the positions found in these files. It gives a warning if it detects the wrong fast axis and it flips the lines for snake scans.\n');
+    fprintf('                                           If the filename structure includes an %% it tries to read positions assuming one dat file for each scan. If it does not include an %% then it uses spec_read.\n');
+    fprintf('                                           Default is %s, set to empty =[ ] to avoid warnings \n',pos_file);
+    fprintf('''Positions_only'',<0-no, 1-yes>        if you only use the routine to get and check positions of a scan but you don''t have a MCS scalar measurement, default is %d.\n', positions_only);
+    fprintf('The data are in the format ''fast to slow axis'', i.e., the first dimension is the MCS channel,\n');
+    fprintf('the second dimension is the exposure index, the third dimension is the fast axis of a mesh scan\n');
+    fprintf('and the fourth dimension is the slow axis of a mesh scan.\n');
+    fprintf('An optional second output ''data_adjusted'' can be obtained. This is a structure that has been adjusted and flipped according to the scan parameters in order to reflect\n');
+    fprintf('  the sample physical orientation. One of the fields is ''transm'' which is the sample transmissivity. Other fields are positions_out, scan_num, and scan_point.\n');
+    error('At least the number of the first scan and the number of line intervals need to be specified as input parameter.');
+end
+
+% accept cell array with name/value pairs as well
+no_of_in_arg = nargin;
+if (nargin == 3)
+    if (isempty(varargin))
+        % ignore empty cell array
+        no_of_in_arg = no_of_in_arg -1;
+    else
+        if (iscell(varargin{1}))
+            % use a filled one given as first and only variable parameter
+            varargin = varargin{1};
+            no_of_in_arg = 1 + length(varargin);
+        end
+    end
+end
+
+% check number of input arguments
+if (rem(no_of_in_arg,2) ~= 0)
+    error('The optional parameters have to be specified as ''name'',''value'' pairs');
+end
+
+% parse the variable input arguments
+vararg = cell(0,0);
+for ind = 1:2:length(varargin)
+    name = varargin{ind};
+    value = varargin{ind+1};
+    switch name
+        case 'ChToPlot'
+            ch_to_plot = value;
+        case 'SnakeScan'
+            snake_scan = value;
+        case 'FastAxisX'
+            fast_axis_x = value;
+        case 'FigNo'
+            fig_no = value;
+        case 'XScale'
+            x_scale = value;
+        case 'YScale'
+            y_scale = value;
+        case 'AxisMinMax'
+            axis_minmax = value;
+        case 'UnhandledParError'
+            unhandled_par_error = value;
+        case 'FnameBase'
+            fname_base = value;
+        case 'DirBase'
+            dir_base = value;
+        case 'DataDir'
+            data_dir = value;
+        case 'FigureDir'
+            figure_dir = value;
+        case 'Pos_file'
+            pos_file = value;
+        case 'Positions_only'
+            positions_only = value;
+        otherwise
+            vararg{end+1} = name; %#ok<AGROW>
+            vararg{end+1} = value; %#ok<AGROW>
+    end
+end
+
+
+% initialize the list of unhandled parameters
+vararg_remain = cell(0,0);
+
+
+% get the current user name
+
+    if (length(fname_base) < 1)
+        [stat,usr]=unix('echo $USER');
+        fname_base = [ sscanf(usr,'%s') '_' ];
+    end
+
+
+% initialize the output figure
+if ~positions_only
+    figure(fig_no);
+    hold off;
+    clf;
+end
+
+last_scan_no = first_scan_no + no_of_intervals;
+store_ind = 1;
+last_draw_time = clock;
+
+% over all the scan lines
+for scan_no = first_scan_no:last_scan_no
+    dir = [ dir_base 'S' num2str(floor(scan_no/1000)*1000,'%05d') '-' ...
+        num2str(floor(scan_no/1000)*1000+999,'%05d') '/S' num2str(scan_no,'%05d') '/' ];
+    filename =  [ fname_base num2str(scan_no,'%05d') '.dat' ];
+    
+    % read the frame until all data are available
+    ind_rep = 0;
+    ind_max = 3;
+    last_no_of_el_read = 0;
+    if ~positions_only
+        while (ind_rep < ind_max)
+            frame = image_read([dir filename ], 'RetryReadSleep',10, ...
+                'RetryReadMax',1, 'RetrySleepWhenFound',10);
+            if ~isempty(frame.data)&&(frame.no_of_el_read{1} >= numel(frame.data))
+                % the complete data set has been read
+                break;
+            end
+            if ~isempty(frame.data)&&(frame.no_of_el_read{1} <= last_no_of_el_read)
+                % no progress, increase timeout counter
+                ind_rep = ind_rep +1;
+            else
+                ind_rep = 0;
+            end
+            if ~isempty(frame.no_of_el_read)
+                last_no_of_el_read = frame.no_of_el_read{1};
+            end
+            if ~isempty(frame.header)
+                exp_time = get_hdr_val(frame.header{1},'Exposure_time','%f',1);
+            end
+            wait_time = numel(frame.no_of_el_read)*exp_time;
+            if (wait_time > 2.0)
+                fprintf('%d/%d: frame incomplete (%d/%d), waiting %.1fs and retrying\n',...
+                    ind_rep+1,ind_max,frame.no_of_el_read{1},numel(frame.data));
+            end
+            pause(wait_time);
+        end
+    end
+    if ~isempty(pos_file)
+        if contains(pos_file,'%')
+            positions_from_spec = false;
+            filepos = sprintf(pos_file,scan_no);
+            try
+                positions = beamline.read_position_file(filepos);
+                positions.data(1,:) = positions.Avg_x;
+                positions.data(2,:) = positions.Avg_y;
+                numpts = numel(positions.Avg_x);
+            catch
+                warning('The reading of positions for sgalil did not work, now trying legacy mode in older sgalil position format')
+                positions = image_read(filepos,'RetryReadSleep',10,'RetryReadMax',0);
+                numpts = size(positions.data,2);
+%                 legacy_sgalil = true;
+            end
+        else % If it does not contain % delimiter then we assume is a spec file
+            positions_from_spec = true;
+            if scan_no == last_scan_no % Only read spec positions once all scans are done, otherwise is too slow to read each time
+                try
+                    fprintf('Reading positions from spec file %s \n',pos_file)
+                    positions = io.read_scan_positions_spec(pos_file,first_scan_no:last_scan_no,{'samx','samy'});
+                catch
+                    warning('io.read_scan_positions_spec failed, pausing 5 seconds and retrying')
+                    pause(5);
+                    positions = io.read_scan_positions_spec(pos_file,first_scan_no:last_scan_no,{'samx','samy'});
+                end
+            else
+                positions.data = [];
+            end
+        end
+    end
+    % initialize return array
+    if (scan_no == first_scan_no)
+        if ~positions_only
+            mcs_data = zeros(size(frame.data,1),size(frame.data,2),...
+                size(frame.data,3) + 1 ,no_of_intervals+1);
+            numpts = size(frame.data,3) + 1;
+        else
+            mcs_data = [];
+            numpts = size(positions.data,2);
+        end
+        
+        [scan_num, scan_point] = meshgrid(first_scan_no:last_scan_no,0:numpts-1);
+        % Check positions of stage for flipping
+        if ~isempty(pos_file)
+            pos_data =  zeros(numpts, no_of_intervals+1, 2);
+        else
+            pos_data = [];
+        end
+    end
+    
+    if ~positions_only
+        store_ind_to = store_ind + size(frame.data,4) - 1;
+    else
+        store_ind_to = store_ind;
+    end
+    %%% Special about MCS is that it does not take the first image because
+    %%% it is triggered in a strange way, so here to match other
+    %%% detectors we make the matrix one element larger and we replicate
+    %%% the first value
+    if ~positions_only
+        mcs_data(:,:,2:end,store_ind:(store_ind+size(frame.data,4)-1)) = frame.data;
+        mcs_data(:,:,1,store_ind:(store_ind+size(frame.data,4)-1)) = frame.data(:,1,2,:);
+    end
+
+    if ~isempty(pos_file)
+        if ~positions_from_spec
+            pos_data(:,store_ind,:) = positions.data.';
+        else % if positions are from spec they are read only at the end and arranged in a structure so they need special handling
+            if scan_no == last_scan_no
+                pos_data =  zeros(size(positions(1).data,1),no_of_intervals+1,2);
+                for ii = numel(positions)
+                    pos_data(:,ii,:) = positions(ii).data;
+                end
+            end
+        end
+    end
+    store_ind = store_ind_to +1;
+    
+    if ((scan_no == first_scan_no) || (scan_no == last_scan_no) ||  ...
+        (etime(clock,last_draw_time) > 10))
+        if (size(mcs_data,2) == 1)
+            data_plot = squeeze(mcs_data(ch_to_plot,1,:,:));
+        elseif ~isempty(mcs_data)
+            data_plot = squeeze(mcs_data(ch_to_plot,:,:,1));
+        else
+            data_plot = [];
+        end
+        if (~positions_only)&&((size(data_plot,1) > 1) && (size(data_plot,2) > 1))
+            %CHANGE
+            [data_plot, positions_out] = adjust_projection(data_plot, snake_scan, fast_axis_x, pos_data);
+            % 2D plot
+            x_values = (1:size(data_plot,2)) * x_scale;
+            y_values = (1:size(data_plot,1)) * y_scale;
+
+            if (~isempty(axis_minmax))
+                caxis(axis_minmax);
+            end
+            %plot the image
+            figure(fig_no)
+            imagesc(data_plot);
+            axis xy;
+            axis equal;
+            axis tight;
+            colormap gray;
+            colorbar;
+            title( [ fname_base ': #'  num2str(first_scan_no) ' -' num2str(scan_no)] );
+            drawnow;
+        elseif (~positions_only)
+            % 1D plot if only one line has been read
+            x_values = (1:length(data_plot)) * x_scale;
+            plot(x_values,data_plot);
+        else
+            [data_plot, positions_out] = adjust_projection(data_plot, snake_scan, fast_axis_x, pos_data);
+        end
+        last_draw_time = clock;
+    end
+end
+
+[scan_num]   = adjust_projection(scan_num,   snake_scan, fast_axis_x, pos_data);
+[scan_point] = adjust_projection(scan_point, snake_scan, fast_axis_x, pos_data);
+
+data_adjusted.transm = data_plot;
+data_adjusted.positions_out = positions_out;
+data_adjusted.scan_num = scan_num;
+data_adjusted.scan_point = scan_point;
+
+% file name for saving
+filename = sprintf('stxm_scans_%05d-%05d_mcs',first_scan_no,...
+    last_scan_no);
+
+% save figures
+if (~isempty(figure_dir))&&(~positions_only)
+    figure(fig_no);
+
+    % create output directories and write the plot in different formats
+    if (~exist(figure_dir,'dir'))
+        mkdir(figure_dir)
+    end
+    if ((figure_dir(end) ~= '/') && (figure_dir(end) ~= '\')) 
+        figure_dir = [ figure_dir '/' ];
+    end
+    fprintf('output directory for figures is %s\n',figure_dir);
+    
+    subdir = [ figure_dir 'jpg/' ];
+    if (~exist(subdir,'dir'))
+        mkdir(subdir);
+    end
+    fprintf('saving %s.jpg\n',filename);
+    print('-djpeg','-r300',[subdir filename '.jpg'] );
+    
+    subdir = [ figure_dir 'eps/' ];
+    if (~exist(subdir,'dir'))
+        mkdir(subdir);
+    end
+    fprintf('saving %s.eps\n',filename);
+    print('-depsc','-r1200',[subdir filename '.eps'] );
+    
+    subdir = [ figure_dir 'fig/' ];
+    if (~exist(subdir,'dir'))
+        mkdir(subdir);
+    end
+    fprintf('saving %s.fig\n',filename);
+    hgsave([subdir filename '.fig']);    
+end
+
+% save resulting data
+if (~isempty(data_dir))
+    if ((data_dir(end) ~= '/') && (data_dir(end) ~= '\')) 
+        data_dir = [ data_dir '/' ];
+    end
+    
+    % create output directory
+    if (~exist(data_dir,'dir'))
+        mkdir(data_dir)
+    end
+    
+    % save data
+    fprintf('saving %s.mat\n',[data_dir filename]);
+    save([data_dir filename],'mcs_data','first_scan_no','last_scan_no','pos_data','data_adjusted');
+end
+
+return
+

+io/movefile_fast.m

@@ -0,0 +1,58 @@
+% MOVEFILE_FAST  faster alternative to the matlab movefile function 
+% 
+% movefile_fast(source, destination)
+% 
+
+
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function movefile_fast(source, destination)
+
+     Sd = java.io.File(destination); 
+     if Sd.isDirectory
+         % if provided path to move is a directory, create full file path from the source name 
+         [~, filename, ext] = fileparts(source); 
+         destination = fullfile(destination, [filename, ext]);
+         Sd = java.io.File(destination); 
+     end
+     Ss = java.io.File(source);
+     assert(Ss.canRead, sprintf('File %s does not exist or is not readable', source))
+     % move the file using java
+     Ss.renameTo(Sd);
+     assert(Sd.canWrite, sprintf('Moving file %s to %s failed', source, destination))
+end
+
+

+io/multiple_mcs_headers.m

@@ -0,0 +1,75 @@
+% [] = multiple_mcs_headers(scan_no_from,scan_no_to)
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [] = multiple_mcs_headers(scan_no_from,scan_no_to)
+import utils.compile_x12sa_filename
+
+if (nargin ~= 2)
+    fprintf('Usage: %s <scan no. from> <scan no. to>\n',mfilename);
+    fprintf('Searches for multiple MCS headers, renames the matching files and\n');
+    fprintf('writes a new file with only the last header and its data.\n');
+    fprintf('This is just a workaround for the current implementation of the MCS\n');
+    fprintf('where spec stores the MCS data several times in case a cont_line is repeated.\n');
+end
+
+for (scan_no = scan_no_from:scan_no_to)
+    dirname = compile_x12sa_filename(scan_no, -1, 'BasePath','~/Data10/mcs/');
+    dirinfo = dir([ dirname '*.dat' ]);
+    if (length(dirinfo) < 1)
+        fprintf('%s not found\n',filename);
+    else
+        for (ind = 1:length(dirinfo))
+            filename = [dirname dirinfo(ind).name];
+            fid = fopen(filename,'r');
+            % read all data at once
+            [fdat,~] = fread(fid,'uint8=>uint8');
+            fclose(fid);
+            % check for multiple fileheaders
+            header_start = strfind(fdat','# MCS file version');
+            if (length(header_start) > 1)
+                fprintf('%s has %d headers\n',filename, length(header_start));
+                % rename the file
+                movefile(filename, [filename '_org']);
+                % use the last file part
+                fdat = fdat(header_start(end):end);
+                % store the truncated data
+                fid = fopen(filename,'w');
+                fwrite(fid,fdat);
+                fclose(fid);
+            end
+        end
+    end
+end

+io/private/image_orient_help.m

@@ -0,0 +1,119 @@
+% [] = image_orient_help(m_file_name,varargin)
+% parameter help for image_orient and calling functions
+
+% Filename: $RCSfile: image_orient_help.m,v $
+%
+% $Revision: 1.3 $  $Date: 2009/01/16 15:30:29 $
+% $Author:  $
+% $Tag: $
+%
+% Description:
+% parameter help for image_orient and calling functions
+%
+% Note:
+% none
+%
+% Dependencies:
+% none
+%
+%
+% history:
+%
+% May 27th 2008: 1st version
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [] = image_orient_help(m_file_name,varargin)
+import io.*
+
+% check minimum number of input arguments
+if (nargin < 1)
+    error('At least the m-file name has to be specified as input parameter.');
+end
+
+% accept cell array with name/value pairs as well
+no_of_in_arg = nargin;
+if (nargin == 2)
+    if (isempty(varargin))
+        % ignore empty cell array
+        no_of_in_arg = no_of_in_arg -1;
+    else
+        if (iscell(varargin{1}))
+            % use a filled one given as first and only variable parameter
+            varargin = varargin{1};
+            no_of_in_arg = 1 + length(varargin);
+        end
+    end
+end
+
+% check number of input arguments
+if (rem(no_of_in_arg,2) ~= 1)
+    error('The optional parameters have to be specified as ''name'',''value'' pairs');
+end
+
+
+% parse the variable input arguments
+parameters_only = 0;
+for ind = 1:2:length(varargin)
+    name = varargin{ind};
+    value = varargin{ind+1};
+    switch name
+        case 'ParametersOnly' 
+            parameters_only = value;
+        otherwise
+            error('unknown parameter %s',name);
+    end
+end
+
+if (~parameters_only)
+    fprintf('Usage:\n')
+    fprintf('[data_out]=image_orient(<frame> [[,<name>,<value>] ...]);\n');
+    fprintf('The optional <name>,<value> pairs are:\n');
+end
+fprintf('''OrientExtension'',<''extension''>      file name extension to determine the orientation from\n');
+fprintf('''OrientByExtension'',<0-no,1-yes>     use the default orientation for this file type, default yes,\n');
+fprintf('                                     superseeded by following orientation parameters (i.e., parameter order matters)\n');
+fprintf('''Transpose'',<0-no,1-yes>             mirror at the diagonal\n');
+fprintf('''FlipUD'',<0-no,1-yes>                mirror about the horizontal axis\n');
+fprintf('''FlipLR'',<0-no,1-yes>                mirror about the vertical axis\n');
+fprintf('''Orientation'',<[<Transpose> <FlipLR> <FlipUD>]>\n');
+fprintf('                                     specify for each of the three parameters 0 (no) or 1 (yes)\n');
+if (~parameters_only)
+    fprintf('\n');
+    fprintf('Examples:\n');
+    fprintf('[data_out]=%s(data, ''Orientation'',[1 1 0]);\n',mfilename);
+    fprintf('[data_out]=%s(data, ''Transpose'',1, ''FlipLR'',1);\n',mfilename);
+    fprintf('These two calls are equivalent.\n');
+end

+io/private/image_read_sub_help.m

@@ -0,0 +1,124 @@
+% [] = image_read_sub_help(m_file_name,extension,varargin)
+% parameter help for sub-routines of image_read like cbfread, edfread,
+% speread and fliread
+
+% Filename: $RCSfile: image_read_sub_help.m,v $
+%
+% $Revision: 1.1 $  $Date: 2008/06/10 17:05:14 $
+% $Author:  $
+% $Tag: $
+%
+% Description:
+% parameter help for sub-routines of image_read like cbfread, edfread,
+% speread and fliread
+%
+% Note:
+% none
+%
+% Dependencies:
+% none
+%
+%
+% history:
+%
+% May 9th 2008: 1st version
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [] = image_read_sub_help(m_file_name,extension,varargin)
+import io.*
+% check minimum number of input arguments
+if (nargin < 2)
+    error('At least the m-file name and the extension have to be specified as input parameter.');
+end
+
+% accept cell array with name/value pairs as well
+no_of_in_arg = nargin;
+if (nargin == 3)
+    if (isempty(varargin))
+        % ignore empty cell array
+        no_of_in_arg = no_of_in_arg -1;
+    else
+        if (iscell(varargin{1}))
+            % use a filled one given as first and only variable parameter
+            varargin = varargin{1};
+            no_of_in_arg = 2 + length(varargin);
+        end
+    end
+end
+
+% check number of input arguments
+if (rem(no_of_in_arg,2) ~= 0)
+    error('The optional parameters have to be specified as ''name'',''value'' pairs');
+end
+
+
+% parse the variable input arguments
+examples = 1;
+extension_returned = 0;
+for ind = 1:2:length(varargin)
+    name = varargin{ind};
+    value = varargin{ind+1};
+    switch name
+        case 'Examples' 
+            examples = value;
+        case 'ExtensionReturned'
+            extension_returned = 1;
+        otherwise
+            error('unknown parameter %s',name);
+    end
+end
+
+fprintf('Usage:\n')
+if (extension_returned)
+    fprintf('[frame]=%s(<filename> [[,<name>,<value>] ...]);\n',...
+        m_file_name);
+else
+    fprintf('[frame]=%s(<filename> [[,<name>,<value>] ...]);\n',...
+        m_file_name);
+end
+fprintf('The optional <name>,<value> pairs are:\n');
+fprintf('''RetryReadSleep'',<seconds>           if greater than zero retry opening after this time (default: 0.0)\n');
+fprintf('''RetryReadMax'',<0-...>               maximum no. of retries, 0 for infinity (default: 0)\n');
+fprintf('''MessageIfNotFound'',<0-no,1-yes>     display a mesage if not found, 1-yes is default\n');
+fprintf('''ErrorIfNotFound'',<0-no,1-yes>       exit with an error if not found, default is 1-yes\n');
+if (examples)
+    fprintf('\n');
+    fprintf('Examples:\n');
+    fprintf('[frame]=%s(''~/Data10/roper/image.%s'');\n',...
+        m_file_name,extension);
+    fprintf('\n');
+    fprintf('The returned structure has the fields data, header and extension.\n');
+end

+io/private/ptycho_reader/Makefile

@@ -0,0 +1,63 @@
+ifndef MATLAB_HOME
+  $(error "MATLAB_HOME is undefined, load matlab/2018a module!")
+endif
+
+ifndef HDF5_SERIAL_HOME
+  $(error "HDF5_SERIAL_HOME is undefined, load gcc/6.3.0 and hdf5_serial/1.8.18 modules!")
+endif
+
+ifndef TIFF_INCLUDE_DIR
+  ifndef TIFF_LIBRARY_DIR
+    $(error "TIFF_INCLUDE_DIR or TIFF_LIBRARY_DIR undefined, load tiff/4.0.9 module!")
+  endif
+endif
+
+MEXOPTIONS += -R2018a
+
+IO_DIR := ../..
+RDOBJ_TARGET_BASE := $(IO_DIR)/ptycho_read
+RDMSR_TARGET_BASE := $(IO_DIR)/read_measurement
+TARGET_SUFFIX := mexa64
+RDOBJ_TARGET_FILE := $(RDOBJ_TARGET_BASE).$(TARGET_SUFFIX)
+RDMSR_TARGET_FILE := $(RDMSR_TARGET_BASE).$(TARGET_SUFFIX)
+
+all: $(RDOBJ_TARGET_FILE) $(RDMSR_TARGET_FILE)
+
+ptycho_read: $(RDOBJ_TARGET_FILE)
+
+read_measurement: $(RDMSR_TARGET_FILE)
+
+debug_helper.o : debug_helper.cc debug_helper.h
+	mex $(MEXOPTIONS) -c $<
+
+hdf5_helper.o: hdf5_helper.cc hdf5_helper.h
+	mex $(MEXOPTIONS) -c $<
+
+read_object_data.o: read_object_data.cc read_object_data.h precision.h hdf5_helper.h debug_helper.h env_helper.h multi_processing.h
+	mex $(MEXOPTIONS) -c $<
+
+read_eiger_data.o: read_eiger_data.cc read_eiger_data.h precision.h hdf5_helper.h debug_helper.h env_helper.h multi_processing.h
+	mex $(MEXOPTIONS) -c $<
+
+read_data_threaded.o: read_data_threaded.cc read_data_threaded.h precision.h debug_helper.h env_helper.h ${TIFF_INCLUDE_DIR}/tiffio.h
+	mex $(MEXOPTIONS) -I$(TIFF_INCLUDE_DIR) -c $<
+
+# Put the resultinf mex file into +io/+HDF folder
+$(RDOBJ_TARGET_FILE): readObjectData.cc debug_helper.o hdf5_helper.o read_object_data.o mex_helper.h env_helper.h precision.h
+	mex $(MEXOPTIONS) $< read_object_data.o hdf5_helper.o debug_helper.o $(HDF5_SERIAL_HOME)/lib/libhdf5.a -lz -ldl -output $(RDOBJ_TARGET_BASE)
+
+$(RDMSR_TARGET_FILE): readMeasurementData.cc read_eiger_data.o hdf5_helper.o read_data_threaded.o debug_helper.o env_helper.h mex_helper.h precision.h read_eiger_data.h read_data_threaded.h $(TIFF_LIBRARY_DIR)/libtiff.a
+	mex $(MEXOPTIONS) $< read_eiger_data.o hdf5_helper.o read_data_threaded.o debug_helper.o $(HDF5_SERIAL_HOME)/lib/libhdf5.a $(TIFF_LIBRARY_DIR)/libtiff.a -lz -ldl -output $(RDMSR_TARGET_BASE)
+
+doc:
+	test -r doxygen.conf && rm -rf doc/html
+	test -d doc || mkdir doc
+	doxygen doxygen.conf
+
+clean:
+	rm -f *~ *.o
+
+proper: clean
+	rm -f $(RDOBJ_TARGET_FILE) $(RDMSR_TARGET_FILE)
+
+.PHONY: all clean proper ptycho_read read_measurement doc

+io/private/ptycho_reader/Readme.md

@@ -0,0 +1,149 @@
+This directory contains code for reading ptychographic reconstruction results and measurement data.
+
+ptycho_read
+===========
+
+This is a MEX function for reading ptychographically constructed object datasets from HDF5 files.
+
+The parameters to the function are
+1. number of reader processes
+2. precision of the returned array: either 'single' or 'double'
+3. desired dimension of the returned objects
+   This is an optional parameter. If absent, the dimension of the object in the first file is assumed to be the desired object dimension.
+   Objects that have different dimension will be adapted (either some pixels at the borders left away, or zero pixels added)
+4. path to the object dataset within the HDF5 file (e.g. '/reconstruction/object')
+5. cell array with the file paths (e.g. {'the/path/to/example.h5'})
+   If there is only one file, the parameter can also be a character array
+
+From MATLAB call it in one of the following ways:
+
+    [A, ind] = io.ptycho_read(1, 'single', [21,22], '/reconstruction/object', {'test1_20x20_c.h5', 'test2_20x20_c.h5'});
+    A = io.ptycho_read(1, 'single', [21,22], '/reconstruction/object', {'test1_20x20_c.h5', 'test2_20x20_c.h5'});
+
+The array *A* will contain the object datasets as a multidimensional array (Ncols, Nrows, Nmodes, Nslices, Nobjects). Nslices and Nmodes dimensions will be dropped if they are of size one. *ind* will contain a list of file indices that could not be read, if the first form is used. In the second form you'll get an error message if a file cannot be read. The function will print out some information about the read speed, if the environment variable *PTYCHO_READ_VERBOSE* is set to one of the values '1', 'yes', 'true'.
+
+    setenv('PTYCHO_READ_VERBOSE', 'yes')
+
+The above command accomplishes this inside MATLAB.
+
+read_measurement
+================
+
+This is a MEX function for reading data from Eiger detector files in HDF5 format. The data must be located at /eh5/images within the file.
+
+The parameters to the function are
+1. parameter structure
+
+The parameter structure contains some general fields:
+* 'asize' with the result image dimensions (Ncols, Nrows) (optional, derived from data by default)
+* 'ctr' with the regioin of interest center [col, row] relative to the data (optional, derived from data by default)
+* 'precision' with the desired precision - 'single' or 'double'
+
+For the **Eiger** detector, the parameter structure must contain the following fields:
+* 'extension' with value 'h5'
+* 'data_path' cell array (size 1 or N) with with the paths to the HDF5 files conaining the Eiger data
+* 'data_location' cell array (size 1 or N) with the dataset location(s) within the file(s)
+* 'nthreads' with the desired number of parallel read processes
+
+Data is expected to be present in the following form:
+
+    GROUP "/" {
+       GROUP "eh5" {
+          DATASET "images" {
+             DATATYPE  H5T_STD_U32LE
+             DATASPACE  SIMPLE { ( 464, 514, 1030 ) / ( 464, 514, 1030 ) }
+          }
+       }
+    }
+
+Here, there are 464 images with 514x1030 pixels. The file must be readable with the standard HDF5 library version 1.10.2
+
+For the **Pilatus** detector
+* 'extension' with value 'cbf'
+* 'data_path' cell array with the paths of the CBF data files
+* 'nthreads' with the desired number of parallel reader threads
+Every data file must contain lines like
+
+    conversions="x-CBF_BYTE_OFFSET"
+    X-Binary-Size-Fastest-Dimension: 1475
+    X-Binary-Size-Second-Dimension: 1679
+
+where 1475x1679 are the image dimensions and the image data should be byte offset encoded.
+
+For the **Moench** detector
+* 'extension' with value 'tiff'
+* 'data_path' cell array with the paths of the TIFF data files
+* 'nthreads' with the desired number of parallel reader threads
+The files are expected to contain raw single precision floating point data in one image plane, stored in IMAGELENGTH chunks of size IMAGEWIDTH. These tags must also give the image dimensions.
+
+From MATLAB call it in the following way:
+
+    arg = struct()
+    arg.extension = 'h5'
+    arg.precision = 'single'
+    arg.nthreads = 1
+    arg.data_path = { 'test.h5' }
+    arg.data_location = { '/eiger/images' }
+    A = io.read_measurement(arg);
+
+The array *A* will contain the measurement datasets as a multidimensional array [Ncols, Nrows, Nimages]. The function will print out some information about the arguments, if the environment variable *PTYCHO_READ_VERBOSE* is set to one of the values '1', 'yes', 'true'.
+
+    setenv('PTYCHO_READ_VERBOSE', 'yes')
+
+The above command accomplishes this inside MATLAB.
+
+Debug output
+------------
+
+If debug output about the inner workings of the functions is desired, the environment variable *PTYCHO_READ_DEBUG* should be set to the filename where debug output will end up.
+
+    setenv('PTYCHO_READ_DEBUG', '/tmp/debug_output.txt')
+
+The above command accomplishes this inside MATLAB.
+
+Compilation
+-----------
+
+The code works only with MATLAB versions at or above 2018a, because the code requires the new interleaved complex array format. You need to load the matlab/2018a or a later MATLAB environment module in order to compile the code. Aditionally the gcc version and an apropriate HDF5 serial and TIFF environment module need to be loaded. This can be done by sourcing the *setup-environment.sh* script. After these steps module list shoud approximately like this:
+
+    [stadler_h@ra-l-002 ~]$ module list
+    Currently Loaded Modulefiles:
+      1) gcc/6.3.0            2) hdf5_serial/1.8.18   3) matlab/2018a         4) tiff/4.0.9
+
+Now you should be ready to compile the code:
+
+    [stadler_h@ra-l-002 ptycho_reader]$ make
+    mex -R2018a -c debug_helper.cc
+    Building with 'g++'.
+    MEX completed successfully.
+    ...
+    mex -R2018a readObjectData.cc read_object_data.o hdf5_helper.o debug_helper.o /opt/psi/Compiler/hdf5_serial/1.8.18/gcc/6.3.0/lib/libhdf5.a -lz -ldl -output ../../ptycho_read
+    Building with 'g++'.
+    MEX completed successfully.
+    ...
+    mex -R2018a readMeasurementData.cc read_eiger_data.o hdf5_helper.o read_data_threaded.o debug_helper.o /opt/psi/Compiler/hdf5_serial/1.8.18/gcc/6.3.0/lib/libhdf5.a -lz -ldl -output ../../read_measurement
+    Building with 'g++'.
+    MEX completed successfully.
+
+The make process will procude the ptycho_read and the read_measurement MEX file in the io package: *io.ptycho_read* and *io.read_measurement*
+
+Intermediate files of the make process can be cleaned using the command
+
+    [stadler_h@ra-l-002 ptycho_reader]$ make clean
+    rm -f *~ *.o
+
+If you also want to delete the ptycho_read MEX file, use the command
+
+    [stadler_h@ra-l-002 ptycho_reader]$ make proper
+    rm -f *~ *.o
+    rm -f ../../ptycho_read.mexa64 ../../read_measurement.mexa64
+
+Code documentation in HTML format can be produced using doxygen (tested with version 1.8.13)
+
+    [stadler_h@ra-l-002 ptycho_reader]$ make doc
+    test -r doxygen.conf && rm -rf doc/html
+    test -d doc || mkdir doc
+    doxygen doxygen.conf
+    ...
+
+Hope everything works as expected!

+io/private/ptycho_reader/debug_helper.cc

@@ -0,0 +1,6 @@
+#include <fstream>
+#include <memory>
+
+namespace debug {
+    std::unique_ptr<std::ofstream> out; //!< pointer to debug log file
+}

+io/private/ptycho_reader/debug_helper.h

@@ -0,0 +1,60 @@
+/*!
+ * \file
+ * Helper code for debug handling
+ */
+
+#ifndef DEBUG_HELPER
+#define DEBUG_HELPER
+
+/*!
+ * \brief Debug functionality
+ *
+ * The environment variable PTYCHO_READ_DEBUG gives the name of the debug log file
+ */
+namespace debug {
+    extern std::unique_ptr<std::ofstream> out; //!< pointer to debug file
+
+    /*!
+     * \brief Check debug output stream
+     * \return true if stream is operational, otherwise false
+     */
+    inline bool check_out()
+    {
+        if (out.get())
+            return out.get()->good();
+        return false;
+    }
+
+    /*!
+     * \brief Initialize debug output stream
+     */
+    inline void debug_init()
+    {
+        out.reset(nullptr);
+        const char *fname = std::getenv("PTYCHO_READ_DEBUG");
+        if (fname) {
+            out.reset(new std::ofstream(fname, std::ios::app));
+            if (! out.get()->good())
+                out.reset(nullptr);
+        }
+    }
+}
+
+/*!
+ * \brief Initialize debug stream
+ */
+#define DEBUG_INIT debug::debug_init()
+
+/*!
+ * \brief Start debug code block
+ * Only access the debug stream inside such a block
+ */
+#define DEBUG if (debug::check_out())
+
+/*!
+ * \brief Access debug output stream
+ * \return debug output stream reference
+ */
+#define OUT (*debug::out.get())
+
+#endif

+io/private/ptycho_reader/env_helper.h

@@ -0,0 +1,34 @@
+#ifndef ENV_HELPER_H
+#define ENV_HELPER_H
+
+/*!
+ * \file
+ * Helper functions for environment parsing
+ */
+
+namespace env {
+
+    /*!
+     * \brief Check if environment variable steered feature is enabled
+     *
+     * \param envvar environment variable name
+     * \return true if feature is enabled
+     */
+    inline bool is_enabled (const std::string &envvar)
+    {
+        char *val = std::getenv(envvar.c_str());
+        if (! val)
+            return false;
+        std::string value(val);
+        if (value == "1")
+            return true;
+        if (value == "yes")
+            return true;
+        if (value == "true")
+            return true;
+        return false;
+    }
+
+} // namespace
+
+#endif

+io/private/ptycho_reader/hdf5_helper.cc

@@ -0,0 +1,39 @@
+/*!
+ * \file
+ * Helper definitions for HDF5 handling
+ */
+
+#include "hdf5_helper.h"
+
+namespace {
+
+    bool initialized = false;   //!< Datastructures initialized ?
+
+} // namespace
+
+namespace hdf5 {
+
+    template<> hid_t type_trait<double>::type = H5T_NATIVE_DOUBLE;                  //!< HDF5 double
+    template<> hid_t type_trait<float>::type = H5T_NATIVE_FLOAT;                    //!< HDF5 float
+    template<> hid_t type_trait<uint64_t>::type = H5T_NATIVE_UINT64;                //!< HDF5 uint64_t
+    template<> hid_t type_trait<unsigned int>::type = H5T_NATIVE_UINT;              //!< HDF5 unsigned int
+    template<> hid_t type_trait<std::complex<float>>::type = type_trait<std::complex<float>>::invalid;      //!< HDF5 complex<float>
+    template<> hid_t type_trait<std::complex<double>>::type = type_trait<std::complex<double>>::invalid;    //!< HDF5 complex<double>
+
+    void initialize()
+    {
+        if (! initialized) {
+            type_trait<std::complex<float>>::type = create_complex_type<double>();
+            type_trait<std::complex<double>>::type = create_complex_type<float>();
+        }
+    }
+
+    void cleanup()
+    {
+        if (initialized) {
+            h5type_trait::close(type_trait<std::complex<float>>::type);
+            h5type_trait::close(type_trait<std::complex<double>>::type);
+        }
+    }
+
+}

+io/private/ptycho_reader/hdf5_helper.h

@@ -0,0 +1,216 @@
+/*!
+ * \file
+ * Helper code for HDF5 handling
+ */
+
+#include <complex>
+#include <utility>
+#include "hdf5.h"
+
+/*!
+ * \brief HDF5 helper classes and functions
+ */
+namespace hdf5 {
+
+    /*!
+     * \brief Type specific HDF5 features
+     *
+     * \tparam T native type
+     */
+    template <typename T>
+    struct type_trait final {
+        static constexpr hid_t invalid = -1;    //!< invalid type value
+        static hid_t type;                      //!< HDF5 type handle for native type
+    };
+
+    /*!
+     * \brief HDF5 object with id
+     *
+     * \tparam trait trait for HDF5 object type
+     */
+    template <typename trait>
+    class obj final {
+        obj(const obj &rhs) = delete;                           //!< no copy
+        obj& operator= (const obj &rhs) = delete;               //!< no copy
+        hid_t id;                                               //!< Wrapped object id
+      public:
+        static constexpr hid_t invalid = -1;                    //!< invalid id value
+        /*!
+         * \brief Constructor
+         *
+         * \param id_ set id to this
+         */
+        explicit obj(hid_t id_) noexcept : id(id_)
+        {}
+
+        /*!
+         * \brief Destructor
+         */
+        ~obj() noexcept
+        {
+            if (valid())
+                trait::close(id);
+        }
+
+        operator hid_t() noexcept { return id; }                //!< Conversion to hid_t
+        bool valid() const noexcept { return id >= 0; }         //!< Valid? (true if id is valid)
+
+        /*!
+         * \brief Set new and return old id value
+         *
+         * \param val new id value
+         * \return new id value
+         */
+        hid_t set(hid_t val) noexcept
+        {
+            if (valid())
+                trait::close(id);
+            id = val;
+            return val;
+        }
+
+        /*!
+         * \brief Grab id and invalidate this object
+         *
+         * \return id
+         */
+        hid_t grab() noexcept
+        {
+            hid_t val = id;
+            id = invalid;
+            return val;
+        }
+    };
+
+    /*! \brief HDF5 File trait */
+    struct file_trait final {
+        static void close(hid_t id) noexcept { H5Fclose(id); }         //!< Close operation
+    };
+
+    /*! \brief HDF5 Attribute trait */
+    struct attribute_trait final {
+        static void close(hid_t id) noexcept { H5Aclose(id); }         //!< Close operation
+    };
+
+    /*! \brief HDF5 Group trait */
+    struct group_trait final {
+        static void close(hid_t id) noexcept { H5Gclose(id); }         //!< Close operation
+    };
+
+    /*! \brief HDF5 Dataset trait */
+    struct dataset_trait final {
+        static void close(hid_t id) noexcept { H5Dclose(id); }         //!< Close operation
+    };
+
+    /*! \brief HDF5 Dataspace trait */
+    struct dataspace_trait final {
+        static void close(hid_t id) noexcept { H5Sclose(id); }         //!< Close operation
+    };
+
+    /*! \brief HDF5 Type trait */
+    struct h5type_trait final {
+        static void close(hid_t id) noexcept { H5Tclose(id); }         //!< Close operation
+    };
+
+    /*! \brief HDF5 property trait */
+    struct property_trait final {
+        static void close(hid_t id) noexcept { H5Pclose(id); }         //!< Close operation
+    };
+
+    using file = obj<file_trait>;               //!< HDF5 file
+    using attribute = obj<attribute_trait>;     //!< HDF5 attribute
+    using group = obj<group_trait>;             //!< HDF5 group
+    using dataset = obj<dataset_trait>;         //!< HDF5 dataset
+    using dataspace = obj<dataspace_trait>;     //!< HDF5 dataspace
+    using h5type = obj<h5type_trait>;           //!< HDF5 type
+    using property = obj<property_trait>;       //!< HDF5 property
+
+    /*!
+     * \brief HDF5 exception
+     */
+    class exception final : public std::exception {
+        std::string msg;    //!< error message
+
+      public:
+        /*!
+         * \brief Constructor
+         *
+         * \param message error message
+         */
+        exception(std::string &&message)
+            : msg(message)
+        {}
+
+        /*!
+         * \brief Destructor
+         */
+        virtual ~exception() {}
+
+        /*!
+         * \brief Get error message
+         *
+         * \return error message
+         */
+        const char* what() const noexcept override
+        {
+            return msg.c_str();
+        }
+    };
+
+    /*!
+     * \brief Check hdf5 result
+     *
+     * \throw exception if result is not ok
+     */
+    inline void check_result(herr_t id, std::string &&message) {
+        if (id < 0)
+            throw exception(std::forward<decltype(message)>(message));
+    }
+
+    /*!
+     * \brief Check hdf5 object
+     *
+     * \throw exception if object is not ok
+     */
+    inline void check_object(hid_t id, std::string &&message) {
+        if (id < 0)
+            throw exception(std::forward<decltype(message)>(message));
+    }
+
+    /*!
+     * \brief Initialize data
+     *
+     * Must be called before using the complex data types
+     */
+    void initialize();
+
+    /*!
+     * \brief Cleanup data
+     */
+    void cleanup();
+
+    /*!
+     * \brief Convenience initializer class
+     */
+    struct initializer final {
+        initializer() { initialize(); } //!< Constructor initializes internal hdf5 helper data structures
+        ~initializer() { cleanup(); }   //!< Destructor cleans up internal hdf5 helper data structures
+    };
+
+    /*!
+     * \brief Create complex compound type
+     *
+     * \return type id
+     * \throw hdf5::exception if an error happens
+     */
+    template <typename T>
+    hid_t create_complex_type()
+    {
+        hdf5::h5type type(H5Tcreate(H5T_COMPOUND, sizeof(std::complex<T>)));
+        hdf5::check_result(type, "Unable to create complex compound type");
+        hdf5::check_result(H5Tinsert(type, "r", 0, hdf5::type_trait<T>::type), "Unable to insert first complex compound member");
+        hdf5::check_result(H5Tinsert(type, "i", sizeof(T), hdf5::type_trait<T>::type), "Unable to insert second complex compound member");
+        return type.grab();
+    }
+
+} // namespace hdf5

+io/private/ptycho_reader/mex_helper.h

@@ -0,0 +1,444 @@
+#ifndef MEX_HELPER
+#define MEX_HELPER
+
+/*!
+ * \file
+ * Helper code for MEX data
+ *
+ * include \<cstdint\> before this one
+ */
+
+/*!
+ * \brief Unique pointer for MATLAB allocated object
+ *
+ * \tparam T type of object
+ */
+template <typename T>
+struct mx_deleter final {
+    /*!
+     * \brief deleter for pointer
+     *
+     * \param p pointer
+     */
+    void operator()(T *p) const noexcept {
+        if (p)
+            mxFree(p);
+    }
+};
+
+/*!
+ * \brief Unique pointer for MATLAB array
+ */
+template <>
+struct mx_deleter<mxArray> final {
+    /*!
+     * \brief deleter for MATLAB array
+     *
+     * \param p MATLAB array
+     */
+    void operator()(mxArray *p) const noexcept {
+        mxDestroyArray(p);
+    }
+};
+
+/*!
+ * \brief unique_ptr for MATLAB objects
+ *
+ * \tparam T object type
+ */
+template <typename T>
+using mx_ptr = std::unique_ptr<T, mx_deleter<T>>;
+
+/*!
+ * \brief Type specific MATLAB features
+ *
+ * \tparam T native type
+ */
+template <typename T>
+struct mx_trait final {
+};
+
+/*!
+ * \brief Numeric real
+ */
+struct mx_numeric {
+    /*!
+     * \brief first int64_t
+     * \param pa array
+     * \param index array index
+     * \return first int64_t in data
+     */
+    virtual int64_t get_int64(const mxArray *pa, std::size_t index=0) = 0;
+};
+
+/*!
+ * \brief Double specific MATLAB features
+ */
+template <>
+struct mx_trait<double> final : public mx_numeric {
+    using type = double;                                    //!< base type
+    using complex_type = mxComplexDouble;                   //!< complex type
+    constexpr static mxClassID class_id = mxDOUBLE_CLASS;   //!< MATLAB class id
+
+    /*!
+     * \brief Get data pointer
+     *
+     * \param pa complex MATLAB array
+     * \return pointer to complex array data
+     */
+    static complex_type* get_complex(mxArray *pa)
+    {
+        return mxGetComplexDoubles(pa);
+    }
+    /*!
+     * \brief Get data pointer
+     * \param pa MATLAB array
+     * \return pointer to array data
+     */
+    static type* get(mxArray *pa)
+    {
+        return mxGetDoubles(pa);
+    }
+    /*!
+     * \brief first int64_t
+     * \param pa array
+     * \param index array index
+     * \return first int64_t in data
+     */
+    int64_t get_int64(const mxArray *pa, std::size_t index=0) override
+    {
+        return *(get(const_cast<mxArray*>(pa)) + index);
+    }
+};
+
+/*!
+ * \brief Float specific MATLAB features
+ */
+template <>
+struct mx_trait<float> final : public mx_numeric {
+    using type = float;                                     //!< base type
+    using complex_type = mxComplexSingle;                   //!< complex type
+    constexpr static mxClassID class_id = mxSINGLE_CLASS;   //!< MATLAB class id
+
+    /*!
+     * \brief Get data pointer
+     *
+     * \param pa complex MATLAB array
+     * \return pointer to complex array data
+     */
+    static complex_type* get_complex(mxArray *pa)
+    {
+        return mxGetComplexSingles(pa);
+    }
+    /*!
+     * \brief Get data pointer
+     * \param pa MATLAB array
+     * \return pointer to array data
+     */
+    static type* get(mxArray *pa)
+    {
+        return mxGetSingles(pa);
+    }
+    /*!
+     * \brief first int64_t
+     * \param pa array
+     * \param index array index
+     * \return first int64_t in data
+     */
+    int64_t get_int64(const mxArray *pa, std::size_t index=0) override
+    {
+        return *(get(const_cast<mxArray*>(pa)) + index);
+    }
+};
+
+/*!
+ * \brief int8 specific MATLAB features
+ */
+template <>
+struct mx_trait<int8_t> final : public mx_numeric {
+    using type = int8_t;                                    //!< base type
+    constexpr static mxClassID class_id = mxINT8_CLASS;     //!< MATLAB class id
+    /*!
+     * \brief Get data pointer
+     * \param pa MATLAB array
+     * \return pointer to array data
+     */
+    static type* get(mxArray *pa)
+    {
+        return mxGetInt8s(pa);
+    }
+    /*!
+     * \brief first int64_t
+     * \param pa array
+     * \param index array index
+     * \return first int64_t in data
+     */
+    int64_t get_int64(const mxArray *pa, std::size_t index=0) override
+    {
+        return *(get(const_cast<mxArray*>(pa)) + index);
+    }
+};
+
+/*!
+ * \brief uint8 specific MATLAB features
+ */
+template <>
+struct mx_trait<uint8_t> final : public mx_numeric {
+    using type = uint8_t;                                   //!< base type
+    constexpr static mxClassID class_id = mxUINT8_CLASS;    //!< MATLAB class id
+    /*!
+     * \brief Get data pointer
+     * \param pa MATLAB array
+     * \return pointer to array data
+     */
+    static type* get(mxArray *pa)
+    {
+        return mxGetUint8s(pa);
+    }
+    /*!
+     * \brief first int64_t
+     * \param pa array
+     * \param index array index
+     * \return first int64_t in data
+     */
+    int64_t get_int64(const mxArray *pa, std::size_t index=0) override
+    {
+        return *(get(const_cast<mxArray*>(pa)) + index);
+    }
+};
+
+/*!
+ * \brief int16 specific MATLAB features
+ */
+template <>
+struct mx_trait<int16_t> final : public mx_numeric {
+    using type = int16_t;                                   //!< base type
+    constexpr static mxClassID class_id = mxINT16_CLASS;    //!< MATLAB class id
+    /*!
+     * \brief Get data pointer
+     * \param pa MATLAB array
+     * \return pointer to array data
+     */
+    static type* get(mxArray *pa)
+    {
+        return mxGetInt16s(pa);
+    }
+    /*!
+     * \brief first int64_t
+     * \param pa array
+     * \param index array index
+     * \return first int64_t in data
+     */
+    int64_t get_int64(const mxArray *pa, std::size_t index=0) override
+    {
+        return *(get(const_cast<mxArray*>(pa)) + index);
+    }
+};
+
+/*!
+ * \brief uint16 specific MATLAB features
+ */
+template <>
+struct mx_trait<uint16_t> final : public mx_numeric {
+    using type = uint16_t;                                  //!< base type
+    constexpr static mxClassID class_id = mxUINT16_CLASS;   //!< MATLAB class id
+    /*!
+     * \brief Get data pointer
+     * \param pa MATLAB array
+     * \return pointer to array data
+     */
+    static type* get(mxArray *pa)
+    {
+        return mxGetUint16s(pa);
+    }
+    /*!
+     * \brief first int64_t
+     * \param pa array
+     * \param index array index
+     * \return first int64_t in data
+     */
+    int64_t get_int64(const mxArray *pa, std::size_t index=0) override
+    {
+        return *(get(const_cast<mxArray*>(pa)) + index);
+    }
+};
+
+/*!
+ * \brief int32 specific MATLAB features
+ */
+template <>
+struct mx_trait<int32_t> final : public mx_numeric {
+    using type = int32_t;                                   //!< base type
+    constexpr static mxClassID class_id = mxINT32_CLASS;    //!< MATLAB class id
+    /*!
+     * \brief Get data pointer
+     * \param pa MATLAB array
+     * \return pointer to array data
+     */
+    static type* get(mxArray *pa)
+    {
+        return mxGetInt32s(pa);
+    }
+    /*!
+     * \brief first int64_t
+     * \param pa array
+     * \param index array index
+     * \return first int64_t in data
+     */
+    int64_t get_int64(const mxArray *pa, std::size_t index=0) override
+    {
+        return *(get(const_cast<mxArray*>(pa)) + index);
+    }
+};
+
+/*!
+ * \brief uint32 specific MATLAB features
+ */
+template <>
+struct mx_trait<uint32_t> final : public mx_numeric {
+    using type = uint32_t;                                  //!< base type
+    constexpr static mxClassID class_id = mxUINT32_CLASS;   //!< MATLAB class id
+    /*!
+     * \brief Get data pointer
+     * \param pa MATLAB array
+     * \return pointer to array data
+     */
+    static type* get(mxArray *pa)
+    {
+        return mxGetUint32s(pa);
+    }
+    /*!
+     * \brief first int64_t
+     * \param pa array
+     * \param index array index
+     * \return first int64_t in data
+     */
+    int64_t get_int64(const mxArray *pa, std::size_t index=0) override
+    {
+        return *(get(const_cast<mxArray*>(pa)) + index);
+    }
+};
+
+/*!
+ * \brief int64 specific MATLAB features
+ */
+template <>
+struct mx_trait<int64_t> final : public mx_numeric {
+    using type = int64_t;                                   //!< base type
+    constexpr static mxClassID class_id = mxINT64_CLASS;    //!< MATLAB class id
+    /*!
+     * \brief Get data pointer
+     * \param pa MATLAB array
+     * \return pointer to array data
+     */
+    static type* get(mxArray *pa)
+    {
+        return mxGetInt64s(pa);
+    }
+    /*!
+     * \brief first int64_t
+     * \param pa array
+     * \param index array index
+     * \return first int64_t in data
+     */
+    int64_t get_int64(const mxArray *pa, std::size_t index=0) override
+    {
+        return *(get(const_cast<mxArray*>(pa)) + index);
+    }
+};
+
+/*!
+ * \brief uint64 specific MATLAB features
+ */
+template <>
+struct mx_trait<uint64_t> final : public mx_numeric {
+    using type = uint64_t;                                  //!< base type
+    constexpr static mxClassID class_id = mxUINT64_CLASS;   //!< MATLAB class id
+    /*!
+     * \brief Get data pointer
+     * \param pa MATLAB array
+     * \return pointer to array data
+     */
+    static type* get(mxArray *pa)
+    {
+        return mxGetUint64s(pa);
+    }
+    /*!
+     * \brief first int64_t
+     * \param pa array
+     * \param index array index
+     * \return first int64_t in data
+     */
+    int64_t get_int64(const mxArray *pa, std::size_t index=0) override
+    {
+        return *(get(const_cast<mxArray*>(pa)) + index);
+    }
+};
+
+/*!
+ * \brief get numeric trait for real numeric array
+ * \param pa array
+ * \return numeric trait or nullptr if not real numeric
+ */
+inline mx_numeric* get_numeric_trait(const mxArray *pa)
+{
+    if (mxIsComplex(pa))
+        return nullptr;
+    switch (mxGetClassID(pa)) {
+        case mxDOUBLE_CLASS:
+            return new mx_trait<double>;
+        case mxSINGLE_CLASS:
+            return new mx_trait<float>;
+        case mxINT8_CLASS:
+            return new mx_trait<int8_t>;
+        case mxUINT8_CLASS:
+            return new mx_trait<uint8_t>;
+        case mxINT16_CLASS:
+            return new mx_trait<int16_t>;
+        case mxUINT16_CLASS:
+            return new mx_trait<uint16_t>;
+        case mxINT32_CLASS:
+            return new mx_trait<int32_t>;
+        case mxUINT32_CLASS:
+            return new mx_trait<uint32_t>;
+        case mxINT64_CLASS:
+            return new mx_trait<int64_t>;
+        case mxUINT64_CLASS:
+            return new mx_trait<uint64_t>;
+    }
+    return nullptr;
+}
+
+template <>
+struct mx_trait<char> final {
+    constexpr static mxClassID class_id = mxCHAR_CLASS;     //!< MATLAB class id
+    /*!
+     * \brief Get string
+     * \param pa array
+     * \return string
+     */
+    static std::string get_string(const mxArray *pa)
+    {
+        const std::size_t nchars = mxGetN(pa);
+        std::unique_ptr<char> buf(new char[nchars+1]);
+        if (mxGetString(pa, buf.get(), nchars+1))
+            return std::string();
+        return std::string(buf.get(), nchars);
+    }
+};
+
+/*!
+ * \brief get char trait for class id
+ * \param pa array
+ * \return numeric trait or nullptr if not numeric
+ */
+inline mx_trait<char>* get_char_trait(const mxArray *pa)
+{
+    switch (mxGetClassID(pa)) {
+        case mxCHAR_CLASS:
+            return new mx_trait<char>;
+    }
+    return nullptr;
+}
+
+#endif // ifndef MEX_HELPER

+io/private/ptycho_reader/multi_processing.h

@@ -0,0 +1,268 @@
+#ifndef MULTI_PROCESSING_H
+#define MULTI_PROCESSING_H
+
+/*!
+ * \file
+ * Provide functionality for multi processing
+ */
+
+/*!
+ * \brief Multiprocessing support functions and types
+ */
+namespace mp {
+
+    /*!
+     * \brief Buffer object managing destination and shared memory areas
+     * \tparam e_type buffer element type
+     */
+    template <typename e_type>
+    struct buf {
+        const unsigned int msg_size;        //!< maximum message size per process
+        e_type *destination;                //!< pointer to destination memory
+        e_type *shared;                     //!< pointer to shared memory
+        std::size_t length;                 //!< total data elements
+        std::size_t chunk_size;             //!< data elements for one chunk
+        int chunks_per_proc;                //!< number of chunks for a process to handle
+        int first_full;                     //!< first process with non reduced number of chunks
+        int num_procs;                      //!< number of read processes
+
+        /*!
+         * \brief Constructor
+         *
+         * \param dest Destination pointer
+         * \param dims Result matrix dimensions
+         * \param nprocs Number of read processes
+         * \param message_size Size of message area per child process in bytes
+         */
+        buf(e_type *dest, const std::vector<mwSize> &dims, int nprocs, unsigned int message_size=64u)
+            : msg_size(message_size ? message_size : 64u), destination(dest), shared(nullptr), length(0), chunk_size(1), chunks_per_proc(0), first_full(0), num_procs(nprocs)
+        {
+            assert(! dims.empty());
+            assert(nprocs >= 1);
+            int nchunks = dims[0];
+            chunks_per_proc = (nchunks + nprocs - 1) / nprocs;
+            first_full = (nprocs * chunks_per_proc) - nchunks;
+
+            for (int i=1; i<dims.size(); i++)
+                chunk_size *= dims[i];
+            length = nchunks * chunk_size;
+
+            DEBUG {
+                OUT << nprocs << "np " << nchunks << "nc " << chunks_per_proc << "cpp " << first_full << "ff " << chunk_size << "cs " << length << 'l' << std::endl;
+            }
+
+            if (nprocs > 1) {
+                void *sbuf = mmap(nullptr, (length - offset(1) * chunk_size) * sizeof(e_type) + (num_procs - 1) * msg_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+                if (sbuf == MAP_FAILED)
+                    throw std::runtime_error(std::string("mmap error - ") + strerror(errno));
+                shared = static_cast<e_type *>(sbuf);
+            }
+        }
+
+        /*!
+         * \brief Destructor
+         */
+        ~buf() noexcept
+        {
+            if (shared)
+                munmap(shared, (length - offset(1) * chunk_size) * sizeof(e_type) + (num_procs - 1) * msg_size);
+        }
+
+        /*!
+         * \brief Get shareable buffer pointer
+         *
+         * \param proc process number
+         * \return Pointer to buffer area accessible from parent process
+         */
+        e_type *get(int proc) noexcept
+        {
+            if (! proc) {
+                DEBUG {
+                    OUT << "buffer " << proc << ": destination+0 at address " << destination << std::endl;
+                }
+                return destination;
+            }
+            std::size_t buf_start = offset(proc, true) * chunk_size;
+            DEBUG {
+                OUT << "buffer " << proc << ": shared+" << buf_start << " at address " << &shared[buf_start] << std::endl;
+            }
+            return &shared[buf_start];
+        }
+
+        /*!
+         * \brief Destination buffer
+         *
+         * \param proc process number
+         * \return Pointer to destination buffer area for process proc
+         */
+        e_type *get_dest(int proc) noexcept
+        {
+            return destination + offset(proc) * chunk_size;
+        }
+
+        /*!
+         * \brief Copy data in shared memory to destination
+         */
+        void copy() noexcept
+        {
+            if (shared) {
+                DEBUG {
+                    OUT << "Copy " << (length - offset(1) * chunk_size) << " elements to destination+" << (offset(1) * chunk_size) << std::endl;
+                }
+                std::copy(shared, shared + (length - offset(1) * chunk_size), destination + (offset(1) * chunk_size));
+            }
+        }
+
+        /*!
+         * \brief Object offset
+         *
+         * \param proc process number
+         * \param shared offset into shared memory?
+         * \return First chunk index for proc
+         */
+        int offset(int proc, bool shared=false) noexcept
+        {
+            int offset = proc * chunks_per_proc;
+            if (proc < first_full)
+                offset -= proc;
+            else
+                offset -= first_full;
+            if (shared)
+                offset -= chunks_per_proc - (first_full ? 1 : 0);
+            return offset;
+        }
+
+        /*!
+         * \brief Get pointer to message
+         *
+         * For child processes returns a pointer to the message area of size msg_size
+         *
+         * \param proc process number (must not be the parent process 0)
+         * \return pointer to message area
+         */
+        char *get_msg(int proc)
+        {
+            if (proc)
+                return reinterpret_cast<char *>(&shared[length - offset(1) * chunk_size]) + (proc - 1) * msg_size;
+            throw std::logic_error("no message area for parent process");
+            return nullptr;
+        }
+
+        /*!
+         * \brief Record message
+         *
+         * Used to record a child process error message in the shared memory area
+         *
+         * \param msg message
+         * \param proc process number (must not be the parent process 0)
+         */
+        void message(const char *msg, int proc)
+        {
+            std::strncpy(get_msg(proc), msg, msg_size-1);
+        }
+    }; // struct buf
+
+    struct exception : public std::exception {
+        bool converted = false;
+        int process = 0;
+        std::string error_message;
+
+        inline exception ()
+        {}
+
+        inline exception (int proc, const std::string &msg)
+            : process(proc), error_message(msg)
+        {}
+
+        inline const char* what() const noexcept override
+        {
+            try {
+                if (! converted) {
+                    std::string &msg = const_cast<std::string&>(error_message);
+                    std::ostringstream oss;
+                    oss << msg << " (proc " << process << ')';
+                    msg = oss.str();
+                    const_cast<bool&>(converted) = true;
+                }
+            } catch (...) {}
+            return error_message.c_str();
+        }
+    }; // struct exception
+
+    template<typename e_type>
+    inline void run (int nprocs, buf<e_type> &buffer,
+                     std::function<void(int, buf<e_type>&)> &&func)
+    {
+        if (nprocs <= 0)
+            return;
+
+        int proc = 0;             // process number starting with 0
+        int pids[nprocs] = {0};   // first entry (parent) unused
+        try {
+            for (proc=nprocs-1; proc; --proc) {
+                if ((pids[proc] = fork()) < 0) {
+                    proc = 0;
+                    throw exception(0, "process spawning failed");
+                }
+                if (! pids[proc])
+                    break;
+            }
+
+            DEBUG_INIT;
+            func(proc, buffer);
+
+            if (proc) {
+                DEBUG {
+                    OUT << "Process " << proc << " finished" << std::endl;
+                }
+                std::_Exit((EXIT_SUCCESS));
+            }
+        } catch (std::exception &ex) {
+            DEBUG {
+                OUT << "Process " << proc << " exception: " << ex.what() << std::endl;
+            }
+            if (proc) {
+                buffer.message(ex.what(), proc);
+                std::_Exit((EXIT_FAILURE));
+            }
+            for (proc=nprocs-1; proc; --proc) {
+                if (pids[proc] > 0) {
+                    int status;
+                    waitpid(pids[proc], &status, 0); // prevent zombies
+                }
+            }
+            throw;
+        }
+
+        bool failed = false;
+        exception ex;
+        for (proc=nprocs-1; proc; --proc) {
+            int status;
+            bool with_error = false;
+            if (waitpid(pids[proc], &status, 0) != pids[proc]) {
+                with_error = true;
+                ex.process = proc;
+                ex.error_message = std::strerror(errno);
+            } else if (! WIFEXITED(status)) {
+                with_error = true;
+                ex.process = proc;
+                ex.error_message = "did not exit normally";
+            } else if (WEXITSTATUS(status) != (EXIT_SUCCESS)) {
+                with_error = true;
+                ex.process = proc;
+                ex.error_message = buffer.get_msg(proc);
+            }
+            DEBUG {
+                OUT << "Process " << proc << ": finished " << (with_error ? "with error" : "successfully") << std::endl;
+            }
+            failed |= with_error;
+        }
+        if (failed)
+            throw ex;
+
+        buffer.copy();
+    }
+
+} // namespace mp
+
+#endif // ifndef MULTI_PROCESSING_H

+io/private/ptycho_reader/precision.h

@@ -0,0 +1,59 @@
+#ifndef PRECISION_H
+#define PRECISION_H
+
+/*!
+ * \file
+ * Precision handling support
+ *
+ * \pre needs standard exceptions and string
+ */
+
+/*!
+ * \brief Precision related functions and types
+ */
+namespace precision {
+
+    /*!
+     * \brief Single or double precision
+     */
+    enum type : unsigned char {
+        Single,    //!< single precision
+        Double     //!< double precision
+    };
+
+    /*!
+     * \brief Precision from string
+     * \arg str precision string
+     * \return precision
+     */
+    inline type from_str(const std::string &str)
+    {
+        type prec = Single;
+        if (str == "double")
+            prec = Double;
+        else if (str != "single")
+            throw std::invalid_argument("undefined precision string, must be 'single' or 'double'");
+        return prec;
+    }
+
+    /*!
+     * \brief Precision to string
+     * \arg prec precision
+     * \return string
+     */
+    inline const char* to_str(type prec)
+    {
+        switch (prec) {
+            case Single:
+                return "single";
+            case Double:
+                return "double";
+            default:
+                throw std::invalid_argument("undefined precision value");
+        }
+        return "undef";
+    }
+
+} // namespace precision
+
+#endif

+io/private/ptycho_reader/readMeasurementData.cc

@@ -0,0 +1,310 @@
+/*!
+ * \file
+ * Read Measurement Data
+ *
+ * This file contains the main mex code for reading measurement data into MATLAB memory.
+ */
+
+#include <string>
+#include <memory>
+#include <vector>
+#include <stdexcept>
+#include <fstream>
+#include "mex.h"
+#include "precision.h"
+#include "read_eiger_data.h"
+#include "read_data_threaded.h"
+#include "debug_helper.h"
+
+namespace {
+
+    namespace p {
+        constexpr char format[] = "extension";                          //!< extension field
+        constexpr char image_size[] = "asize";                          //!< image size field
+        constexpr char roi_center[] = "ctr";                            //!< ROI center field
+        constexpr char scan_number[] = "scan_number";                   //!< scan number field
+        constexpr char data_path[] = "data_path";                       //!< data path field
+        constexpr char data_location[] = "data_location";               //!< data location field
+        constexpr char data_prefix[] = "data_prefix";                   //!< data prefix field
+        constexpr char scan_string_format[] = "scan_string_format";     //!< scan string format field
+        constexpr char precision[] = "precision";                       //!< precision string ('single' or 'double')
+        constexpr char nthreads[] = "nthreads";                         //!< number of parallel threads or processes
+    } // namespace p
+
+    #include "env_helper.h"
+    #include "mex_helper.h"
+
+    /*!
+     * \brief Get field from struct
+     *
+     * \param pa Pointer to MATLAB struct array
+     * \param field_name Field name
+     * \return Pointer to MATALAB array of field value
+     */
+    mxArray* getField(const mxArray *pa, const char *field_name)
+    {
+        mxArray *field = mxGetField(pa, 0, field_name);
+        if (! field)
+            throw std::invalid_argument(std::string("Field ")+field_name+" not found in structure!");
+        return field;
+    }
+
+    /*!
+     * \brief Get char field
+     *
+     * \param pa MATLAB struct array
+     * \param field_name field name
+     * \return field string value
+     */
+    std::string getCharField(const mxArray *pa, const char *field_name)
+    {
+        mxArray *field = getField(pa, field_name);
+        if (! mxIsChar(field))
+            mexErrMsgIdAndTxt("psi:ptycho:arg:field:type:notchar", "Field %s not a char array!", field_name);
+        if (mxGetNumberOfDimensions(field)!=2 || mxGetM(field)!=1)
+            mexErrMsgIdAndTxt("psi:ptycho:arg:field:dim", "Field %s is not a single char array!", field_name);
+        std::size_t nchars = mxGetN(field);
+        char buf[nchars+1];
+        if (mxGetString(field, buf, nchars+1))
+            mexErrMsgIdAndTxt("psi:ptycho:string:extract", "Unable to extract characters of field %s!", field_name);
+        return buf;
+    }
+
+    /*!
+     * \brief Assign values to vector
+     *
+     * \param a Vector of length at least nelements
+     * \param b Pointer to at least nelements values
+     * \param nelements number of elements to assign
+     * \tparam A Vector element type
+     * \tparam B Value type
+     */
+    template <typename A, typename B>
+    void assign(std::vector<A> &a, const B *b, std::size_t nelements)
+    {
+        if (b == nullptr)
+            mexErrMsgIdAndTxt("psi:ptycho:arg:field:retrieval", "Unable to extract values from numeric vector field!");
+        for (std::size_t i=0; i<nelements; i++)
+            a[i] = b[i];
+    }
+
+    /*!
+     * \brief Get vector field with long integer values
+     *
+     * \param pa MATLAB struct array
+     * \param field_name field name
+     * \param nelements expected number of elements (0 means undefined and is the default)
+     * \return field string value
+     * \tparam T Vector element type
+     */
+    template <typename T>
+    std::vector<T> getVectorField(const mxArray *pa, const char *field_name, unsigned int nelements=0)
+    {
+        mxArray *field = getField(pa, field_name);
+        if (! mxIsNumeric(field))
+            mexErrMsgIdAndTxt("psi:ptycho:arg:field:type:notnumeric", "Field %s not a numeric array!", field_name);
+        if (mxGetNumberOfDimensions(field)!=2 || mxGetM(field)!=1)
+            mexErrMsgIdAndTxt("psi:ptycho:arg:field:dim", "Field %s is not a single numeric array!", field_name);
+        std::size_t N = mxGetN(field);
+        if (nelements && N != nelements)
+            mexErrMsgIdAndTxt("psi:ptycho:arg:field:length", "Field %s doesn't have %u elements!", field_name, nelements);
+        mxClassID mx_class = mxGetClassID(field);
+        std::vector<T> buf(N);
+        switch (mx_class) {
+            case mxINT8_CLASS:
+                assign(buf, mxGetInt8s(field), N);
+                break;
+            case mxUINT8_CLASS:
+                assign(buf, mxGetUint8s(field), N);
+                break;
+            case mxINT16_CLASS:
+                assign(buf, mxGetInt16s(field), N);
+                break;
+            case mxUINT16_CLASS:
+                assign(buf, mxGetUint16s(field), N);
+                break;
+            case mxINT32_CLASS:
+                assign(buf, mxGetInt32s(field), N);
+                break;
+            case mxUINT32_CLASS:
+                assign(buf, mxGetUint32s(field), N);
+                break;
+            case mxINT64_CLASS:
+                assign(buf, mxGetInt64s(field), N);
+                break;
+            case mxUINT64_CLASS:
+                assign(buf, mxGetUint64s(field), N);
+                break;
+            case mxSINGLE_CLASS:
+                assign(buf, mxGetSingles(field), N);
+                break;
+            case mxDOUBLE_CLASS:
+                assign(buf, mxGetDoubles(field), N);
+                break;
+            default:
+                std::string class_name(mxGetClassName(field));
+                mexErrMsgIdAndTxt("psi:ptycho:arg:field:class", "Field %s doesn't contain integer elements (it has class %s)!", field_name, class_name.c_str());
+        }
+        return buf;
+    }
+
+    std::vector<std::string> getStrings(const mxArray *pa, const char *field_name, unsigned int nelements=0)
+    {
+        mxArray *field = getField(pa, field_name);
+        if (mxGetM(field) != 1)
+            mexErrMsgIdAndTxt("psi:ptycho:arg:wrongDimensions:notOne", "Argument %s must be a 1xN one-dimensional character or cell array!", field_name);
+        std::unique_ptr<mx_trait<char>> ct(get_char_trait(field));
+        if (ct.get()) { // char array
+            return std::vector<std::string>({ ct->get_string(field) });
+        } else {        // cell array
+            if (! mxIsCell(field))
+                mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:type", "Argument %s must be either character or cell array!", field_name);
+            std::vector<std::string> result_vec;
+            for (std::size_t j=0; j<mxGetN(field); j+=1) {
+                mxArray *pa = mxGetCell(field, j);
+                if (mxGetM(pa) != 1)
+                    mexErrMsgIdAndTxt("psi:ptycho:arg:wrongDimensions:notOne", "Cell fields in argument %s must be one-dimensional character arrays!", field_name);
+                ct.reset(get_char_trait(pa));
+                if (! ct.get())
+                    mexErrMsgIdAndTxt("psi:ptycho:arg:wrongType:notChar", "Cell field in argument %s must be of type char!", field_name);
+                result_vec.push_back(ct->get_string(pa));
+                if (! result_vec.back().size())
+                    mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:empty", "Empty string in %s argument!", field_name);
+            }
+            if (! result_vec.size())
+                mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:empty", "Empty %s argument!", field_name);
+            return result_vec;
+        }
+    }
+
+} // namespace
+
+/*!
+ * \brief MEX function readMeasurementData
+ *
+ * Input arguments:
+ * 0 : csax p structure
+ *
+ * Output arguments:
+ * 0 : Measurement data (3d or 4d array [cols, rows, n_bursts, n_positions], n_burst is optional and only used if the HHDF5 dataset is 4d)
+ *
+ * \param nlhs Number of left hand side (result) arguments
+ * \param plhs Pointer array to left hand side arguments
+ * \param nrhs Number of right hand side (input) arguments
+ * \param prhs Pointer array to right hand side arguments
+ */
+void mexFunction(int nlhs, mxArray *plhs[],
+                 int nrhs, const mxArray *prhs[])
+{
+    constexpr unsigned int ninputs = 1;
+    constexpr unsigned int noutputs = 1;
+
+    DEBUG_INIT;
+    if (nrhs != ninputs)
+        mexErrMsgIdAndTxt("psi:ptycho:arg:in:wrongNumber", "Wrong number of arguments, use: array = readMeasurementData(argument structure)");
+    if (nlhs != noutputs)
+        mexErrMsgIdAndTxt("psi:ptycho:arg:out:wrongNumber", "Wrong number of output arguments, use: array = readObjectData(argument structure)");
+
+    if (! mxIsStruct(prhs[0]))
+        mexErrMsgIdAndTxt("psi:ptycho:arg:wrongType:notStruct", "Argument must be of type struct!");
+    if (mxGetNumberOfDimensions(prhs[0])!=2 || mxGetM(prhs[0])!=1 || mxGetN(prhs[0])!=1)
+        mexErrMsgIdAndTxt("psi:ptycho:arg:wrongDimensions:notOne", "Argument must be a 1x1 struct array!");
+
+    bool verbose = env::is_enabled("PTYCHO_READ_VERBOSE");
+
+    precision::type prec = precision::type::Single;
+    try {
+        prec = precision::from_str(getCharField(prhs[0], p::precision));
+    } catch (std::exception &ex) {
+        mexErrMsgIdAndTxt("psi:ptycho:arg:precision", "Precision field error: %s", ex.what());
+    }
+    try {
+        std::string format(getCharField(prhs[0], p::format));
+        std::vector<long> image_size;
+        try {
+            image_size = getVectorField<long>(prhs[0], p::image_size, 2);
+        } catch (std::invalid_argument &ex) {}
+        std::vector<long> roi_center;
+        try {
+            roi_center = getVectorField<long>(prhs[0], p::roi_center, 2);
+        } catch (std::invalid_argument &ex) {}
+        std::vector<long> nthreads(getVectorField<long>(prhs[0], p::nthreads, 1));
+        std::vector<std::string> data_path(getStrings(prhs[0], p::data_path));
+        // std::string scan_string_format(getCharField(prhs[0], p::scan_string_format));
+        // std::string data_prefix(getCharField(prhs[0], p::data_prefix));
+        // std::vector<long> scan_number(getVectorField<long>(prhs[0], p::scan_number));
+
+        // Format dependent arguments
+        std::vector<std::string> data_location;
+        if (format == "h5")
+            data_location = getStrings(prhs[0], p::data_location);
+
+        if (verbose) {
+            mexPrintf("extension: %s\n", format.c_str());
+            if (data_path.size() == 1) {
+                mexPrintf("data_path: %s\n", data_path[0].c_str());
+            } else {
+                mexPrintf("data_path first: %s\n", data_path[0].c_str());
+                mexPrintf("data_path last : %s\n", data_path.back().c_str());
+            }
+            mexPrintf("roi_center: ");
+            if (roi_center.empty()) {
+                mexPrintf("from data\n");
+            } else {
+                mexPrintf("%ld %ld\n", roi_center[0], roi_center[1]);
+            }
+            mexPrintf("image_size: ");
+            if (image_size.empty()) {
+                mexPrintf("from data\n");
+            } else {
+                mexPrintf("%ld %ld\n", image_size[0], image_size[1]);
+            }
+            mexPrintf("precision: %s\n", precision::to_str(prec));
+            //     mexPrintf("data_prefix: %s\n", data_prefix.c_str());
+            //     mexPrintf("scan_number: [ ");
+            //     for (long l : scan_number)
+            //         mexPrintf("%ld ", l);
+            //     mexPrintf("]\n");
+            const char *debuglog = std::getenv("PTYCHO_READ_DEBUG");
+            mexPrintf("debuglog: %s - ", (debuglog ? debuglog : "not set"));
+            DEBUG {
+                mexPrintf("ok\n");
+                OUT << "----------- read measurement data ----------------\n";
+            } else {
+                mexPrintf("not functional\n");
+            }
+
+            // Format dependend verbose output
+            if (format == "h5") {
+                if (data_location.size() == 1) {
+                    mexPrintf("data_location: %s\n", data_location[0].c_str());
+                } else {
+                    mexPrintf("data_location first: %s\n", data_location[0].c_str());
+                    mexPrintf("data_location last : %s\n", data_location.back().c_str());
+                }
+            }
+        }
+
+        if (format == "h5") {
+            DEBUG {
+                OUT << "Reading Eiger data..." << std::endl;
+            }
+            plhs[0] = data_prep::read_eiger_data(nthreads[0], data_path, data_location, image_size, roi_center, prec);
+        } else if (format == "cbf") {
+            DEBUG {
+                OUT << "Reading Pilatus data..." << std::endl;
+            }
+            plhs[0] = data_prep::read_data_threaded("pilatus", nthreads[0], data_path, image_size, roi_center, prec);
+        } else if (format == "tiff") {
+            DEBUG {
+                OUT << "Reading Moench data..." << std::endl;
+            }
+            plhs[0] = data_prep::read_data_threaded("moench", nthreads[0], data_path, image_size, roi_center, prec);
+        } else {
+            mexErrMsgIdAndTxt("psi:ptycho:format:unknown", "Unknown extension: %s", format.c_str());
+        }
+    } catch (std::exception &ex) {
+        mexErrMsgIdAndTxt("psi:ptycho:struct:arg", "%s", ex.what());
+    }
+    return;
+}

+io/private/ptycho_reader/readObjectData.cc

@@ -0,0 +1,169 @@
+/*!
+ * \file
+ * Read Object Data
+ *
+ * This file contains the main mex code for reading object data into MATLAB memory.
+ */
+
+#include <string>
+#include <cstdlib>
+#include <cstdint>
+#include <memory>
+#include <vector>
+#include <stdexcept>
+#include <fstream>
+#include "precision.h"
+#include "env_helper.h"
+#include "mex.h"
+#include "mex_helper.h"
+#include "read_object_data.h"
+#include "debug_helper.h"
+
+#define USAGE "use: array = readObjectData(nProcesses, precision, [nrows, ncols], datasetPath, { filePaths, ... })"
+
+/*!
+ * \brief MEX function readObjectData
+ *
+ * Input arguments:
+ * 0 : Number of read processes
+ * 1 : 'double' or 'single' precision
+ * 2 : Output image size [nrows, ncolumns] (optional, default value = size of first image)
+ * 3 : Input object dataset path within the HDF5 file
+ * 4 : Input HDF5 file paths
+ *
+ * Output arguments:
+ * 0 : Object data (3d, 4d[with modes], or 5d[with slices]) [nobjects, nslices*, nmodes*, nrows, ncolumns]
+ * 1 : (Optional) 1d array of file indices that could not be read
+ *
+ * \param nlhs Number of left hand side (result) arguments
+ * \param plhs Pointer array to left hand side arguments
+ * \param nrhs Number of right hand side (input) arguments
+ * \param prhs Pointer array to right hand side arguments
+ */
+void mexFunction(int nlhs, mxArray *plhs[],
+                 int nrhs, const mxArray *prhs[])
+{
+    constexpr int max_nprocs = 512;
+    constexpr unsigned int min_inputs = 4;
+    constexpr unsigned int max_inputs = 5;
+    constexpr unsigned int min_outputs = 1;
+    constexpr unsigned int max_outputs = 2;
+
+    DEBUG_INIT;
+    if ((nrhs < min_inputs) || (nrhs > max_inputs))
+        mexErrMsgIdAndTxt("psi:ptycho:arg:in:wrongNumber", "Wrong number of arguments, " USAGE);
+    if ((nlhs < min_outputs) || (nlhs > max_outputs))
+        mexErrMsgIdAndTxt("psi:ptycho:arg:out:wrongNumber", "Wrong number of output arguments, " USAGE);
+
+    unsigned int i = 0;
+    int64_t nprocs = 1;
+    { // 0
+        if (! mxIsScalar(prhs[i]))
+            mexErrMsgIdAndTxt("psi:ptycho:arg:wrongType:notScalar", "Argument %d must be scalar!", i+1);
+        std::unique_ptr<mx_numeric> nt(get_numeric_trait(prhs[i]));
+        if (! nt.get())
+            mexErrMsgIdAndTxt("psi:ptycho:arg:wrongType:notScalar", "Argument %d must be numeric!", i+1);
+        nprocs = nt->get_int64(prhs[i]);
+        if (nprocs < 1)
+            mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:notPositive", "Number of processes must be positive!");
+        if (nprocs > max_nprocs)
+            mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:tooMany", "Too many processes, max is %d!", max_nprocs);
+        i += 1;
+    }
+    precision::type prec = precision::type::Single;
+    { // 1
+        if (mxGetM(prhs[i]) != 1)
+            mexErrMsgIdAndTxt("psi:ptycho:arg:wrongDimensions:notOne", "Argument %d must be one-dimensional character array!", i+1);
+        std::unique_ptr<mx_trait<char>> ct(get_char_trait(prhs[i]));
+        if (! ct.get())
+            mexErrMsgIdAndTxt("psi:ptycho:arg:wrongType:notChar", "Argument %d must be of type char!", i+1);
+        std::string precString = ct->get_string(prhs[i]);
+        if (precString == "double") {
+            prec = precision::type::Double;
+        } else if (precString != "single") {
+            mexErrMsgIdAndTxt("psi:ptycho:arg:type", "Illegal type argument (%s): must be either 'single' or 'double'!", precString.c_str());
+        }
+        i += 1;
+    }
+    std::array<int64_t, 2> dims{0, 0};
+    do { // 2 optionial
+        std::unique_ptr<mx_numeric> nt(get_numeric_trait(prhs[i]));
+        if (! nt.get())
+            break;
+        if ((mxGetN(prhs[i]) != 2) || (mxGetM(prhs[i]) != 1))
+            mexErrMsgIdAndTxt("psi:ptycho:arg:wrongDimensions:oneByTwo", "Argument %d must be 2x1 numeric array!", i+1);
+        dims[0] = nt->get_int64(prhs[i], 0);
+        dims[1] = nt->get_int64(prhs[i], 1);
+        if ((dims[0] < 1) || (dims[1] < 1))
+            mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:illegalSize", "Size argument must be at least 1 in every dimension!");
+        i += 1;
+    } while(false);
+    std::string object_path;
+    { // 3
+        if (mxGetM(prhs[i]) != 1)
+            mexErrMsgIdAndTxt("psi:ptycho:arg:wrongDimensions:notOne", "Argument %d must be one-dimensional character array!", i+1);
+        std::unique_ptr<mx_trait<char>> ct(get_char_trait(prhs[i]));
+        if (! ct.get())
+            mexErrMsgIdAndTxt("psi:ptycho:arg:wrongType:notChar", "Argument %d must be of type char!", i+1);
+        object_path = ct->get_string(prhs[i]);
+        if (! object_path.size())
+            mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:empty", "Illegal object path argument!");
+        i += 1;
+    }
+    if (i == nrhs)
+        mexErrMsgIdAndTxt("psi:ptycho:arg:in:wrongNumber", "Missing arguments, " USAGE);
+    std::vector<std::string> file_paths;
+    { // 4
+        if (mxGetM(prhs[i]) != 1)
+            mexErrMsgIdAndTxt("psi:ptycho:arg:wrongDimensions:notOne", "Argument %d must be a 1xN one-dimensional character or cell array!", i+1);
+        std::unique_ptr<mx_trait<char>> ct(get_char_trait(prhs[i]));
+        if (ct.get()) { // char array
+            file_paths.push_back(ct->get_string(prhs[i]));
+        } else {        // cell array
+            if (! mxIsCell(prhs[i]))
+                mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:type", "Argument %d must be either character or cell array!", i+1);
+            for (std::size_t j=0; j<mxGetN(prhs[i]); j+=1) {
+                mxArray *pa = mxGetCell(prhs[i], j);
+                if (mxGetM(pa) != 1)
+                    mexErrMsgIdAndTxt("psi:ptycho:arg:wrongDimensions:notOne", "Cell fields in argument %d must be one-dimensional character arrays!", i+1);
+                ct.reset(get_char_trait(pa));
+                if (! ct.get())
+                    mexErrMsgIdAndTxt("psi:ptycho:arg:wrongType:notChar", "Cell field in argument %d must be of type char!", i+1);
+                file_paths.push_back(ct->get_string(pa));
+                if (! file_paths.back().size())
+                    mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:empty", "Illegal file path argument!");
+            }
+            if (! file_paths.size())
+                mexErrMsgIdAndTxt("psi:ptycho:arg:illegal:empty", "Empty file path argument!");
+        }
+        i += 1;
+    }
+    if (i != nrhs)
+        mexErrMsgIdAndTxt("psi:ptycho:arg:in:wrongNumber", "Superfluous arguments, " USAGE);
+
+    bool verbose = env::is_enabled("PTYCHO_READ_VERBOSE");
+    if (verbose) {
+        mexPrintf("precision: %s\n", precision::to_str(prec));
+        const char *debuglog = std::getenv("PTYCHO_READ_DEBUG");
+        mexPrintf("debuglog: %s - ", (debuglog ? debuglog : "not set"));
+        DEBUG {
+            mexPrintf("ok\n");
+            OUT << "----------- read object data ----------------\n";
+        } else {
+            mexPrintf("not functional\n");
+        }
+        mexPrintf("object_location: %s\n", object_path.c_str());
+        if (file_paths.size() == 1) {
+            mexPrintf("data_location: %s\n", file_paths[0].c_str());
+        } else {
+            mexPrintf("data_location first: %s\n", file_paths[0].c_str());
+            mexPrintf("data_location last : %s\n", file_paths.back().c_str());
+        }
+    }
+
+    data_prep::read_stat r_stat;
+    plhs[0] = data_prep::read_object_data((int)nprocs, prec, dims, object_path, file_paths, (nlhs == 2) ? &plhs[1] : nullptr, verbose ? &r_stat : nullptr);
+    if (verbose)
+        mexPrintf("Read %fMB in %fs, bandwidth is %fMB/s\n", (r_stat.nbytes / 1000000.), r_stat.seconds, (r_stat.nbytes / (r_stat.seconds * 1000000.)));
+    return;
+}

+io/private/ptycho_reader/read_data_threaded.cc

@@ -0,0 +1,625 @@
+/*!
+ * \file
+ * Support for data formats with many separate compnents that can be read using threads
+ *
+ * TODO: make the code safe to type inconsistencies
+ */
+
+#include <map>
+#include <cerrno>
+#include <cstring>
+#include <algorithm>
+#include <fstream>
+#include <cassert>
+#include <cstdint>
+#include <regex>
+#include <mutex>
+#include <future>
+#include <atomic>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <stdio.h>
+#include <dirent.h>
+#include "tiffio.h"
+#include "precision.h"
+#include "mex.h"
+#include "mex_helper.h"
+#include "debug_helper.h"
+#include "read_data_threaded.h"
+
+namespace {
+    thread_local unsigned int thread_id = 0;
+    std::mutex mutex_cout;
+    std::atomic<bool> thread_error;
+
+#define LOCK(m) std::lock_guard<std::mutex> _lock(m);
+
+    /*!
+     * \brief Adapt file and destination dimensions
+     *
+     * \param fdims (IN) file image dimensions (2D)
+     * \param mdims (IN) MATLAB destination array dimensions (> 2D) (last dimensions are image dimensions)
+     * \param im_ctr (IN) image center
+     * \param fstride (OUT) file image strides
+     * \param mstride (OUT) destination image strides
+     * \param count (OUT) elements to copy in each image dimension
+     */
+    void adapt_dimensions(const std::vector<long> &fdims, const std::vector<mwSize> mdims,
+                          const std::vector<long> &im_ctr,
+                          std::vector<long> &fstride,
+                          std::vector<long> &mstride,
+                          std::vector<long> &count)
+    {
+        const mwSize *md = &(*(mdims.end() - 2));
+        const std::vector<mwSize> mh{md[0]/2, md[1]/2};
+        for (unsigned int i=0; i<2; i++) {
+            if (im_ctr[i] >= mh[i]) {
+                mstride[i] = 0;
+                fstride[i] = im_ctr[i] - mh[i];
+                if (fstride[i] + md[i] > fdims[i])
+                    count[i] = fdims[i] - fstride[i];
+                else
+                    count[i] = md[i];
+            } else {
+                fstride[i] = 0;
+                mstride[i] = mh[i] - im_ctr[i];
+                if (mstride[i] + fdims[i] <= md[i])
+                    count[i] = fdims[i];
+                else
+                    count[i] = md[i] - mstride[i];
+            }
+        }
+    }
+
+    /*!
+     * \brief Append element to file path
+     *
+     * Path separator is '/'
+     *
+     * \param path path appended to
+     * \param elem element to append
+     * \return new path with extra element
+     */
+    std::string path_append (const std::string &path, const std::string &elem)
+    {
+        std::string result(path);
+        if (result.size())
+            result.push_back('/');
+        return result + elem;
+    }
+
+    // ----------------------------------------------------------------
+
+    /* PILATUS CBF DATA
+        _array_data.data
+        ;
+        --CIF-BINARY-FORMAT-SECTION--
+        Content-Type: application/octet-stream;
+            conversions="x-CBF_BYTE_OFFSET"
+        Content-Transfer-Encoding: BINARY
+        X-Binary-Size: 2499331
+        X-Binary-ID: 1
+        X-Binary-Element-Type: "signed 32-bit integer"
+        X-Binary-Element-Byte-Order: LITTLE_ENDIAN
+        Content-MD5: XoY7+gfct1+OKiJlzKOiRw==
+        X-Binary-Number-of-Elements: 2476525
+        X-Binary-Size-Fastest-Dimension: 1475
+        X-Binary-Size-Second-Dimension: 1679
+        X-Binary-Size-Padding: 4095
+     */
+    std::regex pilatus_ncols_regex(R"(X-Binary-Size-Fastest-Dimension: (\d+))");    //!< Regex for pilatus ncols
+    std::regex pilatus_nrows_regex(R"(X-Binary-Size-Second-Dimension: (\d+))");     //!< Regex for pilatus nrows
+
+    /*!
+     * \brief Read CBF image dimensions
+     *
+     * \param data (IN) CBF image file data
+     * \param dims (OUT) to be filled with [nrows, ncols] from the data
+     */
+    void cbf_dims (const std::vector<char> &data, std::vector<long> &dims)
+    {
+        assert(dims.size() == 2);
+        {
+            std::cmatch match;
+            if (! std::regex_search(&data[0], &data[0] + data.size(), match, pilatus_ncols_regex))
+                throw std::runtime_error("unable to find number of columns");
+            dims[1] = std::stol(match[1].str());
+            if (dims[1] <= 0)
+                throw std::runtime_error("dimension along row is not positive");
+        }
+        {
+            std::cmatch match;
+            if (! std::regex_search(&data[0], &data[0] + data.size(), match, pilatus_nrows_regex))
+                throw std::runtime_error("unable to find number of rows");
+            dims[0] = std::stol(match[1].str());
+            if (dims[0] <= 0)
+                throw std::runtime_error("dimension along column is not positive");
+        }
+    }
+
+    /*!
+     * \brief Read pilatus metadata
+     *
+     * \param paths (IN) paths to pilatus CBF image files
+     * \param dims (OUT) to be filled with [n_images, n_series(only if several series are present), nrows, ncols], the last two from the first data file
+     * \param im_sz (INOUT) if empty, fill it with [nrows, ncols] from the first data file
+     */
+    void pilatus_read_meta(const std::vector<std::string> &paths,
+                           std::vector<long> &dims,
+                           std::vector<long> &im_sz)
+    {
+        // Take the first file to determine image dimensions size and center if not given
+        if (paths.empty())
+            throw std::invalid_argument("no CBF data files");
+        DEBUG {
+            OUT << paths.size() << " data files\n"
+                << "first file " << paths[0] << std::endl;
+        }
+        std::vector<long> fdim{0, 0};
+        {
+            std::vector<char> buf(4*1024);  // 4K max header size in CBF file
+            std::ifstream ifs(paths[0]);
+            ifs.read(&buf[0], buf.size());
+            if (ifs.fail() && !ifs.eof())
+                throw std::runtime_error("unable to read first data file");
+            buf.resize(ifs.gcount());
+            cbf_dims(buf, fdim);
+        }
+        dims.resize(3);
+        dims[0] = paths.size();
+        dims[1] = fdim[0];
+        dims[2] = fdim[1];
+        if (im_sz.empty()) {
+            im_sz.resize(2);
+            im_sz[0] = fdim[0];
+            im_sz[1] = fdim[1];
+            DEBUG {
+                OUT << "setting image size to " << im_sz[0] << 'x' << im_sz[1] << std::endl;
+            }
+        }
+    }
+
+    /*!
+     * \brief Read image data from pilatus CBF file
+     *
+     * \param tid thread id starting from 0
+     * \param destination MATLAB array image data destination
+     * \param path pilatus CBF data file path
+     * \param mdims MATLAB array dimensions
+     * \param im_ctr image center relative to data
+     * \tparam f_type float or double array element type
+     */
+    template<typename f_type>
+    void pilatus_read_data(unsigned int tid,
+                           f_type * destination,
+                           const std::string &path,
+                           const std::vector<mwSize> &mdims,
+                           const std::vector<long> &im_ctr)
+    {
+        // Use Heiners method to read the data
+        // Adapt dimensions for every image file
+        std::vector<char> fbuf;
+        {
+            FILE *fin = fopen(path.c_str(), "r");
+            if (! fin)
+                throw std::runtime_error(std::string("unable to open file ") + path + ": " + std::strerror(errno));
+            try {
+                off_t fsz;
+                {
+                    struct stat sbuf;
+                    if (fstat(fileno(fin), &sbuf) == -1) {
+                        throw std::runtime_error(std::string("unable to stat file ") + path + ": " + std::strerror(errno));
+                    }
+                    fsz = sbuf.st_size;
+                }
+                fbuf.resize(fsz);
+                fread(&fbuf[0], 1, fsz, fin);
+                if (ferror(fin))
+                    throw std::runtime_error(std::string("unable to read file ") + path + ": " + std::strerror(errno));
+                fclose(fin);
+            } catch (...) {
+                fclose(fin);
+                throw;
+            }
+        }
+        std::vector<long> fdims{0, 0};
+        cbf_dims(fbuf, fdims);
+        unsigned long finger;    // compressed data index
+        {
+            std::vector<char> sig{ '\x0c', '\x1a', '\x04', '\xd5' };
+            auto p = std::search(fbuf.begin(), fbuf.end(), sig.begin(), sig.end());
+            if (p == fbuf.end())
+                throw std::runtime_error(std::string("data signature not found within file " + path));
+            finger = p - fbuf.begin() + 4;
+        }
+        unsigned long nelems = fdims[0] * fdims[1];
+        std::vector<f_type> data(nelems);
+        int current = 0;
+        for (unsigned int i=0; i<nelems; i++) {
+            if (*((uint8_t *)&fbuf[finger]) != 0x80) {    // | xx |
+                current += *((int8_t *)&fbuf[finger]);
+                finger += 1;
+            } else if (*((uint16_t *)&fbuf[finger+1]) != 0x8000) {  // | 80 | xx | xx |
+                current += *((int16_t *)&fbuf[finger+1]);
+                finger += 3;
+            } else {    // | 80 | 80 | 00 | xx | xx | xx | xx |
+                current += *((int32_t *)&fbuf[finger+3]);
+                finger += 7;
+            }
+            if (finger + 7 > fbuf.size())
+                throw std::runtime_error(std::string("data inconsistency in file ") + path);
+            //if (current < -1)   // allow value -1, which is used to mark detector gaps
+            //    throw std::runtime_error(std::string("data error in file ") + path);
+            data[i] = current;
+        }
+        fbuf.clear();
+        std::vector<long> fstride{0, 0};
+        std::vector<long> mstride{0, 0};
+        std::vector<long> count{0, 0};
+        adapt_dimensions(fdims, mdims, im_ctr, fstride, mstride, count);
+        DEBUG {
+            LOCK(mutex_cout);
+            OUT << thread_id << ":     fstride=[" << fstride[0] << ',' << fstride[1] << "], mstride=[" << mstride[0] << ',' << mstride[1] << "], count=[" << count[0] << ',' << count[1] <<']' << std::endl;
+        }
+        {
+            auto sz = mdims.size() - 2;
+            auto msize = mdims[sz] * mdims[sz+1];
+            std::memset(destination, 0, msize * sizeof(f_type));
+        }
+        for (unsigned long row=0; row<count[0]; row++) {
+            auto col = data.begin() + fdims[1] * (row + fstride[0]) + fstride[1];
+            std::copy(col, col + count[1], destination + mdims.back() * (row + mstride[0]) + mstride[1]);
+        }
+    }
+
+    //-----------------------------------------------------
+
+    /*!
+     * \brief Read moench tiff image dimensions
+     *
+     * \param tiff_handle (IN) TIFF file handle
+     * \param fdims (OUT) to be filled with [nrows, ncols]
+     */
+    void tiff_dims (TIFF *tiff_handle, std::vector<long> &fdims)
+    {
+        assert(fdims.size() == 2);
+        uint32_t image_length, image_width;
+        if (! TIFFGetField(tiff_handle, TIFFTAG_IMAGELENGTH, &image_length))
+            throw std::runtime_error("unable to get image length");
+        if (! image_length)
+            throw std::runtime_error("dimension along column is not positive");
+        if (! TIFFGetField(tiff_handle, TIFFTAG_IMAGEWIDTH, &image_width))
+            throw std::runtime_error("unable to get image width");
+        if (! image_width)
+            throw std::runtime_error("dimension along row is not positive");
+        fdims[0] = image_length;
+        fdims[1] = image_width;
+    }
+
+    /*!
+     * \brief Read moench metadata
+     *
+     * \param paths (IN) list of moench TIFF image data file paths
+     * \param dims (OUT) data dimensions [n_images, nrows, ncols], the last two from the first data file
+     * \param im_sz (INOUT) if empty, set to [nrows, ncols] from first data file
+     */
+    void moench_read_meta(const std::vector<std::string> &paths,
+                          std::vector<long> &dims,
+                          std::vector<long> &im_sz)
+    {
+        // Take the first file to determine image dimensions size and center if not given
+        if (paths.empty())
+            throw std::invalid_argument("no TIFF data files");
+        DEBUG {
+            OUT << paths.size() << " data files\n"
+                << "first file " << paths[0] << std::endl;
+        }
+        std::vector<long> fdim(2);
+        {
+            TIFF *tiff_handle = TIFFOpen(paths[0].c_str(), "r");
+            if (! tiff_handle)
+                throw std::runtime_error("unable to open first image data file");
+            try {
+                tiff_dims(tiff_handle, fdim);
+            } catch (...) {
+                TIFFClose(tiff_handle);
+                throw;
+            }
+            TIFFClose(tiff_handle);
+        }
+        dims.resize(3);
+        dims[0] = paths.size();
+        dims[1] = fdim[0];
+        dims[2] = fdim[1];
+        if (im_sz.empty()) {
+            im_sz.resize(2);
+            im_sz[0] = fdim[0];
+            im_sz[1] = fdim[1];
+            DEBUG {
+                OUT << "setting image size to " << im_sz[0] << 'x' << im_sz[1] << std::endl;
+            }
+        }
+    }
+
+    /*!
+     * \brief Read TIFF image data
+     *
+     * \param tf TIFF file descriptor
+     * \param buf char buffer with enough space for a data strip
+     * \param image image data buffer
+     * \param num_strip number of data strips
+     * \tparam sample_type image data sample type
+     * \tparam result_type image buffer data type
+     */    
+    template<typename sample_type, typename result_type>
+    void tiff_read(TIFF *tf, std::vector<char> &buf, std::vector<result_type> &image, tstrip_t num_strips)
+    {
+        std::uint32_t idx = 0;
+        for (tstrip_t strip=0; strip<num_strips; strip++) {
+            tsize_t nbytes = TIFFReadEncodedStrip(tf, strip, buf.data(), buf.size());
+            if (nbytes < 0)
+                throw std::runtime_error("unable to read strip from tiff file");
+            for (tsize_t i=0; i<nbytes; i+=sizeof(sample_type))
+                image[idx++] = *((sample_type *)&buf[i]);
+        }
+        if (idx != image.size())
+            throw std::runtime_error("tiff image data size mismatch");
+    }
+
+
+    /*!
+     * \brief Read moench TIFF image data into MATLAB array
+     *
+     * \param tid thread id, starting from 0
+     * \param destination MATLAB array destination data buffer
+     * \param path tiff file path
+     * \param mdims MTLAB array dimensions
+     * \param im_ctr image center relative to data
+     */
+    template<typename f_type>
+    void moench_read_data (unsigned int tid,
+                           f_type * destination,
+                           const std::string &path,
+                           const std::vector<mwSize> &mdims,
+                           const std::vector<long> &im_ctr)
+    {
+        /*
+        std::vector<long> fdims{0, 0};
+        std::vector<float> fbuf(0);
+        {
+            TIFF *tiff_handle = TIFFOpen(path.c_str(), "r");
+            if (! tiff_handle)
+                throw std::runtime_error(std::string("unable to open file ") + path);
+            try {
+                tiff_dims(tiff_handle, fdims);
+                long nelems = fdims[0] * fdims[1];
+                fbuf.resize(nelems);
+                tmsize_t res, sz = fdims[1] * sizeof(float);
+                for (uint32_t strip=0; strip<fdims[0]; strip++) {
+                    float *pos = &fbuf[fdims[1] * strip];
+                    res = TIFFReadRawStrip(tiff_handle, strip, pos, sz);
+                    if (res != sz)
+                        throw std::runtime_error("unable to read tiff data");
+                }
+            } catch (...) {
+                TIFFClose(tiff_handle);
+                throw;
+            }
+            TIFFClose(tiff_handle);
+        }
+        */
+        std::vector<long> fdims{0, 0};
+        std::vector<f_type> fbuf(0);
+        {
+            TIFF *tiff_handle = TIFFOpen(path.c_str(), "r");
+            if (! tiff_handle)
+                throw std::runtime_error(std::string("unable to open file ") + path);
+            try {
+                tiff_dims(tiff_handle, fdims);
+                long nelems = fdims[0] * fdims[1];
+                fbuf.resize(nelems);
+                std::uint16_t bps;
+                if (TIFFGetField(tiff_handle, TIFFTAG_BITSPERSAMPLE, &bps) != 1)
+                    throw std::runtime_error(std::string("unable to read number of bits per sample for file ") + path);
+                std::uint16_t format;
+                if (TIFFGetFieldDefaulted(tiff_handle, TIFFTAG_SAMPLEFORMAT, &format) != 1)
+                    throw std::runtime_error(std::string("unable to read sample format for file") + path);
+                tstrip_t num_strips = TIFFNumberOfStrips(tiff_handle);
+                tsize_t strip_sz = TIFFStripSize(tiff_handle);
+                std::vector<char> cbuf(strip_sz);
+                switch (format) {
+                case 1:
+                    switch (bps) {
+                    case 8:
+                        tiff_read<std::uint8_t, f_type>(tiff_handle, cbuf, fbuf, num_strips); break;
+                    case 16:
+                        tiff_read<std::uint16_t, f_type>(tiff_handle, cbuf, fbuf, num_strips); break;
+                    case 32:
+                        tiff_read<std::uint32_t, f_type>(tiff_handle, cbuf, fbuf, num_strips); break;
+                    default:
+                        throw std::runtime_error(std::string("unsupported number of bits per unsigned integer sample in file ") + path);
+                    }
+                    break;
+                case 2:
+                    switch (bps) {
+                    case 8:
+                        tiff_read<std::int8_t, f_type>(tiff_handle, cbuf, fbuf, num_strips); break;
+                    case 16:
+                        tiff_read<std::int16_t, f_type>(tiff_handle, cbuf, fbuf, num_strips); break;
+                    case 32:
+                        tiff_read<std::int32_t, f_type>(tiff_handle, cbuf, fbuf, num_strips); break;
+                    default:
+                        throw std::runtime_error(std::string("unsupported number of bits per integer sample in file ") + path);
+                    }
+                    break;
+                case 3:
+                    switch (bps) {
+                    case 32:
+                        tiff_read<float, f_type>(tiff_handle, cbuf, fbuf, num_strips); break;
+                    case 64:
+                        tiff_read<double, f_type>(tiff_handle, cbuf, fbuf, num_strips); break;
+                    default:
+                        throw std::runtime_error(std::string("unsupported number of bits per ieee sample in file ") + path);
+                    }
+                    break;
+                default:
+                    throw std::runtime_error(std::string("unsupported sample format in file ") + path);
+                }
+            } catch (...) {
+                TIFFClose(tiff_handle);
+                throw;
+            }
+            TIFFClose(tiff_handle);
+        }
+        std::vector<long> fstride{0, 0};
+        std::vector<long> mstride{0, 0};
+        std::vector<long> count{0, 0};
+        adapt_dimensions(fdims, mdims, im_ctr, fstride, mstride, count);
+        DEBUG {
+            LOCK(mutex_cout);
+            OUT << thread_id << ":     fstride=[" << fstride[0] << ',' << fstride[1] << "], mstride=[" << mstride[0] << ',' << mstride[1] << "], count=[" << count[0] << ',' << count[1] <<']' << std::endl;
+        }
+        {
+            unsigned long msize = *(mdims.end() - 2) * mdims.back();
+            std::memset(destination, 0, msize * sizeof(f_type));
+        }
+        for (unsigned long row=0; row<count[0]; row++) {
+            auto col = fbuf.begin() + fdims[1] * (row + fstride[0]) + fstride[1];
+            std::copy(col, col + count[1], destination + mdims.back() * (row + mstride[0]) + mstride[1]);
+        }
+    }
+
+    //-----------------------------------------------------
+
+    /*!
+     * \brief Read data files in parallel
+     *
+     * \param format data format name
+     * \param nthreads desired number of threads
+     * \param paths paths to detector data files
+     * \param image_size desired image size [nrows, ncols]
+     * \param roi_center desired image center [row, col] relative to data
+     * \return MATLAB array [n_images, n_series(only if several are present), nrows, ncols]
+     */
+    template<typename f_type>
+    mxArray* read_data_parallel(const std::string &format,
+                                long nthreads,
+                                const std::vector<std::string> &paths,
+                                std::vector<long> &image_size, std::vector<long> &roi_center)
+    {
+        struct read_func final {
+            void (*read_meta) (const std::vector<std::string> &paths,
+                               std::vector<long> &dims,
+                               std::vector<long> &im_sz);
+            void (*read_data) (unsigned int tid,
+                               f_type * destination,
+                               const std::string &path,
+                               const std::vector<mwSize> &mdims,
+                               const std::vector<long> &im_ctr);
+        } detector_functions[2] = {
+            { pilatus_read_meta, pilatus_read_data<f_type> },   // pilatus
+            { moench_read_meta, moench_read_data<f_type> }      // moench
+        };
+
+        const std::map<std::string, read_func&> dfunc {
+            { "pilatus",  detector_functions[0] },
+            { "moench", detector_functions[1] }
+        };
+
+        const auto elem = dfunc.find(format);
+        if (elem == dfunc.end())
+            throw std::invalid_argument("unsupported data format");
+        const auto &func = elem->second;
+        std::vector<long> fdims;
+        func.read_meta(paths, fdims, image_size);
+        if (fdims.size() < 3)
+            throw std::invalid_argument("inconsistent data (dimensionality too low)");
+        if (paths.size() != fdims[0])
+            throw std::invalid_argument("inconsistent data (number of paths / array dimension mismatch)");
+        if (image_size.size() != 2)
+            throw std::invalid_argument("bad image size, must be two dimensional");
+        if (roi_center.empty()) {
+            roi_center.resize(2);
+            roi_center[0] = fdims[fdims.size() - 2] / 2;
+            roi_center[1] = fdims.back() / 2;
+        }
+        std::vector<mwSize> mdims(fdims.size());
+        std::transform(fdims.begin(), fdims.end(), mdims.begin(), [](long e)->mwSize { return (mwSize)e; });
+        auto sz = mdims.size() - 2;
+        mdims[sz] = image_size[0];
+        mdims[sz + 1] = image_size[1];
+        DEBUG {
+            OUT << "fdims  " << fdims[0] << 'x' << fdims[1] << 'x' << fdims[2] << '\n'
+                << "mdims  " << mdims[0] << 'x' << mdims[1] << 'x' << mdims[2] << '\n'
+                << "center " << roi_center[0] << 'x' << roi_center[1] << std::endl;
+        }
+        mx_ptr<mxArray> matrix;
+        {
+            std::vector<mwSize> rdims(mdims);
+            std::reverse(rdims.begin(), rdims.end());
+            matrix.reset(mxCreateNumericArray(rdims.size(), &rdims[0], mx_trait<f_type>::class_id, mxREAL));
+        }
+        if (! matrix.get())
+            throw std::runtime_error("matrix creation failed");
+        f_type *destination = mx_trait<f_type>::get(matrix.get());
+
+        if (nthreads > paths.size())
+            nthreads = paths.size();
+        std::vector<std::future<void>> threads;
+        for (unsigned int tid=0; tid<nthreads; tid++) {
+            threads.push_back(std::async(std::launch::async, [nthreads, tid, &destination, &func, &paths, &fdims, &mdims, &roi_center]() {
+                thread_id = tid;
+                const unsigned int npaths = (paths.size() + nthreads - 1) / nthreads;
+                const unsigned int first_full = (nthreads * npaths) - paths.size();
+                const unsigned int first = (tid * npaths) - (tid < first_full ? tid : first_full);
+                const unsigned int last = first + npaths - (tid < first_full ? 1 : 0);
+                const auto pdim = &mdims[mdims.size() - 2];
+                const auto sz = pdim[0] * pdim[1];
+                DEBUG {
+                    LOCK(mutex_cout);
+                    OUT << thread_id << ": range " << first << '-' << last << ", sz=" << sz << std::endl;
+                }
+                for (unsigned int i=first; i<last; i++) {
+                    if (thread_error.load())
+                        throw std::runtime_error("received stop signal");
+                    DEBUG {
+                        LOCK(mutex_cout);
+                        OUT << thread_id << ": " << paths[i] << std::endl;
+                    }
+                    func.read_data(tid, &destination[i * sz], paths[i], mdims, roi_center);
+                }
+            }));
+        }
+
+        bool success{true};
+        for (unsigned int tid=0; tid<nthreads; tid++) {
+            try {
+                threads[tid].wait();
+                threads[tid].get();
+            } catch (const std::exception &ex) {
+                mexPrintf("Thread %u error: %s\n", tid, ex.what());
+                thread_error.store(false);
+            }
+        }
+
+        if (thread_error)
+            throw std::runtime_error("Failed to read data!");
+
+        return matrix.release();
+    }
+
+}
+
+namespace data_prep {
+    mxArray* read_data_threaded(const std::string &format,
+                                long nthreads,
+                                const std::vector<std::string> &data_path,
+                                std::vector<long> &image_size, std::vector<long> &roi_center,
+                                precision::type prec)
+    {
+        if (prec == precision::type::Double)
+            return read_data_parallel<double>(format, nthreads, data_path, image_size, roi_center);
+        else
+            return read_data_parallel<float>(format, nthreads, data_path, image_size, roi_center);
+    }
+}

+io/private/ptycho_reader/read_data_threaded.h

@@ -0,0 +1,31 @@
+#ifndef READ_DATA_THREADED_H
+#define READ_DATA_THREADED_H
+
+/*!
+ * \file
+ * Header file for the read Moench and Pilatus data functionality
+ */
+
+/*!
+ * \brief Data preparator functions
+ */
+namespace data_prep {
+    /*!
+     * \brief read Moench or Pilatus data
+     *
+     * \param format data format string
+     * \param nthreads number of parallel read threads
+     * \param data_path paths to data files
+     * \param image_size size of returned image data
+     * \param roi_center region of interest center relative to measured data
+     * \param prec precision of returned data
+     * \return MATLAB array with resized and recentered images
+     */
+    mxArray* read_data_threaded(const std::string &format,
+                                long nthreads,
+                                const std::vector<std::string> &data_path,
+                                std::vector<long> &image_size, std::vector<long> &roi_center,
+                                precision::type prec);
+}
+
+#endif

+io/private/ptycho_reader/read_eiger_data.cc

@@ -0,0 +1,700 @@
+/*!
+ * \file
+ * Read Object Data
+ *
+ * This file contains the main mex code for reading object data into MATLAB memory.
+ */
+
+#include <string>
+#include <memory>
+#include <chrono>
+#include <cassert>
+#include <array>
+#include <vector>
+#include <cstring>
+#include <cerrno>
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <exception>
+#include <stdexcept>
+#include <cstdio>
+#include <functional>
+#include <algorithm>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/wait.h>
+#include "hdf5_helper.h"
+#include "mex.h"
+#include "debug_helper.h"
+#include "precision.h"
+#include "multi_processing.h"
+
+namespace {
+
+    #include "mex_helper.h"
+
+    /*!
+     * \defgroup dataLayoutKind Kind of data layout
+     * @{
+     */
+    constexpr int kind_11 = 0;                   //!< 1 file - 1 dataset   (new eiger)
+    constexpr int kind_N1 = 1;                   //!< N files - 1 dataset  (old eiger)
+    constexpr int kind_1N = 2;                   //!< 1 file - N datasets  (ESRF)
+    constexpr int kind_NN = 3;                   //!< N (file, dataset) pairs
+    /*! @} */
+
+    /*!
+     * \brief Vector to string
+     * Outputs a string containing v0[,v1[,v2...]]
+     * \param v vector
+     * \param separator separator between vector elements, defaults to comma
+     * \tparam T vector element type
+     */
+    template <typename T>
+    std::string vec_to_str(const std::vector<T> &v, char separator=',')
+    {
+        std::ostringstream oss;
+        oss << v[0];
+        for (std::size_t i=1; i<v.size(); i++)
+            oss << separator << v[i];
+        return oss.str();
+    }
+
+    /*!
+     * \brief Adapt data space layout
+     *
+     * Adapt the memory and file dataset layouts by selecting apropriate hyperslabs
+     *
+     * \param mspace memory data space layout (must fit to parameters first, last)
+     * \param fspace file dataset layout
+     * \param mdim memory space dimensions (must fit to parameters first last if kind=kind_11)
+     * \param fdim file dataset dimensions
+     * \param first first chunk of file space to include
+     * \param last one past last chunk of file space to include
+     * \param roi_center image center
+     * \tparam kind kind of data layout
+     */
+    template<int kind>
+    void adapt_spaces(hdf5::dataspace &mspace, hdf5::dataspace &fspace,
+                      const std::vector<hsize_t> &mdim, const std::vector<hsize_t> &fdim,
+                      int first , int last,
+                      const std::vector<long> &roi_center)
+    {
+        if (mdim.size() <= roi_center.size())
+            throw std::invalid_argument("result matrix must have more dimensions than the image");
+
+        std::vector<hsize_t> fstart(fdim.size());
+        std::vector<hsize_t> mstart(fdim.size());
+        std::vector<hsize_t> fcount(fdim);
+        std::vector<hsize_t> mcount(mdim);
+        bool mselect = false;
+        bool fselect = false;
+
+        if (kind == kind_11) {
+            // adapt first dimension (chunk dimension)
+            if ((first != 0) || (last != fdim[0])) {
+                fstart[0] = first;
+                fcount[0] = last - first;
+                fselect = true;
+                // DEBUG {
+                //     OUT << "  adapted chunk dimension: " << fstart[0] << '+' << fcount[0] << std::endl;
+                // }
+            }
+        }
+
+        // adapt last dimensions (normally detector image dimensions)
+        {
+            auto fi = fdim.size();
+            auto mi = mdim.size();
+            auto i = roi_center.size();
+            do {
+                i--; fi--; mi--;
+                hsize_t fh = (fdim[fi] + 1) / 2;
+                hsize_t mh = (mdim[mi] + 1) / 2;
+                hsize_t rc = roi_center[i];
+                if (rc<0 || rc>=fdim[fi])
+                    throw std::invalid_argument("roi_center out of image bounds");
+                if (mh <= rc) {
+                    fstart[fi] = rc - mh;
+                    if (fstart[fi] + mdim[mi] >= fdim[fi]) {
+                        fcount[fi] = fdim[fi] - fstart[fi];
+                        mcount[mi] = fcount[fi];
+                        mselect = mselect || (mcount[mi] != mdim[mi]);
+                    } else {
+                        fcount[fi] = mdim[mi];
+                    }
+                    fselect = fselect || ((fcount[fi] != fdim[fi]) || fstart[fi]);
+                } else {
+                    if (rc + mh <= fdim[fi]) {
+                        mcount[mi] = fcount[fi] = rc + mh;
+                        mstart[mi] = mdim[mi] - mcount[mi];
+                        fselect = fselect || (fcount[fi] != fdim[fi]);
+                    } else {
+                        mcount[mi] = fcount[fi];
+                        mstart[mi] = rc - mh;
+                    }
+                    mselect = true;
+                }
+                // DEBUG {
+                //     OUT << "  adapting image dimension " << i << ": fdim=" << fdim[fi] << '/' << fh << ", mdim=" << mdim[mi] << '/' << mh << ", rc=" << rc << "\n  fsel=" << fselect << ", msel=" << mselect << std::endl;
+                // }
+            } while(i);
+        }
+        if (mselect) {
+            DEBUG {
+                OUT << "memory subspace start=" << vec_to_str(mstart) << " count=" << vec_to_str(mcount) << "\nfile space is " << vec_to_str(fdim) << std::endl;
+            }
+            if (H5Sselect_hyperslab(mspace, H5S_SELECT_SET, &mstart[0], nullptr, &mcount[0], nullptr) < 0)
+                throw hdf5::exception("unable to set hdf5 memory space");
+        }
+        if (fselect) {
+            DEBUG {
+                OUT << "file subspace start=" << vec_to_str(fstart) << " count=" << vec_to_str(fcount) << "\nmem space is " << vec_to_str(mdim) << std::endl;
+            }
+            if (H5Sselect_hyperslab(fspace, H5S_SELECT_SET, &fstart[0], nullptr, &fcount[0], nullptr) < 0)
+                throw hdf5::exception("unable to set hdf5 dataset space");
+        }
+    }
+
+    /*!
+     * \brief Read object from file into buffer
+     * \param file HDF5 file
+     * \param dataset_path path to dataset
+     * \param first first buffer chunk to read
+     * \param last last buffer chunk to read
+     * \param mdims buffer dimensions
+     * \param ddims dataset dimensions
+     * \param buf pointer to parent acessible buffer
+     * \param roi_center image center
+     * \tparam f_type buffer element type
+     * \tparam kind kind of data layout
+     */
+    template <typename f_type, int kind>
+    void read_data_chunk(hdf5::file &file, const std::string &dataset_path,
+                         int first, int last,
+                         const std::vector<mwSize> &mdims,
+                         const std::vector<hsize_t> &ddims,
+                         f_type *buf,
+                         const std::vector<long> &roi_center)
+    {
+        // Open dataset and data space
+        hdf5::dataset dataset(H5Dopen(file, dataset_path.c_str(), H5P_DEFAULT));
+        if (! dataset.valid())
+            throw hdf5::exception("unable to open dataset");
+        hdf5::dataspace dataspace(H5Dget_space(dataset));
+        if (! dataspace.valid())
+            throw hdf5::exception("unable to open data space for dataset");
+
+        // Read and check data space
+        std::vector<hsize_t> sdims(ddims.size()); // source dimensions
+        {
+            int ndims = H5Sget_simple_extent_ndims(dataspace);
+            if (ndims != ddims.size())
+                throw hdf5::exception("data space dimension mismatch");
+            if (H5Sget_simple_extent_dims(dataspace, &sdims[0], nullptr) != ndims)
+                throw hdf5::exception("unable to get dimension sizes");
+            for (unsigned int i=0; i<ddims.size()-2; i++)
+                if (sdims[i] != ddims[i])
+                    throw std::runtime_error("data space dimension inconsistency");
+        }
+
+        std::vector<hsize_t> tdims(sdims);  // target dimensions
+        if (kind == kind_11) {
+            auto i = sdims.size() - mdims.size();
+            decltype(i) j = 0;
+            for (; i<sdims.size(); i++, j++)
+                tdims[i] = mdims[j];
+            tdims[0] = last - first;
+        } else {
+            auto i = sdims.size();
+            auto j = mdims.size();
+            while ((i > 0) && (j > 1)) { // don't copy first matrix dimension (number of files/datasets)
+                i--; j--;
+                tdims[i] = mdims[j];
+            }
+            while (i) { // check that remaining data file dimensions are 1
+                i--;
+                if (tdims[i] != 1)
+                    throw std::runtime_error("data space dimension error");
+            }
+        }
+        DEBUG {
+            OUT << "init mem space: " << vec_to_str(tdims, 'x') << " at address " << buf << std::endl;
+        }
+        hdf5::dataspace memspace(H5Screate_simple(tdims.size(), &tdims[0], nullptr));
+        if (! memspace.valid())
+            throw hdf5::exception("unable to create memory data space");
+        adapt_spaces<kind>(memspace, dataspace, tdims, sdims, first, last, roi_center);
+        // Read dataset
+        if (H5Dread(dataset, hdf5::type_trait<f_type>::type, memspace, dataspace, H5P_DEFAULT, buf) < 0) {
+            DEBUG {
+                std::FILE *out = std::fopen(std::getenv("PTYCHO_READ_DEBUG"), "a+");
+                if (! out) {
+                    OUT << "unable to open file stream: " << std::strerror(errno) << std::endl;
+                } else {
+                    H5Eprint(H5E_DEFAULT, out);
+                    std::fclose(out);
+                }
+            }
+            throw hdf5::exception("unable to read dataset");
+        }
+
+        DEBUG {
+            OUT << "read finished - [0]=" << buf[0] << std::endl;
+        }
+    }
+
+    /*!
+     * \brief Open HDF5 file
+     * \param file_path path to HDF5 file
+     * \return HDF5 file object id
+     */
+    hid_t open_file(const std::string &file_path)
+    {
+        hdf5::file file(H5Fopen(file_path.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT));
+        if (! file.valid())
+            throw hdf5::exception(std::string("unable to open file ") + file_path);
+        return file.grab();
+    }
+
+    /*!
+     * \brief Drop leading dimensions of size 1
+     * \param dims vector for which leading dimensions of size 1 beyond the second dimension will be dropped
+     * \tparam T type of dimension size
+     */
+    template <typename T>
+    void collapse_dims (std::vector<T> &dims) noexcept
+    {
+        int i=0;
+        for (; i<dims.size()-2; i++) {
+            if (dims[i] != 1)
+                break;
+        }
+        int j=0;
+        for (; i<dims.size(); i++, j++)
+            dims[j] = dims[i];
+        return dims.resize(j);
+    }
+
+    /*!
+     * \brief Get HDF5 data size
+     *
+     * \param file HDF5 file
+     * \param dataset_path dataset path
+     * \param dims filled with dataset dimensions
+     */
+    void get_data_size(hdf5::file &file, const std::string &dataset_path, std::vector<hsize_t> &dims)
+    {
+        hdf5::dataset dataset(H5Dopen(file, dataset_path.c_str(), H5P_DEFAULT));
+        if (! dataset.valid())
+            throw hdf5::exception("unable to open dataset");
+        hdf5::dataspace dataspace(H5Dget_space(dataset));
+        if (! dataspace.valid())
+            throw hdf5::exception("unable to get data space");
+        int ndims = H5Sget_simple_extent_ndims(dataspace);
+        if (ndims <= 0)
+            throw hdf5::exception("wrong number of dimensions in dataset");
+        dims.resize(ndims);
+        if (H5Sget_simple_extent_dims(dataspace, &dims[0], nullptr) != ndims)
+            throw hdf5::exception("unable to get dimension sizes");
+    }
+
+    /*!
+     * \brief Traits for kind of data
+     * \tparam kind kind of data layout
+     */
+    template<int kind>
+    struct ktrait final {};
+
+    /*!
+     * \brief Traits for new eiger data
+     */
+    template<>
+    struct ktrait<kind_11> final {
+        /*!
+         * \brief Assign matrix dimensions from data dimensions
+         * \param mdims (OUT) matrix dimensions to be assigned
+         * \param ddims (IN) data dimensions
+         * \param data_path (IN) path of data file
+         * \param data_location (IN) location of dataset
+         */
+        static void assign_matrix_dims(std::vector<mwSize> &mdims, const std::vector<hsize_t> &ddims, const std::vector<std::string> &data_path, const std::vector<std::string> &data_location)
+        {
+            auto ndims = ddims.size();
+            mdims.resize(ndims);
+            for (decltype(ndims) i=0; i<ndims; i++) {
+                mdims[i] = static_cast<mwSize>(ddims[i]);
+            }
+        }
+
+        /*!
+         * \brief Path range start offset
+         * \param first start offset in work range
+         * \param last end offset in work range (exclusive)
+         * \return start path start range offset
+         */
+        static int start_path(int first, int last)
+        {
+            return 0;
+        }
+
+        /*!
+         * \brief Path range end offset
+         * \param first start offset in work range
+         * \param last end offset in work range (exclusive)
+         * \return path range end offset (exclusive)
+         */
+        static int end_path(int first, int last)
+        {
+            return 1;
+        }
+
+        /*!
+         * \brief Location range start offset
+         * \param f file index
+         * \param first start offset in work range
+         * \param last end offset in work range (exclusive)
+         * \return start location start range offset
+         */
+        static int start_location(int f, int first, int last)
+        {
+            return 0;
+        }
+
+        /*!
+         * \brief Location range end offset
+         * \param f file index
+         * \param first start offset in work range
+         * \param last end offset in work range (exclusive)
+         * \return start location end range offset
+         */
+        static int end_location(int f, int first, int last)
+        {
+            return 1;
+        }
+
+        /*!
+         * \brief Buffer chunk offset
+         * \param f file index
+         * \param d dataset index
+         * \param first start offset in work range
+         * \return offset into chunk buffer
+         */
+        static std::size_t buf_offset(int f, int d, int first)
+        {
+            return 0;
+        }
+    };
+
+    /*!
+     * \brief Traits for old eiger data
+     */
+    template<>
+    struct ktrait<kind_N1> final {
+        static void assign_matrix_dims(std::vector<mwSize> &mdims, const std::vector<hsize_t> &ddims, const std::vector<std::string> &data_path, const std::vector<std::string> &data_location)
+        {
+            auto ndims = ddims.size();
+            mdims.resize(ndims+1);
+            mdims[0] = data_path.size();
+            for (decltype(ndims) i=0; i<ndims; i++) {
+                mdims[i+1] = static_cast<mwSize>(ddims[i]);
+            }
+        }
+
+        static int start_path(int first, int last)
+        {
+            return first;
+        }
+
+        static int end_path(int first, int last)
+        {
+            return last;
+        }
+
+        static int start_location(int f, int first, int last)
+        {
+            return 0;
+        }
+
+        static int end_location(int f, int first, int last)
+        {
+            return 1;
+        }
+
+        static std::size_t buf_offset(int f, int d, int first)
+        {
+            return f - first;
+        }
+    };
+
+    /*!
+     * \brief Traits for ESRF data
+     */
+    template<>
+    struct ktrait<kind_1N> final {
+        static void assign_matrix_dims(std::vector<mwSize> &mdims, const std::vector<hsize_t> &ddims, const std::vector<std::string> &data_path, const std::vector<std::string> &data_location)
+        {
+            auto ndims = ddims.size();
+            mdims.resize(ndims+1);
+            mdims[0] = data_location.size();
+            for (decltype(ndims) i=0; i<ndims; i++) {
+                mdims[i+1] = static_cast<mwSize>(ddims[i]);
+            }
+        }
+
+        static int start_path(int first, int last)
+        {
+            return 0;
+        }
+
+        static int end_path(int first, int last)
+        {
+            return 1;
+        }
+
+        static int start_location(int f, int first, int last)
+        {
+            return first;
+        }
+
+        static int end_location(int f, int first, int last)
+        {
+            return last;
+        }
+
+        static std::size_t buf_offset(int f, int d, int first)
+        {
+            return d - first;
+        }
+    };
+
+    /*!
+     * \brief Traits for N (file, dataset) pairs
+     */
+    template<>
+    struct ktrait<kind_NN> final {
+        static void assign_matrix_dims(std::vector<mwSize> &mdims, const std::vector<hsize_t> &ddims, const std::vector<std::string> &data_path, const std::vector<std::string> &data_location)
+        {
+            auto ndims = ddims.size();
+            mdims.resize(ndims+1);
+            mdims[0] = data_path.size();
+            for (decltype(ndims) i=0; i<ndims; i++) {
+                mdims[i+1] = static_cast<mwSize>(ddims[i]);
+            }
+        }
+
+        static int start_path(int first, int last)
+        {
+            return first;
+        }
+
+        static int end_path(int first, int last)
+        {
+            return last;
+        }
+
+        static int start_location(int f, int first, int last)
+        {
+            return f;
+        }
+
+        static int end_location(int f, int first, int last)
+        {
+            return f+1;
+        }
+
+        static std::size_t buf_offset(int f, int d, int first)
+        {
+            return f - first;
+        }
+    };
+
+    /*!
+     * \brief Read in objects with multiple processes
+     *
+     * \param nprocs number of read processes
+     * \param data_path paths to hdf5 files
+     * \param data_location paths to datasets within hdf5 files
+     * \param image_size two dimensional image size
+     * \param roi_center center of two dimensional image
+     * \tparam f_type float or double result type
+     * \tparam kind kind of data layout (see above)
+     * \return MATLAB result array
+     */
+    template <typename f_type, int kind>
+    mxArray* read_data_parallel(int nprocs,
+                                const std::vector<std::string> &data_path,
+                                const std::vector<std::string> &data_location,
+                                std::vector<long> &image_size, std::vector<long> &roi_center)
+    {
+        using kt = ktrait<kind>;
+        try {
+            std::vector<mwSize> mdims;      // matrix dimensions
+            std::vector<hsize_t> ddims;     // dataset dimensions
+
+            // Assign matrix dimension values
+            {
+                hdf5::file file(open_file(data_path[0]));
+                if (! file.valid())
+                    throw hdf5::exception("unable to open eiger data file");
+                get_data_size(file, data_location[0], ddims);
+                auto ndims = ddims.size();
+                if (ndims < 2)
+                    throw std::invalid_argument("dataset must have at least two dimensions");
+                DEBUG {
+                    OUT << "data size: " << vec_to_str(ddims, 'x') << std::endl;
+                }
+
+                auto sz = ndims - 2;
+                if (image_size.empty()) {
+                    image_size.resize(2);
+                    image_size[0] = ddims[sz];
+                    image_size[1] = ddims[sz+1];
+                    DEBUG {
+                        OUT << "setting image size to " << image_size[0] << 'x' << image_size[1] << std::endl;
+                    }
+                } else if (image_size.size() != 2)
+                    throw std::invalid_argument("image_size must be two dimensional");
+
+                if (roi_center.empty()) {
+                    roi_center.resize(2);
+                    roi_center[0] = ddims[sz] / 2;
+                    roi_center[1] = ddims[sz+1] / 2;
+                    DEBUG {
+                        OUT << "setting roi center to " << roi_center[0] << 'x' << roi_center[1] << std::endl;
+                    }
+                } else if (roi_center.size() != 2)
+                    throw std::invalid_argument("roi_center must be two dimensional");
+            }
+
+            kt::assign_matrix_dims(mdims, ddims, data_path, data_location);
+            {
+                auto ndims = mdims.size() - 2;
+                mdims[ndims] = image_size[0];
+                mdims[ndims+1] = image_size[1];
+            }
+            collapse_dims(mdims);
+            {
+                mwSize sz = 1;
+                for (const auto &d : mdims)
+                    sz *= d;
+                if (sz <= 0)
+                    throw std::invalid_argument("empty result matrix");
+            }
+
+            // Create space for MATLAB array
+            DEBUG {
+                OUT << "creating MATLAB array: " << vec_to_str(mdims, 'x') << std::endl;
+            }
+            mx_ptr<mxArray> matrix;
+            {
+                std::vector<mwSize> rdims(mdims);
+                std::reverse(rdims.begin(), rdims.end());
+                matrix.reset(mxCreateNumericArray(rdims.size(), &rdims[0], mx_trait<f_type>::class_id, mxREAL));
+            }
+            if (! matrix.get())
+                throw std::runtime_error("matrix creation failed");
+
+            // Prepare (shared) memory buffer
+            if (mdims[0] < nprocs)
+                nprocs = mdims[0];
+            mp::buf<f_type> buf(mx_trait<f_type>::get(matrix.get()), mdims, nprocs);
+
+//             auto r_time = std::chrono::high_resolution_clock::now();
+            // Read objects data into buffer
+            mp::run<f_type>(nprocs, buf, [&data_path, &data_location, &mdims, &ddims, &roi_center](int proc, mp::buf<f_type> &buf) {
+                    int first = buf.offset(proc);
+                    int last = buf.offset(proc+1);
+                    DEBUG {
+                        OUT << "Process " << proc << ": " << first << ".." << last << std::endl;
+                    }
+                    for (int f=kt::start_path(first, last); f<kt::end_path(first, last); f++) {
+                        for (int d=kt::start_location(f, first, last); d<kt::end_location(f, first, last); d++) {
+                            hdf5::file file(open_file(data_path[f]));
+                            f_type *data_buf = buf.get(proc) + kt::buf_offset(f, d, first) * buf.chunk_size;
+                            read_data_chunk<f_type, kind>(file, data_location[d], first, last, mdims, ddims, data_buf, roi_center);
+                        } // dataset locations
+                    } // file paths
+                });
+//             // Set statistics
+//             if (r_stat) {
+//                 r_stat->seconds = std::chrono::duration<double>(std::chrono::high_resolution_clock::now() - r_time).count();
+//                 r_stat->nbytes = buf.length * sizeof(typename mx_trait<f_type>::complex_type);
+//             }
+            // Return result
+            return matrix.release();
+        } catch (std::exception &ex) {
+            mexErrMsgIdAndTxt("ptycho:read:failed", "%s", ex.what());
+            return nullptr;
+        }
+    }
+
+    /*!
+     * \brief Read in objects with multiple processes
+     *
+     * \param nprocs number of read processes
+     * \param data_path paths to hdf5 files
+     * \param data_location paths to datasets within hdf5 files
+     * \param image_size two dimensional image size
+     * \param roi_center center of two dimensional image
+     * \tparam f_type float or double result type
+     * \return MATLAB result array
+     */
+    template <typename f_type>
+    mxArray* read_data_parallel(int nprocs,
+                                const std::vector<std::string> &data_path,
+                                const std::vector<std::string> &data_location,
+                                std::vector<long> &image_size, std::vector<long> &roi_center)
+    {
+        if (data_path.size() == 1) {
+            if (data_location.size() == 1) {
+                DEBUG {
+                    OUT << "11: new eiger" << std::endl;
+                }
+                return read_data_parallel<f_type, kind_11>(nprocs, data_path, data_location, image_size, roi_center);
+            } else {
+                DEBUG {
+                    OUT << "1N: esrf" << std::endl;
+                }
+                return read_data_parallel<f_type, kind_1N>(nprocs, data_path, data_location, image_size, roi_center);
+            }
+        } else {
+            if (data_location.size() == 1) {
+                DEBUG {
+                    OUT << "N1: old eiger" << std::endl;
+                }
+                return read_data_parallel<f_type, kind_N1>(nprocs, data_path, data_location, image_size, roi_center);
+            } else {
+                DEBUG {
+                    OUT << "NN: file/dataset pairs" << std::endl;
+                }
+                return read_data_parallel<f_type, kind_NN>(nprocs, data_path, data_location, image_size, roi_center);
+            }
+        }
+    }
+
+} // namespace
+
+namespace data_prep {
+
+    mxArray* read_eiger_data(long nprocs,
+                             const std::vector<std::string> &data_path,
+                             const std::vector<std::string> &data_location,
+                             std::vector<long> &image_size, std::vector<long> &roi_center,
+                             precision::type prec)
+    {
+        if (nprocs <= 0)
+            nprocs = 1;
+
+        mx_ptr<mxArray> result;
+        {
+            if (prec == precision::type::Double)
+                result.reset(read_data_parallel<double>(nprocs, data_path, data_location, image_size, roi_center));
+            else
+                result.reset(read_data_parallel<float>(nprocs, data_path, data_location, image_size, roi_center));
+        }
+        return result.release();
+    }
+
+} // namespace data_prep

+io/private/ptycho_reader/read_eiger_data.h

@@ -0,0 +1,33 @@
+#ifndef READ_EIGER_DATA_H
+#define READ_EIGER_DATA_H
+
+/*!
+ * \file
+ * Header file for the read Eiger data functionality
+ */
+
+/*!
+ * \brief Data preparator functions
+ */
+namespace data_prep {
+
+    /*!
+     * \brief read Eiger data
+     *
+     * \param nprocs number of parallel read processes
+     * \param data_path paths to Eiger data files
+     * \param data_location location within file, either 1 for all paths or one for each path
+     * \param image_size size of returned image data
+     * \param roi_center region of interest center relative to measured data
+     * \param prec precision of returned data
+     * \return MATLAB array with resized and recentered images
+     * \pre The 1:N or N:N relationship between data_location and data_path must hold
+     */
+    mxArray* read_eiger_data(long nprocs,
+                             const std::vector<std::string> &data_path,
+                             const std::vector<std::string> &data_location,
+                             std::vector<long> &image_size, std::vector<long> &roi_center,
+                             precision::type prec);
+}
+
+#endif

+io/private/ptycho_reader/read_object_data.cc

@@ -0,0 +1,498 @@
+/*!
+ * \file
+ * Read Object Data
+ *
+ * This file contains the main mex code for reading object data into MATLAB memory.
+ */
+
+#include <string>
+#include <memory>
+#include <chrono>
+#include <cassert>
+#include <array>
+#include <vector>
+#include <cstring>
+#include <cerrno>
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <exception>
+#include <stdexcept>
+#include <unistd.h>
+#include <functional>
+#include <algorithm>
+#include <sys/mman.h>
+#include <sys/wait.h>
+#include "precision.h"
+#include "hdf5_helper.h"
+#include "mex.h"
+#include "read_object_data.h"
+#include "debug_helper.h"
+#include "multi_processing.h"
+
+namespace {
+
+    constexpr int min_dims = 2;                 //!< Minimum number of dimensions (row, col)
+    constexpr int max_dims = 4;                 //!< Maximum number of dimensions (nslices, nmodes, row, col)
+
+    #include "mex_helper.h"
+
+    /*!
+     * \brief Read error exception class
+     */
+    struct read_error : public std::exception {
+        bool converted = false;    //!< has the error message been converted (to include 1-based file index)
+        uint32_t file_index;       //!< file index (starting from 0)
+        std::string error_message; //!< error message (converted if converted is true)
+
+        /*!
+         * \brief Constructor
+         * \param index 0-based file index
+         * \param message unconverted error message
+         */
+        read_error(uint32_t index, const std::string &message)
+            : file_index(index), error_message(message)
+        {}
+
+        /*!
+         * \brief Retrieve converted error message
+         * This method first attempts to convert the error message by appending the 1-based file index (MATLAB indexing convention).
+         * If this fails, the unconverted error message is returned.
+         * \return converted error message
+         */
+        const char* what() const noexcept override
+        {
+            try {
+                if (! converted) {
+                    std::ostringstream oss;
+                    oss << error_message << " (file " << (file_index+1) << ")";
+                    const_cast<std::string&>(error_message) = oss.str();
+                    const_cast<bool&>(converted) = true;
+                }
+            } catch (...) {}
+            return error_message.c_str();
+        }
+    };
+
+    /*!
+     * \brief Error collector class
+     * The class either keeps a vector to collect file indices for files that could not be read, or it throws an exception, depending on the in_collect_mode flag.
+     * The class provides a pointer to the singleton instance.
+     */
+    struct error_collector {
+        static error_collector *singleton;  //!< static pointer to make the singleton instance available
+        bool in_collect_mode = false;       //!< is the error method collecting or throwing errors?
+        std::vector<uint32_t> bad_index;    //!< vector of bad file indices
+
+        /*!
+         * \brief Constructor
+         * \param collect_mode true if errors should be collected instead of thrown
+         */
+        explicit error_collector(bool collect_mode)
+            : in_collect_mode(collect_mode)
+        {
+            if (! singleton)
+                singleton = this;
+            else
+                throw std::runtime_error("internal error - multiple error_collector instances!");
+        }
+
+        /*!
+         * \brief Destructor
+         * Zero out the singleton pointer
+         */
+        ~error_collector() noexcept
+        {
+            singleton = nullptr;
+        }
+
+        /*!
+         * \brief Get pointer to singleton instance
+         */
+        static error_collector& get() noexcept
+        {
+            return *singleton;
+        }
+
+        /*!
+         * \brief Collect or throw error
+         * \param ex read error exception that would be thrown if not in collector mode
+         */
+        void error(read_error &&ex)
+        {
+            if (! in_collect_mode)
+                throw ex;
+            bad_index.push_back(ex.file_index);
+            DEBUG {
+                OUT << "error reading file " << ex.file_index << ": " << ex.error_message << std::endl;
+            }
+        }
+
+        /*!
+         * \brief Add index to bad indices vector
+         * \param index index of file that can not be read
+         */
+        void add(uint32_t index) noexcept
+        {
+            bad_index.push_back(index);
+        }
+    };
+
+    error_collector* error_collector::singleton = nullptr;  //!< static error_collectr singleton pointer
+
+    /*!
+     * \brief Adapt data space layout
+     *
+     * Adapt the memory and file dataset layouts by selecting apropriate hyperslabs
+     *
+     * \param mspace memory data space layout
+     * \param fspace file dataset layout
+     * \param mdim memory space dimensions
+     * \param fdim file dataset dimensions
+     * \param doffset Start of the dimsensions present in the file data
+     */
+    void adapt_spaces(hdf5::dataspace &mspace, hdf5::dataspace &fspace, const hsize_t mdim[max_dims], const hsize_t fdim[max_dims], int doffset)
+    {
+        hsize_t mstart[max_dims] = {0};
+        hsize_t fstart[max_dims] = {0};
+        hsize_t mcount[max_dims] = { mdim[0], mdim[1], mdim[2], mdim[3] };
+        hsize_t fcount[max_dims] = { fdim[0], fdim[1], fdim[2], fdim[3] };
+        bool mselect = false;
+        bool fselect = false;
+
+        for (int i=2; i<=3; i++) {
+            if (mdim[i] < fdim[i]) {
+                fstart[i] = (fdim[i] - mdim[i]) / 2;
+                fcount[i] = mdim[i];
+                fselect = true;
+            } else if (mdim[i] > fdim[i]) {
+                mstart[i] = (mdim[i] - fdim[i]) / 2;
+                mcount[i] = fdim[i];
+                mselect = true;
+            }
+        }
+        if (mselect) {
+            DEBUG {
+                OUT << "memory subspace start=" << mstart[0] << ',' << mstart[1] << ',' << mstart[2] << ',' << mstart[3] << '/' << doffset << " count=" << mcount[0] << ',' << mcount[1] << ',' << mcount[2] << ',' << mcount[3] << '/' << doffset << "\nfile space is " << fdim[0] << ',' << fdim[1] << ',' << fdim[2] << ',' << fdim[3] << std::endl;
+            }
+            if (H5Sselect_hyperslab(mspace, H5S_SELECT_SET, &mstart[doffset], nullptr, &mcount[doffset], nullptr) < 0)
+                throw hdf5::exception("unable to set hdf5 memory space");
+        }
+        if (fselect) {
+            DEBUG {
+                OUT << "file subspace start=" << fstart[0] << ',' << fstart[1] << ',' << fstart[2] << ',' << fstart[3] << '/' << doffset << " count=" << fcount[0] << ',' << fcount[1] << ',' << fcount[2] << ',' << fcount[3] << '/' << doffset << "\nmem space is " << mdim[0] << ',' << mdim[1] << ',' << mdim[2] << ',' << mdim[3] << std::endl;
+            }
+            if (H5Sselect_hyperslab(fspace, H5S_SELECT_SET, &fstart[doffset], nullptr, &fcount[doffset], nullptr) < 0)
+                throw hdf5::exception("unable to set hdf5 dataset space");
+        }
+    }
+
+    /*!
+     * \brief Read object from file into buffer
+     * \param file HDF5 file
+     * \param object_path path to object dataset
+     * \param mdims result matrix dimensions (with size 5)
+     * \param buf pointer to parent acessible buffer
+     * \tparam f_type buffer element type
+     */
+    template <typename f_type>
+    void read_complex_matrix(hdf5::file &file, const std::string &object_path,
+                             const std::vector<mwSize> &mdims, typename mx_trait<f_type>::complex_type *buf)
+    {
+        hsize_t fdims[max_dims] = { 1, 1, 0, 0 }; // (nslices, nmodes, nrows, ncols)
+        hsize_t bdims[max_dims] = { 1, 1, 0, 0 }; // file and buffer dimensions
+        unsigned long nelements = 0;
+        int dim_offset = 0;
+        int ndims = 0;
+        int mdim_offset = 0;
+        for (; (mdim_offset < max_dims-2) && (mdims[mdim_offset] == 1); mdim_offset++);
+
+        // Create complex type
+        hdf5::h5type complex_type(hdf5::create_complex_type<f_type>());
+        // Open dataset
+        hdf5::dataset dataset(H5Dopen(file, object_path.c_str(), H5P_DEFAULT));
+        if (! dataset.valid())
+            throw hdf5::exception("unable to open dataset");
+        hdf5::dataspace dataspace(H5Dget_space(dataset));
+        if (! dataspace.valid())
+            throw hdf5::exception("unable to open data space for dataset");
+        ndims = H5Sget_simple_extent_ndims(dataspace);
+        if (ndims < min_dims || ndims > max_dims)
+            throw hdf5::exception("wrong number of dimensions in dataset");
+        dim_offset = max_dims - ndims;
+        if (H5Sget_simple_extent_dims(dataspace, &fdims[dim_offset], NULL) != ndims)
+            throw hdf5::exception("dimension error");
+        for (int i=max_dims-2; i; i--) {
+            if (fdims[i-1] != mdims[i]) {
+                if (i != mdim_offset)
+                    throw hdf5::exception("dimension mismatch");
+            }
+        }
+        // Create memory space and adapt data space
+        bdims[0] = fdims[0]; // nslices
+        bdims[1] = fdims[1]; // nmodes
+        bdims[2] = mdims[3]; // nrows wanted
+        bdims[3] = mdims[4]; // ncols wanted
+        DEBUG {
+            OUT << "bdims(" << bdims[0] << ',' << bdims[1] << ',' << bdims[2] << '/' << fdims[2] << ',' << bdims[3] << '/' << fdims[3] << ')' << dim_offset << std::endl;
+        }
+        hdf5::dataspace memspace(H5Screate_simple(ndims, &bdims[dim_offset], nullptr));
+        if (! memspace.valid())
+            throw hdf5::exception("unable to create memory data space");
+        if ((bdims[2] != fdims[2]) || (bdims[3] != fdims[3]))
+            adapt_spaces(memspace, dataspace, bdims, fdims, dim_offset);
+        // Read dataset
+        if (H5Dread(dataset, complex_type, memspace, dataspace, H5P_DEFAULT, buf) < 0)
+            throw hdf5::exception("unable to read dataset");
+
+        DEBUG {
+            int last = bdims[max_dims-2] * bdims[max_dims-1] - 1;
+            OUT << "read finished - [0]=(" << buf[0].real << ',' << buf[0].imag << ") [" << last << "]=(" << buf[last].real << ',' << buf[last].imag << ')' << std::endl;
+        }
+    }
+
+    /*!
+     * \brief Open HDF5 file
+     * \param file_path path to HDF5 file
+     * \return HDF5 file object id
+     */
+    hid_t open_file(const std::string &file_path)
+    {
+        hdf5::file file(H5Fopen(file_path.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT));
+        if (! file.valid())
+            throw hdf5::exception("unable to open file");
+        return file.grab();
+    }
+
+    /*!
+     * \brief Nonzero dimension size?
+     * \param dims size of twodimensional object
+     * \tparam T type of dimension size
+     * \return are dimension sizes positive?
+     */
+    template <typename T>
+    bool dims_ok (const T *dims) noexcept
+    {
+        return dims[0] > 0 || dims[1] > 0;
+    }
+
+    /*!
+     * \brief Drop leading dimensions of size 1
+     * \param dim dimsension sizes
+     * \param max maximum dimensions
+     * \tparam T type of dimension size
+     * \return number of dimsension of size bigger than 1
+     */
+    template <typename T>
+    int collapse_dims (const std::vector<T> &dim, int max) noexcept
+    {
+        int i=0;
+        for (; i<max-2; i++) {
+            if (dim[i] != 1)
+                break;
+        }
+        return max-i;
+    }
+
+    /*!
+     * \brief Get HDF5 object size
+     *
+     * \param file HDF5 file
+     * \param dataset_path object dataset path
+     * \param dims filled with object dimensions
+     */
+    void get_object_size(hdf5::file &file, const std::string &dataset_path, hsize_t dims[max_dims])
+    {
+        int dim_offset = 0;
+        int ndims;
+
+        hdf5::dataset dataset(H5Dopen(file, dataset_path.c_str(), H5P_DEFAULT));
+        if (! dataset.valid())
+            throw hdf5::exception("unable to open dataset");
+        hdf5::dataspace dataspace(H5Dget_space(dataset));
+        if (! dataspace.valid())
+            throw hdf5::exception("unable to get data space");
+        ndims = H5Sget_simple_extent_ndims(dataspace);
+        if (ndims < min_dims || ndims > max_dims)
+            throw hdf5::exception("wrong number of dimensions in dataset");
+        dim_offset = max_dims - ndims;
+        if (H5Sget_simple_extent_dims(dataspace, &dims[dim_offset], NULL) != ndims)
+            throw hdf5::exception("unable to get dimension sizes");
+    }
+
+    /*!
+     * \brief Read in objects with multiple processes
+     *
+     * \param nprocs number of read processes
+     * \param dims desired result dimensions (or [0, 0] to get the default behaviour)
+     * \param object_path path to hdf5 object within the files
+     * \param file_paths paths to hdf5 objects
+     * \param r_stat fill in if not null
+     * \tparam f_type float or double result type
+     * \return MATLAB result array
+     */
+    template <typename f_type>
+    mxArray* read_objects_parallel(int nprocs, std::array<int64_t, 2> &dims,
+                                   const std::string &object_path, const std::vector<std::string> &file_paths,
+                                   data_prep::read_stat *r_stat)
+    {
+        try {
+            std::vector<mwSize> mdims({ 1, 1, 1, 0, 0 });   // extended matrix dimensions
+            hsize_t odims[max_dims] = { 1, 1, 0, 0 };       // extended object dimensions (nobjects, nslices, nmodes, nrows, ncols)
+
+            // Assign matrix dimension values
+            {
+                hdf5::file file(open_file(file_paths[0]));
+                get_object_size(file, object_path, odims);
+            }
+            for (unsigned int i=1; i<max_dims-1; i++) {
+                mdims[i] = (mwSize)odims[i-1];
+            }
+            if (!dims_ok(&dims[0])) {
+                mdims[max_dims-1] = odims[max_dims-2];
+                mdims[max_dims] = odims[max_dims-1];
+            } else {
+                mdims[max_dims-1] = dims[0];
+                mdims[max_dims] = dims[1];
+            }
+            int nmdims = collapse_dims(mdims, max_dims+1);
+            if (file_paths.size() > 1) {
+                mdims[max_dims - nmdims] = (mwSize)file_paths.size();
+                nmdims++;
+            }
+
+            std::vector<mwSize> matrix_dims(mdims);         // matrix dimensions
+            matrix_dims.erase(matrix_dims.begin(), matrix_dims.begin() + max_dims + 1 - nmdims); // cut away extension dimensions
+
+            // Create space for MATLAB array
+            mx_ptr<mxArray> matrix;
+            {
+                std::vector<mwSize> rdims(matrix_dims);
+                std::reverse(rdims.begin(), rdims.end());
+                matrix.reset(mxCreateNumericArray(nmdims, &rdims[0], mx_trait<f_type>::class_id, mxCOMPLEX));
+            }
+            if (! matrix.get())
+                throw std::runtime_error("matrix creation failed");
+
+            // Prepare (shared) memory buffer
+            // HACK: misuse error message area as array of bad file indices
+            // There should be space for the max number of handled files plus the length field
+            const unsigned int message_size = error_collector::singleton->in_collect_mode ? std::max((file_paths.size() / nprocs + 2) * sizeof(uint32_t), 64ul) : 0;
+            mp::buf<typename mx_trait<f_type>::complex_type> buf(mx_trait<f_type>::get_complex(matrix.get()), matrix_dims, nprocs, message_size);
+
+            auto r_time = std::chrono::high_resolution_clock::now();
+            // Spawn processes if needed
+            mp::run<typename mx_trait<f_type>::complex_type>(nprocs, buf, [&file_paths, &object_path, &mdims](int proc, mp::buf<typename mx_trait<f_type>::complex_type> &buf) {
+                    // Read objects data into buffer
+                    typename mx_trait<f_type>::complex_type *data_buf = buf.get(proc);
+                    int first = buf.offset(proc);
+                    int last = buf.offset(proc+1);
+                    for (int obj=first; obj<last; obj++) {
+                        try {
+                            std::size_t offset = (obj - first) * buf.chunk_size;
+                            DEBUG {
+                                OUT << "Process " << proc << ": " << file_paths[obj] << '@' << offset << std::endl;
+                            }
+                            hdf5::file file(open_file(file_paths[obj]));
+                            read_complex_matrix<f_type>(file, object_path, mdims, &data_buf[offset]);
+                        } catch (std::exception &ex) {
+                            error_collector::singleton->error(read_error(obj, ex.what()));
+                        } catch (...) {
+                            error_collector::singleton->error(read_error(obj, "mysterious read error"));
+                        }
+                    }
+                    if (proc && error_collector::singleton->in_collect_mode) {
+                        // HACK: misuse error message area as array of bad file indices for this process
+                        // Layout: [ N | elem0 | elem1 | ... | elemN-1 ]
+                        uint32_t *msg_area = reinterpret_cast<uint32_t*>(buf.get_msg(proc));
+                        auto &bad_index = error_collector::singleton->bad_index;
+                        msg_area[0] = bad_index.size();
+                        std::copy(bad_index.begin(), bad_index.end(), &msg_area[1]);
+                    }
+                });
+            // Set statistics (not correct for error collection mode)
+            if (r_stat) {
+                r_stat->seconds = std::chrono::duration<double>(std::chrono::high_resolution_clock::now() - r_time).count();
+                r_stat->nbytes = buf.length * sizeof(typename mx_trait<f_type>::complex_type);
+            }
+            // Collect bad indices in error collection mode
+            if (error_collector::singleton->in_collect_mode) {
+                DEBUG {
+                    OUT << "Collecting bad indices:" << std::endl;
+                }
+                for (int proc=1; proc<nprocs; proc++) {
+                    // HACK: misuse error message area as array of bad file indices
+                    // Layout: [ N | elem0 | elem1 | ... | elemN-1 ]
+                    uint32_t *msg_area = reinterpret_cast<uint32_t*>(buf.get_msg(proc));
+                    uint32_t nbad = msg_area[0];
+                    DEBUG {
+                        OUT << "  process " << proc << ": " << nbad << " indices" << std::endl;
+                    }
+                    if (nbad > (file_paths.size() / nprocs + 1))
+                        throw std::runtime_error("transfer of bad indices failed");
+                    for (uint32_t i=1; i<=nbad; i++)
+                        error_collector::singleton->add(msg_area[i]);
+                }
+            }
+            // Return result
+            return matrix.release();
+        } catch (std::exception &ex) {
+            mexErrMsgIdAndTxt("ptycho:read:failed", "%s", ex.what());
+            return nullptr;
+        }
+    }
+
+    /*!
+     * \brief Convert vector<uint32_t> to MATLAB array
+     * \param vec vector to be converted
+     * \return MATLAB array
+     */
+    mxArray* vector_to_array(std::vector<uint32_t> &vec)
+    {
+        mwSize vlen[2] = {1, vec.size()};
+        mx_ptr<mxArray> arr(mxCreateNumericArray(2, vlen, mxUINT32_CLASS, mxREAL));
+        if (! arr.get())
+            throw std::runtime_error("unable to create bad indices array");
+        if (! vec.empty()) {
+            auto* element = mxGetUint32s(arr.get());
+            if (! element)
+                throw std::runtime_error("unable to retrieve pointer for bad indices array elements");
+            for (auto& e : vec)
+                *element++ = e+1;
+        }
+        return arr.release();
+    }
+
+} // namespace
+
+namespace data_prep {
+
+    mxArray* read_object_data(int nprocs, precision::type prec, std::array<int64_t, 2> &dims,
+                              const std::string &object_path, const std::vector<std::string> &file_paths,
+                              mxArray **bad_file_idx, read_stat *r_stat)
+    {
+        if (! file_paths.size())
+            return nullptr;
+
+        if (nprocs <= 0)
+            nprocs = 1;
+        if (nprocs > file_paths.size())
+            nprocs = (int)file_paths.size();
+
+        error_collector errcol(bad_file_idx != nullptr);
+
+        mx_ptr<mxArray> result;
+        {
+            if (prec == precision::type::Double)
+                result.reset(read_objects_parallel<double>(nprocs, dims, object_path, file_paths, r_stat));
+            else
+                result.reset(read_objects_parallel<float>(nprocs, dims, object_path, file_paths, r_stat));
+        }
+
+        if (errcol.in_collect_mode)
+            *bad_file_idx = vector_to_array(errcol.bad_index);
+
+        return result.release();
+    }
+
+} // namespace data_prep

+io/private/ptycho_reader/read_object_data.h

@@ -0,0 +1,43 @@
+#ifndef READ_OBJECT_DATA_H
+#define READ_OBJECT_DATA_H
+
+/*!
+ * \file
+ * Declaration of the read_object_data function
+ */
+
+/*!
+ * \brief Data preparator functions
+ */
+namespace data_prep {
+
+    /*!
+     * \brief Statistics collection
+     */
+    struct read_stat final {
+        double seconds;         //!< read access time in seconds
+        unsigned long nbytes;   //!< number of bytes transferred
+    };
+
+    /*!
+     * \brief Read object data
+     *
+     * The function will either create the bad indices vector or return with exception and error message.
+     *
+     * \param nprocs Number of read processes
+     * \param prec Single or Double precision
+     * \param dims Object size (nrows, ncols)
+     * \param object_path Path to object within HDF5 file
+     * \param file_paths Paths to HDF5 files, each containing a diffraction pattern at object_path
+     * \param bad_file_idx Indices of nonreadable files array, to be created if not NULL
+     * \param r_stat Fill with info if not null
+     * \return MATLAB array with object data
+     */
+    mxArray* read_object_data(int nprocs, precision::type prec, std::array<int64_t, 2> &dims,
+                              const std::string &object_path, const std::vector<std::string> &file_paths,
+                              mxArray **bad_file_idx,
+                              read_stat *r_stat = nullptr);
+
+} // namespace data_prep
+
+#endif

+io/private/ptycho_reader/setup_environment.sh

@@ -0,0 +1,8 @@
+module add gcc/6.3.0
+module add hdf5_serial/1.8.18
+
+module use unstable
+module add matlab/2018a
+
+module use Libraries
+module add tiff/4.0.9

+io/read_scan_positions_spec.m

@@ -0,0 +1,127 @@
+%   [pos] = read_scan_positions_spec(specdatafile,scans,motors)
+%   Returns SPEC scan positions.
+%   positions = read_scan_positions(specdatafile,scan,motors) returns the 
+%   positions of a SPEC file corresponding to two given motor names
+%
+%   Input parameters: 
+%   specdatafile: string with the file name of the SPEC data file 
+%                 (e.g.'~/Data10/specES1/dat-files/specES1_started_2017_07_11_1633.dat' )
+%   scans: scan number(s) (e.g. [2:77,88])
+%   motors: 1x2 cell array with scanning motor names (e.g. {'samx','samy'})
+%
+%   The output is a structure array with length(scans) number of elements,
+%   each element containing in the field 'data' an Nx2 array with N
+%   being the number of points in the scan. The two columns contain motor
+%   positions for motors samx and samy, respectively, if motors={'samx','samy'}
+%
+%   Example:  
+%   pos=read_scan_positions_spec('~/Data10/specES1/dat-files/specES1_started_2017_07_11_1633.dat',[58:59],{'samx','samy'}); 
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [pos] = read_scan_positions_spec(specdatafile,scans,motors)
+
+Sp = io.spec_read(specdatafile,'ScanNr',scans);
+
+if numel(Sp) == 1
+    S{1} = Sp;
+else
+    S = Sp;
+end
+clear Sp
+
+field='data';
+value=cell(1,length(S));
+pos=struct(field,value);
+
+for ii=1:length(S)
+    
+    C = strsplit(S{ii}.S,' ');
+    
+    %Is the scan 1D or 2D
+    if numel(C)<10
+        scanis1D = true;
+    else
+        scanis1D = false;
+    end
+    
+    fastmotorname = C{4};
+        
+    % Which is the fast axis?
+    if strcmpi(fastmotorname,motors{1})      % Fast axis is x
+        fast_axis_index = 1;
+        slow_axis_index = 2;
+    elseif strcmpi(fastmotorname,motors{2})  % Fast axis is y                                  % Fast axis is y
+        fast_axis_index = 2;
+        slow_axis_index = 1;
+    else
+        error(sprintf('The scan motor %s does not match any of the inputs %s %s',fastmotorname,motors{1},motors{2}))
+    end
+    
+    % Is this a continuous scan? Identified by having only one position in
+    % the fast axis
+    if numel(getfield(S{ii},fastmotorname))==1
+        iscont = true;
+    else
+        iscont = false;
+    end
+    
+num=str2num(C{7})+1;    
+
+    if scanis1D
+        positions=zeros(num,2);
+        if iscont
+            step=(str2num(C{6})-str2num(C{5}))/str2num(C{7});
+            positions(:,fast_axis_index) = [str2num(C{5}):step:str2num(C{6})];
+        else
+            positions(:,fast_axis_index) = getfield(S{ii}, motors{fast_axis_index});
+        end
+        positions(:,slow_axis_index) = zeros(num,1) + getfield(S{ii}, motors{slow_axis_index});
+    else
+        num2=str2num(C{11})+1;
+        num=num*num2;
+        slowmotorname = C{8};
+        positions=zeros(num,2);
+        if ~strcmpi(slowmotorname,motors{slow_axis_index}) % Check slow axis name
+            error(sprintf('The scan motor %s does not match the input %s',slowmotorname,motors{slow_axis_index}))
+        end
+        positions(:,fast_axis_index) = getfield(S{ii}, motors{fast_axis_index});
+        positions(:,slow_axis_index) = getfield(S{ii}, motors{slow_axis_index});
+    end
+    pos(ii) = struct(field,positions);
+end
+
+return;
+    

+io/save_append.m

@@ -0,0 +1,54 @@
+% Append the varible to the file
+% function save_append(filename,  vars, x)
+%
+% Input:
+% filename - file to which the varible will be appended
+% vars - the varible to save, eg, saveme
+% x - the string of the variable name, eg, 'saveme'
+%
+% Note:
+% This is a temporary solution because either
+% 'save' nor 'vars=evalin('caller', varargin{1})' 
+% works for parfor..
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function save_append(filename,  vars, x)
+
+s.(x) = vars;  
+save(filename, '-struct', 's','-append'); 
+
+end
+

+io/save_tiff_image.m

@@ -0,0 +1,21 @@
+function [] = save_tiff_image(I, path)
+% Wrapper function for saving 16-bit .tiff image
+% Written by YJ
+% ** I: 2D image
+% ** path: file path
+
+tagstruct.ImageLength     = size(I,1);
+tagstruct.ImageWidth      = size(I,2);
+tagstruct.Photometric     = Tiff.Photometric.MinIsBlack;
+tagstruct.BitsPerSample   = 16;
+tagstruct.SamplesPerPixel = 1;
+tagstruct.RowsPerStrip    = 16;
+tagstruct.PlanarConfiguration = Tiff.PlanarConfiguration.Chunky;
+tagstruct.Software        = 'MATLAB';
+t = Tiff(path,'w');
+t.setTag(tagstruct)
+t.write(uint16(mat2gray(I)*2^16));
+t.close();
+
+end
+

+io/save_tiff_image_stack.m

@@ -0,0 +1,65 @@
+function [ ] = save_tiff_image_stack(input, par)
+% Save 3D arry as a (8-bit) tiff stack 
+% Written by YJ
+% ** input: 3D real array
+% ** par: parameters
+
+input = double(input);
+
+if ~isfield(par,'axis'); par.axis = 3; end
+if ~isfield(par,'range'); par.range = [0,0]; end
+if ~isfield(par,'norm_single_slice'); par.norm_single_slice = false; end
+if ~isfield(par,'name'); par.name = 'image_stack.tiff'; end
+
+if any(par.range)
+    range = par.range;
+else
+    range = [min(input(:)),max(input(:))];
+end
+
+switch par.axis
+    case 3
+        if par.norm_single_slice
+            imwrite(mat2gray(input(:,:,1)),par.name,'tiff')
+        else
+            imwrite(mat2gray(input(:,:,1),range),par.name,'tiff')
+        end
+        for i=2:size(input,3)
+            if par.norm_single_slice
+                imwrite(mat2gray(input(:,:,i)),par.name,'tiff','WriteMode','append')
+            else
+                imwrite(mat2gray(input(:,:,i),range),par.name,'tiff','WriteMode','append')
+            end
+        end
+    case 2
+        if par.norm_single_slice
+            imwrite(mat2gray(squeeze(input(:,1,:))),par.name,'tiff')
+        else
+            imwrite(mat2gray(squeeze(input(:,1,:)),range),par.name,'tiff')
+        end
+        for i=2:size(input,2)
+            if par.norm_single_slice
+                imwrite(mat2gray(squeeze(input(:,i,:))),par.name,'tiff','WriteMode','append')
+            else
+                imwrite(mat2gray(squeeze(input(:,i,:)),range),par.name,'tiff','WriteMode','append')
+            end
+        end
+    case 1
+        if par.norm_single_slice
+            imwrite(mat2gray(squeeze(input(1,:,:))),par.name,'tiff')
+        else
+            imwrite(mat2gray(squeeze(input(1,:,:)),range),par.name,'tiff')
+        end
+        for i=2:size(input,1)
+            if par.norm_single_slice
+                imwrite(mat2gray(squeeze(input(i,:,:))),par.name,'tiff','WriteMode','append')
+            else
+                imwrite(mat2gray(squeeze(input(i,:,:)),range),par.name,'tiff','WriteMode','append')
+            end
+        end
+    otherwise
+        disp('Wrong axis! par.axis should be 1, 2, or 3.')
+end
+
+end
+

+io/savefast_safe.m

@@ -0,0 +1,164 @@
+function savefast_safe(filename, varargin)
+% savefast: fast saves of large arrays to .mat files
+%
+% Matlab's 'save' command can be very slow when saving large arrays,
+% because by default Matlab attempts to use compression. This function
+% provides a much faster alternative, at the cost of larger files.
+%
+% The syntax is identical to that of the Matlab save command.
+%
+% Example:
+% >> ops = struct('algorithm', 'greedy');
+% >> A = int32(randi(20, 1000, 1200, 40));
+% >> B = randn(500, 1800, 60);
+% >> tic; save /tmp/test ops A B; toc
+% Elapsed time is 22.980294 seconds.
+% >> tic; savefast /tmp/test ops A B; toc
+% Elapsed time is 0.571098 seconds.
+
+% Copyright 2013 by Timothy E. Holy
+
+% Modified to safe version - Michal Odstrcil, 2017
+
+
+% 
+% Copyright (c) 2013, Timothy Holy
+% All rights reserved.
+% 
+% Redistribution and use in source and binary forms, with or without
+% modification, are permitted provided that the following conditions are
+% met:
+% 
+% * Redistributions of source code must retain the above copyright
+% notice, this list of conditions and the following disclaimer.
+% * Redistributions in binary form must reproduce the above copyright
+% notice, this list of conditions and the following disclaimer in
+% the documentation and/or other materials provided with the distribution
+% 
+% THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+% AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+% IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+% ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+% LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+% CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+% SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+% INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+% CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+% ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+% POSSIBILITY OF SUCH DAMAGE.
+
+
+    % Append .mat if necessary
+    [filepath, filebase, ext] = fileparts(filename);
+    if isempty(ext)
+        filename = fullfile(filepath, [filebase '.mat']);
+    end
+
+    if ~islogical(varargin{end}) || varargin{end} == false
+      % ask before overwritting 
+        if exist(filename,'file')
+            disp(['File ' filename ' exists,' ])
+            userans = input(['Do you want to overwrite (y/N)? '],'s');
+            if strcmpi(userans,'y')
+                disp(['Saving to  ' filename]);
+            else
+                disp(['Did not save ' filename])
+                return
+            end
+        else
+            display(['Saving to  ' filename]);
+        end
+    end
+    if islogical(varargin{end})
+      varargin = varargin(1:end-1); 
+    end
+
+
+    %% ask before rewritting the current file 
+    if ~exist(filepath,'dir') && ~isempty(filepath)
+        mkdir(filepath); 
+    end
+
+
+
+
+
+  varargin = unique(varargin); 
+
+  % Extract the variable values
+  vars = cell(size(varargin));
+  for i = 1:numel(vars)
+    vars{i} = evalin('caller', varargin{i});
+  end
+  
+  % Separate numeric arrays from the rest
+  isnum = cellfun(@(x) isa(x, 'numeric'), vars);
+  % remove -struct and -v7 options from the string 
+  isopt = cellfun(@(x)strcmp(x(1),'-'), varargin);
+
+
+
+    
+    
+    
+  create_dummy = false;
+  if all(isnum)
+    % Save a dummy variable, just to create the file
+    dummy = 0; %#ok<NASGU>
+    save(filename, '-v7.3', 'dummy');
+    create_dummy = true;
+  else
+    s = struct;
+    for i = 1:numel(isnum)
+      if ~isnum(i) && ~isopt(i)
+        s.(varargin{i}) = vars{i};
+      end
+    end
+    save(filename, '-v7.3', '-struct', 's');
+  end
+  
+  % Delete the dummy, if necessary, just in case the user supplied a
+  % variable called dummy
+  if create_dummy
+    fid = H5F.open(filename,'H5F_ACC_RDWR','H5P_DEFAULT');
+    H5L.delete(fid,'dummy','H5P_DEFAULT');
+    H5F.close(fid);
+  end
+  
+  % Save all numeric variables
+  for i = 1:numel(isnum)
+    if ~isnum(i) || isopt(i)
+      continue
+    end
+    % always assume that single precision is enough for data saving 
+    if isa(vars{i}, 'double')
+        vars{i} = single(vars{i});
+    end
+
+    varname = ['/' varargin{i}];
+    if numel(vars{i}) == 0
+        warning('Skipping empty variable %s', varname)
+        continue
+    else
+        if isreal(vars{i})
+            h5create(filename, varname, size(vars{i}), 'DataType', class(vars{i}));
+            h5write(filename, varname, vars{i});
+        else
+            h5create(filename, [varname,'_r'], size(vars{i}), 'DataType', class(vars{i}));
+            h5create(filename, [varname,'_i'], size(vars{i}), 'DataType', class(vars{i}));
+            h5write(filename, [varname,'_r'], real(vars{i}));
+            h5write(filename, [varname,'_i'], imag(vars{i}));       
+        end
+    end
+end
+  
+  
+  
+  
+  
+  
+  
+  
+  
+end
+

+io/sendSMS.m

@@ -0,0 +1,73 @@
+% SENDSMS send message to given phone number(s)
+% ** recipient          phone number(s);
+% ** subject                subject
+% ** msg                message
+% Written by YJ based on PSI code 
+% *optional*
+% ** binary_path        path to sendmail binary file
+% ** server             server address
+% ** sleep              sleep time between messages
+% ** logfile            logfile+path to keep track of past activities
+%
+% EXAMPLES
+%       sendSMS('0041123456789', 'done!');
+%       sendSMS({'076123456', '076987654'}, 'done!')
+%       sendSMS({'076123456', '076987654'}, {'message1', 'message2'})
+%       sendSMS({'076123456', '076987654'}, 'done!', 'sleep', 3600)
+%
+%
+
+function sendSMS(number, subject, msg)
+
+% =========================================================================
+% YOU NEED TO TYPE IN YOUR OWN EMAIL AND PASSWORDS:
+mail = 'yijiang.argonne@gmail.com';    %Your GMail email address
+password = 'sit_down_reiner';          %Your GMail password
+% =========================================================================
+%subject = '';
+% Format the phone number to 10 digit without dashes
+number = strrep(number, '-', '');
+if length(number) == 11 && number(1) == '1';
+    number = number(2:11);
+end
+carrier = 'att';
+% Information found from
+% http://www.sms411.net/2006/07/how-to-send-email-to-phone.html
+switch strrep(strrep(lower(carrier),'-',''),'&','')
+    case 'alltel';    emailto = strcat(number,'@message.alltel.com');
+    case 'att';       emailto = strcat(number,'@txt.att.net');
+    case 'boost';     emailto = strcat(number,'@myboostmobile.com');
+    case 'cingular';  emailto = strcat(number,'@cingularme.com');
+    case 'cingular2'; emailto = strcat(number,'@mobile.mycingular.com');
+    case 'nextel';    emailto = strcat(number,'@messaging.nextel.com');
+    case 'sprint';    emailto = strcat(number,'@messaging.sprintpcs.com');
+    case 'tmobile';   emailto = strcat(number,'@tmomail.net');
+    case 'verizon';   emailto = strcat(number,'@vtext.com');
+    case 'virgin';    emailto = strcat(number,'@vmobl.com');
+end
+
+%% Set up Gmail SMTP service.
+% Note: following code found from
+% http://www.mathworks.com/support/solutions/data/1-3PRRDV.html
+% If you have your own SMTP server, replace it with yours.
+
+% Then this code will set up the preferences properly:
+setpref('Internet','E_mail',mail);
+setpref('Internet','SMTP_Server','smtp.gmail.com');
+setpref('Internet','SMTP_Username',mail);
+setpref('Internet','SMTP_Password',password);
+
+% The following four lines are necessary only if you are using GMail as
+% your SMTP server. Delete these lines wif you are using your own SMTP
+% server.
+props = java.lang.System.getProperties;
+props.setProperty('mail.smtp.auth','true');
+props.setProperty('mail.smtp.starttls.enable','true');
+props.setProperty('mail.smtp.socketFactory.class', 'javax.net.ssl.SSLSocketFactory');
+props.setProperty('mail.smtp.socketFactory.port','465');
+
+%% Send the email
+sendmail(emailto,subject,msg)
+
+end
+

+io/spec_help.m

@@ -0,0 +1,137 @@
+% [] = spec_help(m_file_name)
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [] = spec_help(m_file_name)
+import io.*
+
+  switch m_file_name
+    case 'spec_read'
+      fprintf(['usage: spec_read(specDatFile)\n' ...
+        '       spec_read(specDatFile,varargin)\n' ...
+        'where ''varagin'' is in ''name''-''value''pairs\n' ...
+        'the output is a structure or a cell of structures\n\n']);
+      name = {'ScanNr', ...
+        'Burst', 'MultExposure', ...
+        'OutPut','Cell', ...
+        'PilatusMask','PilatusPath', ...
+        'ValidMask', ...
+        'UnhandledParError'};
+    case 'spec_plot'
+      fprintf(['usage: spec_plot(specDatFile)\n' ...
+        '       spec_plot(specDatFile,varargin)\n' ...
+        'where ''varagin'' is in ''name''-''value''pairs\n' ...
+        'the output is a cell containing graphics handles\n\n']);
+      name = {'ScanNr', ...
+        'Burst', 'MultExposure', ...
+        'Counter','FigNo','Sleep', ...
+        'PilatusMask','PilatusPath', ...
+        'ValidMask', ...
+        'UnhandledParError'};
+    otherwise
+      fprintf('unknown ''M-file''\n')
+  end
+  outputstr = cell(1,numel(name)+1);
+  outputstr{1} = {'name','value','note',''};
+  filler = 4*ones(1,2);
+  for ii=1:numel(name)
+    note2 = cell(0);
+    switch name{ii}
+      case 'Burst'
+        value = '<pos. integer>';
+        note  = 'number of bursts per point';
+      case 'MultExp'
+        value = '<pos. integer>';
+        note  = 'number of spec-controlled exposures per point';
+      case 'Cell'
+        value = '[01]';
+        note  = 'to force the output to be a cell even in case of a single scan.';
+      case 'Counter'
+        value = '<spec counter>';
+        note  = 'counter whose scalar output is to be displayed';
+      case 'FigNo'
+        value = '<handle>';
+        note  = 'handle to determine graphical output';
+      case 'PilatusMask'
+        value = '<mask>';
+        note  = 'mask (including wild cards) how a Pilatus data file will be named';
+      case 'PilatusPath'
+        value = '<path>';
+        note  = 'path in which to expect the corresponding Pilatus data files';
+      case 'OutPut'
+        value = '[''meta''|''counter''|''motor'']';
+        note  = 'what part of the spec data file is being output';
+        note2 = {'These parameters can be combined in a cell.', ...
+          'If preceeded by ''-'', the corresponding output is suppressed.' ...
+          'If preceeded by ''+'', the corresponding output is enabled.' ...
+          'Without prefix, only the output corresponding to ''value'' is enabled.'};
+      case 'ScanNr'
+        value = '<integer>';
+        note  = 'indicating the scan number to be read\n';          
+        note2 = {'In case of duplicate scan numbers, the last one is used only.', ...
+          'In case of negative input, scans are counted backwords from the end of the  data file.'};
+      case 'Sleep'
+        value = '<in seconds>';
+        note = 'delay in seconds between updating subsequent plots';
+      case 'UnhandledParError'
+        value = '[01]';
+        note  = 'if an error occurs if not all named parameters have been handled.';
+      case 'ValidMask'
+        value = '<path>';
+        note  = 'valid pixel mask';
+      otherwise
+        fprintf('unknown argument ''%s''\n', ...
+          name{ii});
+    end
+    outputstr{ii+1} = {name{ii}, value, note, note2};
+    if (numel(name{ii}) > filler(1))
+      filler(1) = numel(name{ii});
+    end
+    if (numel(value) > filler(2))
+      filler(2) = numel(value);
+    end
+  end
+  outputformat = {sprintf('%%-%ds %%-%ds -> %%s\\n', filler), ...
+    sprintf('%%-%ds %%s\\n', sum(filler)+4)};
+  for ii=1:numel(outputstr)
+    fprintf(outputformat{1},outputstr{ii}{1:3})
+    if (~isempty(outputstr{ii}{4}))
+      for jj=1:numel(outputstr{ii}{4})
+        fprintf(outputformat{2},'',outputstr{ii}{4}{jj})        %#ok<CTPCT>
+      end
+    end
+  end
+  fprintf('\n');
+end

+io/spec_read.m

@@ -0,0 +1,526 @@
+% function [varargout] = spec_read(specDatFile, varargin)
+% call 'spec_read()' for further help
+%
+% please reports bugs, problems, suggestions for improvements to:
+% CXS group
+%
+% the case of duplicate scannumbers in the spec file, and how to
+%   address them remains to be implemented
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [varargout] = spec_read(specDatFile, varargin)
+import io.*
+import utils.find_files
+
+% check minimum number of input arguments
+  if(~isunix)
+    error(['The function ''spec_read'' relies on various system calls,\n' ...
+      '  currently available only on UNIX-like systems\n']);
+  end
+  if (nargin<1) 
+    spec_help(mfilename);
+    error('At least the spec data filename has to be specified as input parameter.')
+  end
+  
+% accept cell array with name/value pairs as well
+  no_of_in_arg = nargin;
+  if (nargin == 2)
+    if (isempty(varargin))
+      % ignore empty cell array
+      no_of_in_arg = no_of_in_arg -1;
+    else
+      if (iscell(varargin{1}))
+        % use a filled one given as first and only variable parameter
+        varargin = varargin{1};
+        no_of_in_arg = 1 + length(varargin);
+      end
+    end
+  end
+% check number of input arguments
+  if (rem(no_of_in_arg,2) ~= 1)
+      error('The optional parameters have to be specified as ''name'',''value'' pairs');
+  end
+  
+% here the default parameters  
+  scannr     = -1;
+  output_arg = {'meta','motor','counter'};
+  unhandled_par_error = 1;
+  force_cell = 0;
+% other parameters
+  burstn0 = 1;
+  mexpn0  = 1;
+  creader = 1;
+  orchestraPath = [];
+  specReadOptions = [];
+  
+% parse the variable input arguments
+  vararg = cell(0,0);
+  for jj = 1:2:length(varargin)
+    name = varargin{jj};
+    value = varargin{jj+1};
+    switch name
+      case 'ScanNr'
+        if (~all(value==round(value)) || any(value==0))
+          error('The value of ''%s'' needs to be a (vector of) integer(s) not equal to zero', name)
+        end
+        scannr = value;
+      case 'Burst'
+        if (~all(value==round(value)) || any(value<=0))
+          error('The value of ''%s'' needs to be a positive integer', name)
+        end
+        burstn0 = max(1,value);
+        mexpn0 = 1;
+      case 'MultExp'
+        if (~all(value==round(value)) || any(value<=0))
+          error('The value of ''%s'' needs to be a positive integer', name)
+        end
+        burstn0 = 1;
+        mexpn0 = max(1,value);
+      case 'OutPut'
+        output_arg = set_TextFlag(output_arg,value);
+      case 'PilatusMask'
+        pilatusMask = value;
+        output_arg = set_TextFlag(output_arg,'+pilatus');
+      case 'PilatusDir'
+        pilatusDir0 = value;
+        output_arg = set_TextFlag(output_arg,'+pilatus');
+      case 'Cell'
+        force_cell = value;
+      case 'CReader'
+        creader = value;
+      case 'Orchestra'
+        orchestraPath = value;
+      case 'UnhandledParError'
+        unhandled_par_error = value;
+      otherwise
+        vararg{end+1} = name;                                   %#ok<AGROW>
+        vararg{end+1} = value;                                  %#ok<AGROW>
+    end
+  end
+    
+  specDatFile = beamline.find_specDatFile(specDatFile);
+  if creader
+      try
+          if ~isempty(orchestraPath)
+              specReadOptions = [specReadOptions ' --orchestra ' orchestraPath];
+          end
+          sysCall = [ fullfile(fileparts(mfilename('fullpath')), './spec_reader/spec_reader') ' -s ' specDatFile ' --scanNr ' strjoin(arrayfun(@(x) num2str(x),scannr,'UniformOutput',false),',') specReadOptions];
+          [stat, out] = system(sysCall);
+          if stat
+              error(['Fast spec reader failed: ' strtrim(out)])
+          end
+          res = jsondecode([ out ]);
+          if length(res)==1
+              varargout{1} = res;
+          else
+              varargout{1} = num2cell(res)';
+          end
+      catch ME
+          if creader
+              warning('Fast spec reader failed.')
+              disp(ME)
+              creader = 0;
+          end
+      end
+  end
+  if ~creader
+      % reading the spec file for the scans
+      if ~exist(specDatFile, 'file')
+          error('Spec file was not found in %s', specDatFile)
+      end
+      [scans,scanline] = spec_readscan(specDatFile, scannr);
+      
+      % reading the spec file for configuration information
+      [motorc,motorline] = spec_readconfig(specDatFile);
+      motorn = cell(size(scannr));
+      for jj=1:numel(scans)
+          motorn{jj} = motorc{find(motorline<scanline(1),1,'last')};
+      end
+      
+      
+      varargout{1} = cell(size(scans));
+      % copying vararg in order to keep its content for each iteration
+      vararg_1 = vararg;
+      vararg_1{end+1} = 'UnhandledParError';
+      vararg_1{end+1} = 0;
+      for jj=1:numel(scans)
+          scanstr = scans{jj}';
+          arrout = regexp(scanstr{1},' +','split');
+          scannr(jj) = str2double(arrout{2});
+          
+          % get the motors
+          line = ~cellfun(@isempty,regexp(scanstr,'^#P'));
+          arrout = reshape(transpose(strvcat((scanstr(line)))),1,[]);
+          arrout = regexp(arrout,' +|(#P[0-9]+)','split');
+          motorv = str2double(arrout(~cellfun(@isempty,arrout)));
+          
+          % get the counters
+          line = ~cellfun(@isempty,regexp(scanstr,'^#L'));
+          counters = regexp(scanstr{line},' +','split');
+          counters = counters(2:end);
+          
+          % get the data
+          line  = ~cellfun(@isempty,regexp(scanstr,'^[0-9+-]'));
+          data = cellfun(@str2num,scanstr(line),'UniformOutput',false);
+          data = cell2mat(data(:));
+          
+          
+          % get Pilatus information if required
+          if (exist('pilatusMask','var'))
+              if (exist('pilatusDir0','var'))
+                  pilatusDir = find_pilatusDir(pilatusDir0,scannr(jj));
+              else
+                  pilatusDir = '';
+              end
+              
+              [pilatusDir, pilatusName, vararg] = ...
+                  find_files(strcat(pilatusDir,pilatusMask), vararg_1);
+              
+              pilatusInd = zeros(numel(pilatusName),2);
+              for ii=1:numel(pilatusName)
+                  arrout = regexp(pilatusName(ii).name,'[_\.]','split');
+                  pilatusInd(ii,:) = str2double(arrout(end-2:end-1));
+              end
+          end
+          
+          % make a structure
+          scan_structure = struct;          
+          if (any(strcmp(output_arg,'motor')) || ...
+                  any(strcmp(output_arg,'motors')))
+              for ii=1:numel(motorn{jj})
+                  scan_structure.(motorn{jj}{ii}) = motorv(ii);
+              end
+          end
+          if (any(strcmp(output_arg,'meta')))
+              scan_structure.S = scanstr{1};
+              
+              % get the time stamp and convert it to a MATLAB-compatible date string
+              line = ~cellfun(@isempty,regexp(scanstr,'^#D'));
+              datestr = regexp(scanstr{line},' +','split');
+              scan_structure.D = sprintf('%s-%s-%s %s', ...
+                  datestr{4}, ...
+                  datestr{3}, ...
+                  datestr{6}, ...
+                  datestr{5});
+              
+              line = ~cellfun(@isempty,regexp(scanstr,'^#C meta'));
+              meta = scanstr(line);
+              for ii=1:length(meta)
+                  metaEntry = strsplit(strtrim(meta{ii}), ' ');
+                  switch metaEntry{4}
+                      case {'int', 'float'}
+                          converterFunc = @str2double;
+                      case 'string'
+                          converterFunc = @(x) x;
+                  end
+                  
+                  tmp = [];
+                  for jjj=1:str2double(metaEntry{5})
+                      tmp = [tmp converterFunc(metaEntry{5+jjj})];
+                  end
+                  scan_structure.(metaEntry{3}) = tmp;
+              end
+                          
+              
+          end        
+          if (any(strcmp(output_arg,'counter')) || ...
+                  any(strcmp(output_arg,'counters')))
+              if (numel(data)>0)
+                  for ii=1:numel(counters)
+                      scan_structure.(counters{ii}) = data(:,ii);
+                  end
+              end
+          end
+          
+          if (any(strcmp(output_arg,'pilatus')))
+              if (burstn0>1)
+                  burstn = burstn0;
+              elseif (isfield(scan_structure,'burstn'))
+                  burstn = max(ones(size(scan_structure.burstn)),scan_structure.burstn);
+              else
+                  burstn = burstn0;
+              end
+              if (mexpn0>1)
+                  mexpn = mexpn0;
+              elseif (isfield(scan_structure,'mexpn'))
+                  mexpn = max(ones(size(scan_structure.burstn)),scan_structure.mexpn);
+              else
+                  mexpn = mexpn0;
+              end
+              
+              if (numel(burstn)==1 && numel(mexpn)==1)
+                  % these pseudo motors shouldn't be scanned and are
+                  %   therefore presumably constant
+                  Pilatus = cell(size(data,1),1);
+                  scanInd = [transpose(floor((0:size(data,1)-1)/mexpn)), ...
+                      transpose(mod(0:size(data,1)-1,mexpn))];
+                  for ii=1:size(data,1)
+                      if (burstn<=1)
+                          indices = all(ones(size(pilatusName))'*scanInd(ii,:)==pilatusInd,2);
+                      else
+                          indices = all(ones(size(pilatusName))'*scanInd(ii,1)==pilatusInd(:,1),2);
+                      end
+                      Pilatus{ii} = {pilatusName(indices).name};
+                      if (numel(indices)<burstn)
+                          % accounting for missing frames
+                          % untested, as of Dec 16
+                          tmp = Pilatus{ii};
+                          Pilatus{ii} = cell(1,burstn);
+                          for kk=1:numel(tmp)
+                              Pilatus{ii}{pilatusInd(2,kk)} = tmp{kk};
+                          end
+                      end
+                  end
+              end
+              scan_structure.Pilatus = Pilatus;
+              scan_structure.PilatusDir = pilatusDir;
+          end
+          
+          % append orchestra data
+          if ~isempty(orchestraPath)
+              [config, data] = readOrchestraFile(orchestraPath, scannr(jj));
+              for oidx=1:length(config)
+                  scan_structure.(config{oidx}) = data{oidx};
+              end
+          end
+          varargout{1}{jj} = scan_structure;
+      end
+      if ((unhandled_par_error) && (~isempty(vararg)))
+          vararg                                                      %#ok<NOPRT>
+          error('Not all named parameters have been handled.');
+      end
+      if (numel(varargout{1})==1 && ~force_cell)
+          varargout{1} = varargout{1}{1};
+      end
+      varargout{2} = vararg;
+      return
+  end
+end
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+function [varargout] = spec_readscan(specDatFile, scannr)
+import io.*
+import utils.*
+% check minimum number of input arguments
+  if (nargin<2) 
+    spec_read_help(mfilename);
+    error('usage ''spec_readscan(specDatFile, scannr)''');
+  end
+  
+  cmd = sprintf('grep -n ''#S '' %s', specDatFile);
+  [~,sysout] = system(cmd);
+  arrout = regexp(sysout,'[:\n]','split');
+  arrout = arrout(~cellfun(@isempty,arrout));
+  arrout = transpose(reshape(arrout,2,[]));
+  [s1,~] = size(arrout);
+  
+  if (any(scannr>0))
+    tmp = regexp(arrout(:,2),' +','split');
+    scanID = zeros(size(tmp));
+    for ii=1:numel(tmp)
+      scanID(ii) = str2double(tmp{ii}{2});
+    end
+  end
+  
+% creating a look up table where the scans begin and end
+  lineI = zeros(size(scannr));
+  scans = cell(size(scannr));
+  for ii=1:numel(scannr)
+    if (scannr(ii)<0)
+      if (-scannr(ii)>size(arrout,1))
+        error('There does not seem to be sufficiently many scans in ''%s''', ...
+          specDatFile);
+      end
+      tmp = regexp(arrout(end+scannr(ii)+1,:),' +','split');
+      lineI(ii) = str2double(tmp{1});
+      if (scannr(ii) < -1)
+        tmp = regexp(arrout(end+scannr(ii)+2,:),' +','split');
+        lineF = str2double(tmp{1})-1;
+      else
+        cmd = sprintf('wc -l %s', specDatFile);
+        [~,sysout] = system(cmd);
+        lineF = str2double(regexp(sysout,'[0-9]+ ','match'));
+      end
+      scannr(ii) = str2double(tmp{2}(2));
+    else
+      tmp = find((scannr(ii)==scanID),1,'last');
+      try 
+          verbose(2,'reading ''%s''',arrout{tmp,2});
+      catch
+          fprintf('reading ''%s''\n',arrout{tmp,2});
+      end
+
+      if (isempty(tmp))
+        fprintf('Do not find scan #S %d in ''%s''.\n', ...
+          scannr(ii),specDatFile);
+        continue
+      end
+      lineI(ii) = str2double(arrout(tmp,1));
+      if (tmp<s1)
+        lineF = str2double(arrout(tmp+1,1))-1;
+      else
+        cmd = sprintf('wc -l %s', specDatFile);
+        [~,sysout] = system(cmd);
+        lineF = str2double(regexp(sysout,'[0-9]+ ','match'));
+      end
+    end
+
+    cmd = sprintf('head -n +%d %s | tail -n -%d', ...
+      lineF, specDatFile, lineF+1-lineI(ii));
+
+  % read the scan
+    [~,sysout] = system(cmd);
+    scans{ii} = regexp(sysout,'\n','split');
+    scans{ii} = scans{ii}(~cellfun(@isempty,scans{ii}));
+  end
+  varargout{1} = scans;
+  varargout{2} = lineI;
+end
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+function [motors,linenr] = spec_readconfig(specDatFile)
+import io.*
+import utils.find_files
+  % finding the end of the last config
+  cmd = sprintf('grep -nE ''^#O[0-9]+ '' %s', ...
+      specDatFile);
+  [~,sysout] = system(cmd);
+  arrout = regexp(sysout,'[:\n]','split');
+  arrout = arrout(~cellfun(@isempty,arrout));
+  arrout = transpose(reshape(arrout,2,[]));
+  [s1,~] = size(arrout);
+  
+  % devide this collection of configuration information into single chuncks
+  linenr = '';
+  motors = cell(0);
+  for ii=1:s1
+    if (~isempty(regexp(arrout{ii,2},'^#O0+','match')))
+      if (isempty(linenr))
+        linenr = str2double(arrout{ii,1});
+      else
+        linenr = vertcat(linenr, ...
+          str2double(arrout{ii,1}));
+      end
+      motors{numel(linenr)} = regexp(arrout{ii,2},'(^#O[0-9]+ *)| *','split');
+    else
+      motors{numel(linenr)} = horzcat(motors{numel(linenr)}, ...
+        regexp(arrout{ii,2},'(^#O[0-9]+ *)| *','split'));
+    end
+  end
+  for ii=1:numel(linenr)
+    motors{ii} = motors{ii}(~cellfun(@isempty,motors{ii}));
+  end
+end
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+function pilatusDir = find_pilatusDir(pilatusDir,scannr)
+import io.*
+import utils.find_files
+  while (any(exist(pilatusDir,'dir') == [2,7]))
+% if the variable pilatusDir is not a complete path to a file
+%   try to guess where a spec data file can be found, by
+%   - look for directories called 'spec' or 'dat-files'
+%   - look for files called '*.dat'
+%   - take the newest one
+    compare_str = pilatusDir;
+    fname = dir(pilatusDir);
+    if (exist(pilatusDir,'dir'))
+      if (pilatusDir(end) ~= '/')
+        pilatusDir = strcat(pilatusDir,'/');
+      end
+      
+      for ii=1:numel(fname)
+        if (strcmp(fname(ii).name,sprintf('S%02d000-%02d999',floor(scannr/1e3),floor(scannr/1e3))) || ...
+            strcmp(fname(ii).name,sprintf('S%05d',scannr)))
+          pilatusDir = strcat(pilatusDir,fname(ii).name);
+          fname = [];
+          break;
+        end
+      end
+      for ii=1:numel(fname)
+        if (strcmp(fname(ii).name,'pilatus'))
+          pilatusDir = strcat(pilatusDir,fname(ii).name);
+          break;
+        end
+      end
+    end
+    if (strcmp(pilatusDir,compare_str))
+      break
+    end
+  end
+end
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+function [flags] = set_TextFlag(flags,value)
+import io.*
+import utils.find_files
+  if (~iscell(value))
+    value = {value};
+  end
+  for ii=1:numel(value)
+    value_i = value{ii};
+    if (regexp(value_i,'^[a-zA-Z]+$'))
+      flags = {value_i};
+      if (ii+1<=numel(value))
+        value{ii+1} = regexprep(value{ii+1},'^([a-zA-Z]*)','+$1');
+      end
+    elseif (regexp(value_i,'^\+[a-zA-Z]+$'))
+      flags = unique([flags,value_i(2:end)]);
+      if (ii+1<=numel(value))
+        value{ii+1} = regexprep(value{ii+1},'^([a-zA-Z]*)','+$1');
+      end
+    elseif (regexp(value_i,'^-[a-zA-Z]+$'))
+      flags = flags(~strcmp(flags,value_i(2:end)));
+      if (ii+1<=numel(value))
+        value{ii+1} = regexprep(value{ii+1},'^([a-zA-Z]*)','-$1');
+      end
+    end
+  end
+end
+
+function [config, data] = readOrchestraFile(orchestraPath, scanNr)
+config = [];
+data = [];
+fname = fullfile(orchestraPath, sprintf('scan_%05d.dat', scanNr));
+if ~exist(fname, 'file')
+    warning('Could not find any orchestra file for scan %d', scanNr);
+    return;
+end
+fid = fopen(fname);
+fgetl(fid);
+config = strsplit(fgetl(fid), ' ');
+
+data=textscan(fid,repmat('%f ', 1, length(config)));
+
+end

+io/speread.m

@@ -0,0 +1,222 @@
+% Call function without arguments for instructions on how to use it
+
+% Filename: $RCSfile: speread.m,v $
+%
+% $Revision: 1.1 $  $Date: 2008/06/10 17:05:14 $
+% $Author:  $
+% $Tag: $
+%
+% Description:
+% Macro for reading SPE CCD data files written by WinView from 
+% Roper Scientific / PI Acton
+%
+% Note:
+% Call without arguments for a brief help text.
+%
+% Dependencies:
+% - image_read_set_default
+% - fopen_until_exists
+% - get_hdr_val
+%
+%
+% history:
+%
+% May 9th 2008: 1st version
+%
+
+%*-----------------------------------------------------------------------*
+%|                                                                       |
+%|  Except where otherwise noted, this work is licensed under a          |
+%|  Creative Commons Attribution-NonCommercial-ShareAlike 4.0            |
+%|  International (CC BY-NC-SA 4.0) license.                             |
+%|                                                                       |
+%|  Copyright (c) 2017 by Paul Scherrer Institute (http://www.psi.ch)    |
+%|                                                                       |
+%|       Author: CXS group, PSI                                          |
+%*-----------------------------------------------------------------------*
+% You may use this code with the following provisions:
+%
+% If the code is fully or partially redistributed, or rewritten in another
+%   computing language this notice should be included in the redistribution.
+%
+% If this code, or subfunctions or parts of it, is used for research in a 
+%   publication or if it is fully or partially rewritten for another 
+%   computing language the authors and institution should be acknowledged 
+%   in written form in the publication: “Data processing was carried out 
+%   using the “cSAXS matlab package” developed by the CXS group,
+%   Paul Scherrer Institut, Switzerland.” 
+%   Variations on the latter text can be incorporated upon discussion with 
+%   the CXS group if needed to more specifically reflect the use of the package 
+%   for the published work.
+%
+% A publication that focuses on describing features, or parameters, that
+%    are already existing in the code should be first discussed with the
+%    authors.
+%   
+% This code and subroutines are part of a continuous development, they 
+%    are provided “as they are” without guarantees or liability on part
+%    of PSI or the authors. It is the user responsibility to ensure its 
+%    proper use and the correctness of the results.
+
+function [frame,vararg_remain] = speread(filename,varargin)
+import io.*
+import utils.fopen_until_exists
+import utils.get_hdr_val
+
+% 0: no debug information
+% 1: some feedback
+% 2: a lot of information
+debug_level = 0;
+
+% initialize return argument
+frame = struct('header',[], 'data',[]);
+
+% check minimum number of input arguments
+if (nargin < 1)
+    image_read_sub_help(mfilename,'SPE');
+    error('At least the filename has to be specified as input parameter.');
+end
+
+% accept cell array with name/value pairs as well
+no_of_in_arg = nargin;
+if (nargin == 2)
+    if (isempty(varargin))
+        % ignore empty cell array
+        no_of_in_arg = no_of_in_arg -1;
+    else
+        if (iscell(varargin{1}))
+            % use a filled one given as first and only variable parameter
+            varargin = varargin{1};
+            no_of_in_arg = 1 + length(varargin);
+        end
+    end
+end
+
+% check number of input arguments
+if (rem(no_of_in_arg,2) ~= 1)
+    error('The optional parameters have to be specified as ''name'',''value'' pairs');
+end
+
+% set default values for the variable input arguments and parse the named
+% parameters: 
+vararg = cell(0,0);
+for ind = 1:2:length(varargin)
+    name = varargin{ind};
+    value = varargin{ind+1};
+    switch name
+        otherwise
+            % pass further arguments on to fopen_until_exists
+            vararg{end+1} = name;
+            vararg{end+1} = value;
+    end
+end
+
+
+% expected maximum length for the text header
+max_header_length = 4100;
+
+% try to open the data file
+if (debug_level >= 1)
+    fprintf('Opening %s.\n',filename);
+end
+[fid,vararg_remain] = fopen_until_exists(filename,vararg);
+if (fid < 0)
+    return;
+end
+
+% read all data at once
+[fdat,fcount] = fread(fid,'uint8=>uint8');
+
+% close input data file
+fclose(fid);
+if (debug_level >= 2)
+    fprintf('%d data bytes read\n',fcount);
+end
+
+% some sanity checks
+if (fdat(4099:4100) ~= [ 85; 85 ])
+    error('%s: no header end signature found',filename);
+    return;
+end
+
+
+% return some selected information from the header as lines of a cell array
+% and extract some variables needed for the data extraction
+% version 2.5 is assumed, otherwise some values may be wrong
+frame.header = cell(0,0);
+xdim = double(typecast(fdat(43:44),'uint16'));
+frame.header{end+1} = sprintf('xdim %.0f',xdim);
+
+ydim = double(typecast(fdat(657:658),'uint16'));
+frame.header{end+1} = sprintf('ydim %.0f',ydim);
+
+exposure_sec = double(typecast(fdat(11:14),'single'));
+frame.header{end+1} = sprintf('exposure %.3f',exposure_sec);
+
+frame.header{end+1} = sprintf('xDimDet %d',typecast(fdat(7:8),'uint16'));
+frame.header{end+1} = sprintf('yDimDet %d',typecast(fdat(19:20),'uint16'));
+
+frame.header{end+1} = sprintf('date %s',char(fdat(21:30)'));
+frame.header{end+1} = sprintf('ExperimentTimeLocal %s:%s:%s',...
+    char(fdat(173:174)'),char(fdat(175:176)'),char(fdat(177:178)'));
+
+data_type = fdat(109);
+frame.header{end+1} = sprintf('datatype %d',data_type);
+switch (data_type)
+    case 0
+        bytes_per_pixel = 4;
+        cast_type = 'single';
+    case 1
+        bytes_per_pixel = 4;
+        cast_type = 'int32';
+    case 2
+        bytes_per_pixel = 2;
+        cast_type = 'int16';
+    case 3
+        bytes_per_pixel = 2;
+        cast_type = 'uint16';
+    otherwise
+        error('unknown data type %d',data_type);
+end
+
+frame.header{end+1} = sprintf('BackGrndApplied %d',typecast(fdat(151:152),'uint16'));
+
+frame.header{end+1} = sprintf('gain %d',typecast(fdat(199:200),'uint16'));
+
+% frame.header{end+1} = sprintf('comments %s',char(fdat(201:600)'));
+
+% number of accumulations
+frame.header{end+1} = sprintf('lavgexp %ld',typecast(fdat(669:672),'uint32'));
+
+frame.header{end+1} = sprintf('ReadoutTime %f',typecast(fdat(673:676),'single'));
+
+frame.header{end+1} = sprintf('sw_version %s',char(fdat(689:703)'));
+
+frame.header{end+1} = sprintf('type %d',typecast(fdat(705:706),'uint16'));
+
+frame.header{end+1} = sprintf('flatFieldApplied %d',typecast(fdat(707:708),'uint16'));
+
+num_frames = double(typecast(fdat(1447:1450),'uint32'));
+frame.header{end+1} = sprintf('NumFrames %.0f',num_frames);
+
+file_header_ver = double(typecast(fdat(1993:1996),'single'));
+frame.header{end+1} = sprintf('file_header_ver %.2f',file_header_ver);
+
+
+% extract the data frames
+data_bytes = bytes_per_pixel * xdim * ydim * num_frames;
+if (fcount-4100 ~= data_bytes)
+    error('%d bytes expected, %d found',data_bytes,fcount-4100);
+end
+frame.data = reshape(typecast(fdat(4101:end),cast_type),...
+    xdim, ydim, num_frames );
+
+% conversion to standard view on PI-SCX4300 data at the SLS/cSAXS beamline
+% if (~original_orientation)
+%     frame.data = permute(frame.data,[2 1 3]);
+% end
+
+% return as double precision floating point
+frame.data = double(frame.data);
+
+