mirror of
https://github.com/c-sooyoung/fold_slice.git
synced 2026-09-17 22:59:07 +09:00
initial commit
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,302 @@
|
||||
/*
|
||||
* Copyright (c) 2010,2011 Joshua V Dillon
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or
|
||||
* without modification, are permitted provided that the
|
||||
* following conditions are met:
|
||||
* * Redistributions of source code must retain the above
|
||||
* copyright notice, this list of conditions and the
|
||||
* following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above
|
||||
* copyright notice, this list of conditions and the
|
||||
* following disclaimer in the documentation and/or other
|
||||
* materials provided with the distribution.
|
||||
* * Neither the name of the author nor the names of its
|
||||
* contributors may be used to endorse or promote products
|
||||
* derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY JOSHUA V DILLON ''AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
||||
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JOSHUA
|
||||
* V DILLON BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
|
||||
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
*/
|
||||
|
||||
#ifndef sharedmatrix_hxx
|
||||
#define sharedmatrix_hxx
|
||||
|
||||
/* Uses mxArray_tag as defined in matrix.h */
|
||||
/*#define COMPLIANTMODE*/
|
||||
|
||||
/* Verbose outputs */
|
||||
// #define DEBUG
|
||||
|
||||
/* This copies memory across, defeating the purpose of this function but useful for testing */
|
||||
// #define SAFEMODE
|
||||
|
||||
/*
|
||||
* sharedmatrix.h
|
||||
*
|
||||
* This Matlab Mex program allows you to share matlab cell arrays and matrices
|
||||
* between different Maatlab processes. It has four functions:
|
||||
* 1) Clone:
|
||||
* Serialize a 2D cell array of 2D non-/sparse matrices or a 2D
|
||||
* non-/sparse matrix to shared memory.
|
||||
* 2) Attach:
|
||||
* "Reconstitute" the shared data into the appropriate Matlab object using
|
||||
* shallow copying.
|
||||
* 3) Detach:
|
||||
* Remove the shallow references and detach the shared memory from the
|
||||
* Matlab data space.
|
||||
* 4) Free:
|
||||
* Mark the shared memory for destruction.
|
||||
*
|
||||
* Written by Joshua V Dillon
|
||||
* August 27, 2010
|
||||
*
|
||||
* Additional Contributors"
|
||||
* Andrew Smith - Februrary 18, 2011
|
||||
*
|
||||
* Revision History:
|
||||
* Sep. 9. 2010 Corrected the mishandling of sparse logical matrices.
|
||||
* Apr. 8, 2011 - Merged Andrew Smith's struct code
|
||||
* - Added Andrew Smith's windows/boost contribution
|
||||
*/
|
||||
|
||||
/*
|
||||
* This code can be compiled from within Matlab or command-line, assuming the
|
||||
* system is appropriately setup. To compile, invoke:
|
||||
*
|
||||
* For 32-bit machines:
|
||||
* mex -O -v sharedmatrix.c
|
||||
* For 64-bit machines:
|
||||
* mex -largeArrayDims -O -v sharedmatrix.c
|
||||
*
|
||||
*/
|
||||
|
||||
/*
|
||||
* Programmer's Notes:
|
||||
*
|
||||
* MEX C API:
|
||||
* http://www.mathworks.com/access/helpdesk/help/techdoc/apiref/bqoqnz0.html
|
||||
*
|
||||
* Testing:
|
||||
*
|
||||
x=sparse((rand(3,4)<.5).*rand(3,4));
|
||||
x=cell(2,2);x{1}=rand(3,4);x{4}=sparse((rand(2,4)>.5).*rand(2,4));
|
||||
shmsiz=sharedmatrix('clone',12345,x)
|
||||
y=sharedmatrix('attach',12345)
|
||||
sharedmatrix('detach',12345,y)
|
||||
*
|
||||
*/
|
||||
|
||||
/*
|
||||
* Possibly useful information/related work:
|
||||
*
|
||||
* http://www.mathworks.com/matlabcentral/fileexchange/24576
|
||||
* http://www.mathworks.in/matlabcentral/newsreader/view_thread/254813
|
||||
* http://www.mathworks.com/matlabcentral/newsreader/view_thread/247881#639280
|
||||
*
|
||||
* http://groups.google.com/group/comp.soft-sys.matlab/browse_thread/thread/c241d8821fb90275/47189d498d1f45b8?lnk=st&q=&rnum=1&hl=en#47189d498d1f45b8
|
||||
* http://www.mk.tu-berlin.de/Members/Benjamin/mex_sharedArrays
|
||||
*/
|
||||
|
||||
/* Possibily useful undocumented functions (see links at end for details): */
|
||||
/* extern mxArray *mxCreateSharedDataCopy(const mxArray *pr); */
|
||||
/* extern bool mxUnshareArray(const mxArray *pr, const bool noDeepCopy); */
|
||||
/* extern mxArray *mxUnreference(const mxArray *pr); */
|
||||
|
||||
#ifdef DEBUG
|
||||
#ifdef __STDC__
|
||||
/*#define DEBUG_HEADER "[%s:% 4d] % 15s(): "*/
|
||||
#define DEBUG_HEADER "\033[1;34;40m[%s:% 4d] \033[1;31;40m% 15s(): \033[0m"
|
||||
#define mydebug(format,args...) \
|
||||
mexPrintf(DEBUG_HEADER format "\n", \
|
||||
__FILE__,__LINE__,__FUNCTION__,##args)
|
||||
#else
|
||||
#define DEBUG_HEADER "DEBUG: "
|
||||
#define mydebug(format,args...) \
|
||||
mexPrintf(DEBUG_HEADER format "\n",##args)
|
||||
#endif
|
||||
#else
|
||||
#define mydebug(format,args...) (void)0
|
||||
#endif
|
||||
|
||||
#ifdef COMPLIANTMODE
|
||||
|
||||
/* We will be accessing the mxArray parts directly via the mxArray_tag struct
|
||||
* defined in matrix.h. */
|
||||
#define ARRAY_ACCESS_INLINING
|
||||
#include "matrix.h"
|
||||
|
||||
#else
|
||||
|
||||
struct mxArray_tag {
|
||||
void *reserved;
|
||||
int reserved1[2];
|
||||
void *reserved2;
|
||||
size_t number_of_dims;
|
||||
unsigned int reserved3;
|
||||
struct {
|
||||
unsigned int flag0 : 1;
|
||||
unsigned int flag1 : 1;
|
||||
unsigned int flag2 : 1;
|
||||
unsigned int flag3 : 1;
|
||||
unsigned int flag4 : 1;
|
||||
unsigned int flag5 : 1;
|
||||
unsigned int flag6 : 1;
|
||||
unsigned int flag7 : 1;
|
||||
unsigned int flag7a: 1;
|
||||
unsigned int flag8 : 1;
|
||||
unsigned int flag9 : 1;
|
||||
unsigned int flag10 : 1;
|
||||
unsigned int flag11 : 4;
|
||||
unsigned int flag12 : 8;
|
||||
unsigned int flag13 : 8;
|
||||
} flags;
|
||||
size_t reserved4[2];
|
||||
union {
|
||||
struct {
|
||||
void *pdata;
|
||||
void *pimag_data;
|
||||
void *reserved5;
|
||||
size_t reserved6[3];
|
||||
} number_array;
|
||||
} data;
|
||||
};
|
||||
|
||||
#endif
|
||||
typedef struct mxArray_tag mxArrayHack;
|
||||
|
||||
/* standard mex include; after hack */
|
||||
#include "mex.h"
|
||||
|
||||
/* max length of directive string */
|
||||
#define MAXDIRECTIVELEN 256
|
||||
|
||||
/* these are used for recording structure field names */
|
||||
const char term_char = ';'; /*use this character to terminate a string containing the list of fields. Do this because it can't be in a valid field name*/
|
||||
const size_t align_size = 8; /*the pointer alignment size, so if pdata is a valid pointer then &pdata[i*align_size] will also be. Ensure this is >= 4*/
|
||||
|
||||
/*
|
||||
* The header_t object will be copied to shared memory in its entirety.
|
||||
*
|
||||
* Immediately after each copied header_t will be the matrix data values
|
||||
* [size array, field names, (_at_most_ the four arrays [pr,pi,ir,jc] and in this order)].
|
||||
*
|
||||
* The data_t objects will never be copied to shared memory and serve only
|
||||
* to abstract away mex calls and simplify the deep traversals in matlab.
|
||||
*
|
||||
*/
|
||||
|
||||
typedef struct data data_t;
|
||||
typedef struct header header_t;
|
||||
|
||||
/* structure used to record all of the data addresses */
|
||||
struct data {
|
||||
mwSize *pSize; /* pointer to the size array */
|
||||
void* pr; /* real data portion */
|
||||
void* pi; /* imaginary data portion */
|
||||
mwIndex *ir; /* row indexes, for sparse */
|
||||
/* OR: may also be list of a structures fields,
|
||||
each field name will be seperated by a null
|
||||
character and terminated with a ";" */
|
||||
mwIndex *jc; /* cumulative column counts, for sparse */
|
||||
data_t *child_dat; /* array of children data structures, for cell */
|
||||
header_t *child_hdr; /* array of corresponding children header structures, for cell */
|
||||
};
|
||||
|
||||
/* captures fundamentals of the mxArray */
|
||||
/* In the shared memory the storage order is [header, size array, field_names,
|
||||
* real dat, image data, sparse index r, sparse index c] */
|
||||
struct header {
|
||||
bool isCell;
|
||||
bool isSparse;
|
||||
bool isComplex;
|
||||
bool isStruct;
|
||||
mxClassID classid; /* matlab class id */
|
||||
size_t nDims; /* dimensionality of the matrix. The size array immediately follows the header */
|
||||
size_t elemsiz; /* size of each element in pr and pi */
|
||||
size_t nzmax; /* length of pr,pi */
|
||||
size_t nFields; /* the number of fields. The field string immediately follows the size array */
|
||||
size_t shmsiz; /* size of serialized object (header + size array + field names string) */
|
||||
};
|
||||
|
||||
/* Remove shared memory references to input matrix (in-situ), recursively */
|
||||
/* if needed. */
|
||||
char* deepdetach (mxArray *mxInput);
|
||||
|
||||
/* Shallow copy matrix from shared memory into Matlab form. */
|
||||
size_t shallowrestore (char *shm, mxArray** p_mxInput);
|
||||
|
||||
/* Recursively descend through Matlab matrix to assess how much space its */
|
||||
/* serialization will require. */
|
||||
size_t deepscan (header_t *hdr, data_t *dat, const mxArray* mxInput);
|
||||
|
||||
/* Descend through header and data structure and copy relevent data to */
|
||||
/* shared memory. */
|
||||
void deepcopy (header_t *hdr, data_t *dat, char *shared_mem, bool allocate_only);
|
||||
|
||||
/* Descend through header and data structure and free the memory. */
|
||||
void deepfree (data_t *dat);
|
||||
|
||||
/* Pads the size to something that guarantees pointer alignment. */
|
||||
__inline size_t pad_to_align(size_t size) {
|
||||
if (size % align_size)
|
||||
size += align_size - (size % align_size);
|
||||
return size;
|
||||
}
|
||||
|
||||
/* Function to find the number of bytes required to store all of the */
|
||||
/* field names of a structure */
|
||||
int FieldNamesSize(const mxArray * mxStruct);
|
||||
|
||||
/* Function to copy all of the field names to a character array */
|
||||
/* Use FieldNamesSize() to allocate the required size of the array */
|
||||
/* returns the number of bytes used in pList */
|
||||
int CopyFieldNames(const mxArray * mxStruct, char* pList);
|
||||
|
||||
/* This function finds the number of fields contained within in a string */
|
||||
/* the string is terminated by term_char, a character that can't be in a */
|
||||
/* field name. pBytes is always an aligned number */
|
||||
int NumFieldsFromString(const char* pString, size_t *pfields, size_t* pBytes);
|
||||
|
||||
/* Function to take point a each element of the char** array at a list of */
|
||||
/* names contained in string */
|
||||
/* ppCharArray must be allocated to length num_names */
|
||||
/* names are seperated by null termination characters */
|
||||
/* each name must start on an aligned address (see CopyFieldNames()) */
|
||||
/* e.g. pCharArray[0] = name_1, pCharArray[1] = name_2 ... */
|
||||
/* returns 0 if successful */
|
||||
int PointCharArrayAtString(char ** pCharArray, char* pString, int nFields);
|
||||
|
||||
/* Function to find the bytes in the string starting from the end of the */
|
||||
/* string returns < 0 on error */
|
||||
int BytesFromStringEnd(const char* pString, size_t* pBytes);
|
||||
|
||||
// #ifdef SAFEMODE
|
||||
/* A convenient function for safe assignment of memory to an mxArray */
|
||||
void* safeCopy(void* pBuffer, mwSize Bytes) {
|
||||
void* pSafeBuffer;
|
||||
|
||||
/* ensure Matlab knows it */
|
||||
pSafeBuffer = mxMalloc(Bytes);
|
||||
if (pSafeBuffer != NULL)
|
||||
memcpy(pSafeBuffer, pBuffer, Bytes); /* and copy the data */
|
||||
|
||||
return pSafeBuffer;
|
||||
}
|
||||
// #endif
|
||||
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
%SHAREDMATRIX Allows 2D (cell) matrix to be shared between Matlab processes.
|
||||
% SHAREDMATRIX allows certain Matlab objects (see below) to be shared
|
||||
% between multiple Matlab sessions, provided they have access to the same
|
||||
% shared memory resources, i.e., the processes are on the same physical
|
||||
% system. This program uses shared memory functions specified by POSIX and
|
||||
% will not work in Windows nor any other non-POSIX environment, although it
|
||||
% probably could be compiled under cygwin.
|
||||
%
|
||||
% WARNING: This program manipulates Matlab objects in a manner that is
|
||||
% highly unstable. It is very likely that using this function will cause
|
||||
% Matlab to crash and could potentially lead to data corruption or loss.
|
||||
% This instability should be limited to the Matlab process itself and
|
||||
% should not affect the stability of the system.
|
||||
%
|
||||
% Since this program is more complicated than the typical Matlab function
|
||||
% please read this manual thoroughly! It is organized in a Q/A style to
|
||||
% make referring to it easier.
|
||||
%
|
||||
% --- What exactly can be shared?
|
||||
%
|
||||
% This program allows multiple Matlab sessions to access one copy of the
|
||||
% following Matlab data objects:
|
||||
% - 2D matrix, non-sparse, non-empty
|
||||
% - 2D matrix, sparse, non-empty
|
||||
% - 2D cell array of above and/or empty matrices (at least one non-empty)
|
||||
% - 2D cell array of any of above (recursive)
|
||||
% In principle, the matrices can be of any type, i.e., UINT32, UINT64,
|
||||
% etc., although only the DOUBLE type has been extensively tested. The
|
||||
% matrices can be real or complex.
|
||||
%
|
||||
% --- What is meant by "shared?"
|
||||
%
|
||||
% A "shared object" is a Matlab matrix or cell array with references to a
|
||||
% segment of "shared memory." Shared memory is special in that it can be
|
||||
% made accessible to any program. For purposes of this program the shared
|
||||
% data object should generally be regarded as read-only, however, by design
|
||||
% there is no mechanism to force this.
|
||||
%
|
||||
% Writing to shared objects can have unpredictable results for two reasons:
|
||||
% - If two or more sessions write to the same element at approximately
|
||||
% the same time, the result is non-deterministic. This phenomenon is
|
||||
% called "resource contention."
|
||||
% - This program violates standard Matlab data reference conventions.
|
||||
% Practically speaking this means that the copy-on-write mechanism
|
||||
% will cause shared objects to become unshared or partially unshared
|
||||
% or simply crash Matlab.
|
||||
%
|
||||
% If you do write to shared memory it is recommended that you share a
|
||||
% non-sparse matrix and write only to internal elements, i.e., no dynamic
|
||||
% resizing. Writes to a shared sparse matrix should not result in a change
|
||||
% to nzmax. As the object becomes more complicated, i.e., sparse matrices
|
||||
% followed by cell arrays, Matlab cross-referencing becomes more
|
||||
% complicated and it becomes harder to trick Matlab into using shared
|
||||
% memory.
|
||||
%
|
||||
% --- What are the typical use-cases for this program?
|
||||
%
|
||||
% Typically, this program is most useful for read-only sharing of a large
|
||||
% matrix among several local Matlab worker sessions, perhaps in the body of
|
||||
% a PARFOR or SPMD. This situation occurs, for example, when fitting a
|
||||
% model to a large amount of training data or evaluating a model under
|
||||
% different parametrizations.
|
||||
%
|
||||
% This program is also useful when repeatedly loading a large amount of data,
|
||||
% i.e., from a .mat file. One could prevent this by loading the data into
|
||||
% shared memory and attaching it as needed rather than reloading it. Since
|
||||
% shared memory persists unless explicitly freed, it can easily be reattached
|
||||
% to Matlab even if Matlab crashes or is exited. In this way there is
|
||||
% essentially zero time to load large data.
|
||||
%
|
||||
% --- Enough already! How do I use this thing?!
|
||||
%
|
||||
% Suppose you have two running Matlab sessions, S0 and S1, and a large sparse
|
||||
% matrix, X, which is loaded on S0.
|
||||
%
|
||||
% % S0:
|
||||
% shmkey = 12345;
|
||||
% shmsiz = sharedmatrix('clone',shmkey,X);
|
||||
% clear X; % not required but no need to keep X locally
|
||||
% % wait for S1 to finish, perhaps S1 is a spmd worker
|
||||
% sharedmatrix('free',shmkey);
|
||||
%
|
||||
% % S1:
|
||||
% shmkey = 12345; % must match the key!
|
||||
% X = sharedmatrix('attach',shmkey);
|
||||
% % do something with X
|
||||
% sharedmatrix('detach',shmkey,X);
|
||||
% clear X; % not required but good practice
|
||||
%
|
||||
% --- What are "directives?"
|
||||
%
|
||||
% This program has four modes of operation which are referred to as
|
||||
% "directives." Directives indicate how to manipulate the memory, as
|
||||
% follows:
|
||||
% 'clone' copy data to shared memory
|
||||
% 'attach' reconstitute shared memory as a local object
|
||||
% 'detach' destroy the local object
|
||||
% 'free' mark the shared memory segment for destruction
|
||||
% All directives minimally require an integer value, or key, which uniquely
|
||||
% identifies a particular shared memory segment.
|
||||
%
|
||||
% --- How do I copy a variable to shared memory?
|
||||
%
|
||||
% The "clone" directive copies the unshared Matlab object into shared
|
||||
% memory. When in shared memory, the object is stored in a custom format
|
||||
% optimized to use the least amount of memory possible.
|
||||
%
|
||||
% Required Arguments:
|
||||
% (1) a unique integer "key" to identify a shared memory segment.
|
||||
% (2) the variable to copy.
|
||||
% Returns:
|
||||
% (1) the size of the shared memory segment (bytes).
|
||||
%
|
||||
% --- How do I load shared memory into Matlab?
|
||||
%
|
||||
% The "attach" directive creates a Matlab object that is a shallow copy of
|
||||
% shared memory data. Locally this object "owns" very little data as most
|
||||
% is a reference to shared memory. The attached shared memory object is
|
||||
% essentially a reference to data stored in shared memory.
|
||||
%
|
||||
% Required Arguments:
|
||||
% (1) a unique integer "key" to identify a shared memory segment.
|
||||
% Returns:
|
||||
% (1) the attached shared memory object.
|
||||
%
|
||||
% --- How do I remove shared memory from Matlab?
|
||||
%
|
||||
% The "detach" directive removes the references the attached shared memory
|
||||
% object makes to shared memory and replaces them with dummy data. This
|
||||
% prevents the Matlab garbage collector from discovering any oddities.
|
||||
%
|
||||
% Required Arguments:
|
||||
% (1) a unique integer "key" to identify a shared memory segment.
|
||||
% (2) the variable to detach.
|
||||
% Returns:
|
||||
% (nil)
|
||||
%
|
||||
% --- How do I free shared memory from my system?
|
||||
%
|
||||
% The "free" directive marks the shared memory segment for deletion. Note:
|
||||
% it is not actually deleted until every attached session explicitly
|
||||
% detaches or is terminated. As soon as the last session detaches, the
|
||||
% system will reclaim the allocated segment. The contact passphrase is
|
||||
% ABEND, see second and third to last questions.
|
||||
%
|
||||
% Required Arguments:
|
||||
% (1) a unique integer "key" to identify a shared memory segment
|
||||
% Returns:
|
||||
% (nil)
|
||||
%
|
||||
% --- How do I prevent crashes?
|
||||
%
|
||||
% Matlab will crash when its internal garbage collection is run on an
|
||||
% attached shared object. This means that you cannot do the following:
|
||||
% - CLEAR an attached shared object
|
||||
% - use PACK when attached shared objects exist.
|
||||
% In some circumstances, you may not be able to save a shared object,
|
||||
% although this statement has not been thoroughly tested.
|
||||
%
|
||||
% The best way to prevent crashes is to attach the shared object only as it
|
||||
% is needed and ALWAYS use the detach directive when done. Continually
|
||||
% reattaching the same object without detaching will result in incorrect
|
||||
% memory reporting and may cause the system to unnecessarily allocate
|
||||
% additional resources, i.e., "OOM kill" other processes (out-of-memory).
|
||||
%
|
||||
% --- How do I manage shared memory outside of this program?
|
||||
%
|
||||
% Often this program will be insufficient for working with shared memory
|
||||
% objects. A common situation is that Matlab will crash before memory can be
|
||||
% appropriately detached or freed. Linux provides several commands which
|
||||
% should augment this program.
|
||||
%
|
||||
% Useful commands/tips/tricks for managing shared memory in Linux:
|
||||
% 1) To see current shared memory maximum (bytes):
|
||||
% cat /proc/sys/kernel/shmmax
|
||||
% cat /proc/sys/kernel/shmall
|
||||
% 2) To do a one-time change shared memory maximum (bytes):
|
||||
% sudo sysctl -w kernel.shmmax=25323843584 # 1TB
|
||||
% sudo sysctl -w kernel.shmall=6182579 # 1TB/(pagesize=4096)
|
||||
% 3) To permanently change the shared memory maximum (bytes):
|
||||
% Edit /etc/sysctl.conf, i.e.,
|
||||
% sudo vim /etc/sysctl.conf
|
||||
% and add the following three lines to the end of the file,
|
||||
% # default shared memory maximum
|
||||
% #kernel.shmmax = 33554432
|
||||
% kernel.shmmax = 25323843584 # 1TB
|
||||
% kernel.shmall = 6182579 # 1TB/(pagesize=4096)
|
||||
% then load the changes via,
|
||||
% sudo sysctl -p /etc/sysctl.conf
|
||||
% 4) To see the currently open shared memory resources:
|
||||
% watch --interval=1 ipcs -m # or just: ipcs -m
|
||||
% 5) To delete (destroy) a shared memory resource:
|
||||
% ipcrm shm xxxxxx
|
||||
% 6) To delete (destroy) all shared memory for current user:
|
||||
% for id in `ipcs -m|grep "$USER"|cut -c12-19`;do ipcrm shm $id; done
|
||||
%
|
||||
% --- I found a bug, what can I do?
|
||||
%
|
||||
% First: calm self. This is a pretty wacky program so you shouldn't be
|
||||
% surprised when you encounter a problem.
|
||||
% Second: simplify the problem into a 10 or maybe 20 line .m script and zip
|
||||
% it up with the necessary .mat data files and email it to the author
|
||||
% at: "jvdillon {at} gmail {dot} com" with a description of the fault.
|
||||
% I will not respond to any email that doesn't have the secret word in
|
||||
% the subject, which you can find from reading this manual.
|
||||
%
|
||||
% This program was only tested on Matlab 7.8.0.347 (R2009a). It is
|
||||
% *certain* that it will not work on older versions. Send me your
|
||||
% $(MATLABROOT)/extern/include/matrix.h
|
||||
% and I will try to make the necessary additions to this program.
|
||||
%
|
||||
% --- I looked at your code and it sucks!
|
||||
%
|
||||
% Well that isn't a question...but I hear ya. I am not a software engineer
|
||||
% but I did badly need shared memory in Matlab (see "typical use-cases"
|
||||
% #1). However, I would very much like to improve this program, so PLEASE
|
||||
% share your complaints, criticisms, and/or code-fu. You can email me at:
|
||||
% "jvdillon {at} gmail {dot} com" and I will reply as soon as I can. I
|
||||
% will not respond to any email that doesn't have the secret word in the
|
||||
% subject, which you can find from reading this manual.
|
||||
%
|
||||
% --- What's next?
|
||||
%
|
||||
% If this is useful to people I will write a semaphore interface for
|
||||
% improved inter-Matlab communication.
|
||||
%
|
||||
%
|
||||
%
|
||||
% See also WHOSSHARED, PARFOR, SPMD, LOAD, PACK.
|
||||
%
|
||||
% Copyright (c) 2010,2011 Joshua V Dillon
|
||||
% All rights reserved. (See file header for details.)
|
||||
|
||||
% Copyright 2010,2011 Joshua V Dillon
|
||||
% $Revision: 0.9.0.0 $ $Date: 2010/08/27 9:24 $
|
||||
% $Revision: 0.9.1.0 $ $Date: 2011/05/26 9:28 $
|
||||
|
||||
% Built-in function.
|
||||
|
||||
+253
@@ -0,0 +1,253 @@
|
||||
% wrapper class around sharedmatrix function , it tries to go around so bugs and
|
||||
% crashes of the code and safely use it for fast interprocess sharing of
|
||||
% data
|
||||
% Use:
|
||||
% self = shm(protected = false, shm_key = randi(1e9))
|
||||
%
|
||||
% COMPILE: mex -R2017b @shm/private/sharedmatrix.c -output @shm/private/sharedmatrix
|
||||
% CLEAN all allocated memory :
|
||||
% !ipcs -m | cut -d' ' -f2 | grep '^[0-9]' | while read x; do ipcrm -m $x; done
|
||||
%
|
||||
% % EXAMPLE:
|
||||
% x = randn(10,'single')+1i;
|
||||
% s = shm();
|
||||
% s.upload(x)
|
||||
% s.detach; % locally forget the data
|
||||
% [s, data] = s.attach % load them back from shared memory
|
||||
% disp(s) % show the SHM class
|
||||
% disp(data)
|
||||
% clear s % or s.detach
|
||||
% disp(data-x)
|
||||
|
||||
%*-----------------------------------------------------------------------*
|
||||
%| |
|
||||
%| Except where otherwise noted, this work is licensed under a |
|
||||
%| Creative Commons Attribution-NonCommercial-ShareAlike 4.0 |
|
||||
%| International (CC BY-NC-SA 4.0) license. |
|
||||
%| |
|
||||
%| Copyright (c) 2018 by Paul Scherrer Institute (http://www.psi.ch) |
|
||||
%| |
|
||||
%| Author: CXS group, PSI |
|
||||
%*-----------------------------------------------------------------------*
|
||||
% You may use this code with the following provisions:
|
||||
%
|
||||
% If the code is fully or partially redistributed, or rewritten in another
|
||||
% computing language this notice should be included in the redistribution.
|
||||
%
|
||||
% If this code, or subfunctions or parts of it, is used for research in a
|
||||
% publication or if it is fully or partially rewritten for another
|
||||
% computing language the authors and institution should be acknowledged
|
||||
% in written form in the publication: “Data processing was carried out
|
||||
% using the “cSAXS matlab package” developed by the CXS group,
|
||||
% Paul Scherrer Institut, Switzerland.”
|
||||
% Variations on the latter text can be incorporated upon discussion with
|
||||
% the CXS group if needed to more specifically reflect the use of the package
|
||||
% for the published work.
|
||||
%
|
||||
% A publication that focuses on describing features, or parameters, that
|
||||
% are already existing in the code should be first discussed with the
|
||||
% authors.
|
||||
%
|
||||
% This code and subroutines are part of a continuous development, they
|
||||
% are provided “as they are” without guarantees or liability on part
|
||||
% of PSI or the authors. It is the user responsibility to ensure its
|
||||
% proper use and the correctness of the results.
|
||||
|
||||
|
||||
|
||||
classdef shm < handle
|
||||
|
||||
properties (GetAccess = 'public', SetAccess = 'private')
|
||||
shm_key;
|
||||
end
|
||||
properties (GetAccess = 'public', SetAccess = 'public')
|
||||
protected; % dont delete shared variable when cleared
|
||||
data_handle;
|
||||
end
|
||||
properties (GetAccess = 'private', SetAccess = 'private')
|
||||
isComplex;
|
||||
end
|
||||
|
||||
%% METHODS:
|
||||
|
||||
methods ( Access = 'public' )
|
||||
function self = shm(varargin)
|
||||
% generate a key for the shared memory, use random number to
|
||||
% avoid collisions between several processes
|
||||
if nargin < 2
|
||||
self.shm_key = randi(1e9);
|
||||
else
|
||||
self.shm_key = varargin{2};
|
||||
end
|
||||
self.isComplex = false;
|
||||
|
||||
% make sure that @shm is under all curcumstancess seen by
|
||||
% matlab, otherwise the share memory may result in matlab crash
|
||||
fpath = mfilename('fullpath');
|
||||
fpath = fpath(1:end-8);
|
||||
if ~contains(path, fpath)
|
||||
addpath(fpath);
|
||||
end
|
||||
|
||||
if nargin < 1
|
||||
self.protected = false;
|
||||
else
|
||||
self.protected = varargin{1};
|
||||
end
|
||||
end
|
||||
function disp(self)
|
||||
if self.isattached()
|
||||
fprintf(['Size of stored data : %i', repmat('x%i',1,ndims(self.data_handle{1})-1),'\n'], size(self.data_handle{1}))
|
||||
fprintf('Class of stored data: %s \n', class(self.data_handle{1}))
|
||||
else
|
||||
disp('No attached data')
|
||||
end
|
||||
fprintf('Protected: %i \n', self.protected)
|
||||
fprintf('Complex: %i \n', self.isComplex)
|
||||
|
||||
!ipcs -m
|
||||
% clear all allocated memory
|
||||
% !ipcs -m | cut -d' ' -f2 | grep '^[0-9]' | while read x; do ipcrm -m $x; done
|
||||
end
|
||||
|
||||
function upload(self, data)
|
||||
% only create the memory space and upload there the data
|
||||
% if isnumeric(data)
|
||||
assert(isnumeric(data) || islogical(data), 'Only numeric arrays are tested to work safely')
|
||||
assert(~isa(data, 'double') || isscalar(data), 'Use single precision to make it fast/memory effecient')
|
||||
assert(numel(data)*4 < 100e9, 'Datasets than 100GB may result in failure, TESTME')
|
||||
% gather the data from GPU
|
||||
if isa(data, 'gpuArray')
|
||||
data = gather(data);
|
||||
end
|
||||
if isscalar(data) && isa(data, 'double')
|
||||
data = single(data);
|
||||
end
|
||||
% end
|
||||
self.free_safe(); % clear the memory if it was used
|
||||
self.create_shm(data, true); % upload to storage
|
||||
end
|
||||
function allocate(self, data)
|
||||
% only create the memory space, do not write any data
|
||||
assert(isnumeric(data), 'Only numeric arrays are tested to work safely')
|
||||
assert(~isa(data, 'double') , 'Use single/integer precision to make it fast/memory effecient')
|
||||
assert(numel(data)*4 < 100e9, 'Datasets than 100GB may result in failure, TESTME')
|
||||
|
||||
self.free_safe(); % clear the memory if it was used
|
||||
self.create_shm(data, false); % use only the array to allocate the storage
|
||||
end
|
||||
|
||||
function [self, data] = attach(self)
|
||||
assert(nargout < 3 && nargout > 0, 'Number of output argument has to be 1 or 2')
|
||||
if ~self.isattached()
|
||||
try
|
||||
self.data_handle=sharedmatrix('attach',self.shm_key);
|
||||
|
||||
catch err
|
||||
if strcmpi(err.identifier,'MATLAB:sharedmatrix:attach')
|
||||
error('No shared data available at address: %i', self.shm_key)
|
||||
end
|
||||
rethrow(err)
|
||||
end
|
||||
end
|
||||
if ~isempty(self.data_handle{2})
|
||||
%% very bad option because memcpy is enforced, but the only solution till sharedmatrix.c is properly fixed
|
||||
data = complex(self.data_handle{:});
|
||||
else
|
||||
data = self.data_handle{1};
|
||||
end
|
||||
end
|
||||
function detach(self)
|
||||
if self.isattached()
|
||||
try
|
||||
sharedmatrix('detach',self.shm_key,self.data_handle);
|
||||
catch err
|
||||
warning(err.message)
|
||||
end
|
||||
self.data_handle = [];
|
||||
end
|
||||
end
|
||||
function free(self)
|
||||
% detach the shared memory
|
||||
self.detach();
|
||||
% and immediatelly delete the memory to avoid pilling it up
|
||||
try sharedmatrix('free',self.shm_key); end
|
||||
end
|
||||
end
|
||||
|
||||
methods ( Access = 'private' )
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
%%%%%%%%%%%%% use MEX file %%%%%%%%%%%%%%%
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
|
||||
varargout = sharedmatrix(varargin)
|
||||
|
||||
%%%%%%%% other auxiliary functions %%%%%%
|
||||
|
||||
function shmsiz = create_shm(self, data, clone)
|
||||
if nargin < 3
|
||||
clone = false; % clone == false -> only create the memory space, do not write any data
|
||||
end
|
||||
if isnumeric(data) && ~isreal(data)
|
||||
%warning('SHM:complex_numbers', 'the sharematrix.c code was modified for matlab newer than R2018b, resulting in very poor performance in complex-valued operations')
|
||||
%warning('off', 'SHM:complex_numbers');
|
||||
data_handle = {real(data), imag(data)};
|
||||
else
|
||||
data_handle = {data, []};
|
||||
end
|
||||
try
|
||||
shmsiz=sharedmatrix('clone',self.shm_key,data_handle, ~clone);
|
||||
catch err
|
||||
if strcmpi(err.identifier, 'MATLAB:mex:ErrInvalidMEXFile')
|
||||
% recompile the MEX file
|
||||
mex @shm/private/sharedmatrix.c -output @shm/private/sharedmatrix
|
||||
shmsiz=sharedmatrix('clone',self.shm_key,data_handle, ~clone);
|
||||
else
|
||||
disp(err)
|
||||
shmsiz = 0;
|
||||
end
|
||||
end
|
||||
|
||||
if shmsiz < numel(data)*class2byte(data)
|
||||
% it was not possible to allocate enough memory for data
|
||||
self.free_safe();
|
||||
disp('Availiable memory:')
|
||||
! free -h
|
||||
fprintf('Required memory: %3.2fGB \n', numel(data)*class2byte(data)/1e9)
|
||||
! ipcs -m
|
||||
disp(' Use following command to clean all allocated memory ')
|
||||
disp('! ipcs -m | cut -d'' '' -f2 | grep ''^[0-9]'' | while read x; do ipcrm -m $x; done')
|
||||
error('Shared memory allocation failed, maybe free memory was too low')
|
||||
end
|
||||
self.isComplex = ~isreal(data);
|
||||
end
|
||||
function status = isattached(self)
|
||||
% return true is the data are attached
|
||||
status = ~(isempty(self.data_handle) || (iscell(self.data_handle) && isempty(self.data_handle{1})));
|
||||
end
|
||||
function delete(self)
|
||||
self.free_safe();
|
||||
end
|
||||
function free_safe(self)
|
||||
% detach the shared memory
|
||||
self.detach();
|
||||
if ~self.protected
|
||||
% and immediatelly delete the memory to avoid pilling it up
|
||||
try sharedmatrix('free',self.shm_key); end
|
||||
%try sharedmatrix('free',self.shm_key+1); end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
function out = class2byte(in)
|
||||
numclass = {'double'; 'single'; 'int8'; 'int16'; 'int32'; 'int64'; 'uint8'; 'uint16'; 'uint32'; 'uint64'};
|
||||
numbytes = [NaN;8;4;1;2;4;8;1;2;4;8];
|
||||
|
||||
[~,loc] = ismember(class(in),numclass);
|
||||
out = numbytes(loc+1);
|
||||
if ~isreal(in)
|
||||
out = out * 2;
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user