swim/fold_slice
1
0
Fork 0
mirror of https://github.com/c-sooyoung/fold_slice.git synced 2026-09-17 22:49:08 +09:00
Code Issues Packages Projects Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
swim
2026-08-07 15:56:42 +09:00
commit 91ad25aca9
1012 changed files with 159314 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tomo/@shm/private/sharedmatrix.c
+1054
View File
File diff suppressed because it is too large Load Diff
tomo/@shm/private/sharedmatrix.h
+302
View File
@@ -0,0 +1,302 @@
/*
* Copyright (c) 2010,2011 Joshua V Dillon
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the
* following conditions are met:
* * Redistributions of source code must retain the above
* copyright notice, this list of conditions and the
* following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the
* following disclaimer in the documentation and/or other
* materials provided with the distribution.
* * Neither the name of the author nor the names of its
* contributors may be used to endorse or promote products
* derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY JOSHUA V DILLON ''AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JOSHUA
* V DILLON BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#ifndef sharedmatrix_hxx
#define sharedmatrix_hxx
/* Uses mxArray_tag as defined in matrix.h */
/*#define COMPLIANTMODE*/
/* Verbose outputs */
// #define DEBUG
/* This copies memory across, defeating the purpose of this function but useful for testing */
// #define SAFEMODE
/*
* sharedmatrix.h
*
* This Matlab Mex program allows you to share matlab cell arrays and matrices
* between different Maatlab processes. It has four functions:
* 1) Clone:
* Serialize a 2D cell array of 2D non-/sparse matrices or a 2D
* non-/sparse matrix to shared memory.
* 2) Attach:
* "Reconstitute" the shared data into the appropriate Matlab object using
* shallow copying.
* 3) Detach:
* Remove the shallow references and detach the shared memory from the
* Matlab data space.
* 4) Free:
* Mark the shared memory for destruction.
*
* Written by Joshua V Dillon
* August 27, 2010
*
* Additional Contributors"
* Andrew Smith - Februrary 18, 2011
*
* Revision History:
* Sep. 9. 2010 Corrected the mishandling of sparse logical matrices.
* Apr. 8, 2011 - Merged Andrew Smith's struct code
* - Added Andrew Smith's windows/boost contribution
*/
/*
* This code can be compiled from within Matlab or command-line, assuming the
* system is appropriately setup. To compile, invoke:
*
* For 32-bit machines:
* mex -O -v sharedmatrix.c
* For 64-bit machines:
* mex -largeArrayDims -O -v sharedmatrix.c
*
*/
/*
* Programmer's Notes:
*
* MEX C API:
* http://www.mathworks.com/access/helpdesk/help/techdoc/apiref/bqoqnz0.html
*
* Testing:
*
x=sparse((rand(3,4)<.5).*rand(3,4));
x=cell(2,2);x{1}=rand(3,4);x{4}=sparse((rand(2,4)>.5).*rand(2,4));
shmsiz=sharedmatrix('clone',12345,x)
y=sharedmatrix('attach',12345)
sharedmatrix('detach',12345,y)
*
*/
/*
* Possibly useful information/related work:
*
* http://www.mathworks.com/matlabcentral/fileexchange/24576
* http://www.mathworks.in/matlabcentral/newsreader/view_thread/254813
* http://www.mathworks.com/matlabcentral/newsreader/view_thread/247881#639280
*
* http://groups.google.com/group/comp.soft-sys.matlab/browse_thread/thread/c241d8821fb90275/47189d498d1f45b8?lnk=st&q=&rnum=1&hl=en#47189d498d1f45b8
* http://www.mk.tu-berlin.de/Members/Benjamin/mex_sharedArrays
*/
/* Possibily useful undocumented functions (see links at end for details): */
/* extern mxArray *mxCreateSharedDataCopy(const mxArray *pr); */
/* extern bool mxUnshareArray(const mxArray *pr, const bool noDeepCopy); */
/* extern mxArray *mxUnreference(const mxArray *pr); */
#ifdef DEBUG
#ifdef __STDC__
/*#define DEBUG_HEADER "[%s:% 4d] % 15s(): "*/
#define DEBUG_HEADER "\033[1;34;40m[%s:% 4d] \033[1;31;40m% 15s(): \033[0m"
#define mydebug(format,args...) \
mexPrintf(DEBUG_HEADER format "\n", \
__FILE__,__LINE__,__FUNCTION__,##args)
#else
#define DEBUG_HEADER "DEBUG: "
#define mydebug(format,args...) \
mexPrintf(DEBUG_HEADER format "\n",##args)
#endif
#else
#define mydebug(format,args...) (void)0
#endif
#ifdef COMPLIANTMODE
/* We will be accessing the mxArray parts directly via the mxArray_tag struct
* defined in matrix.h. */
#define ARRAY_ACCESS_INLINING
#include "matrix.h"
#else
struct mxArray_tag {
void *reserved;
int reserved1[2];
void *reserved2;
size_t number_of_dims;
unsigned int reserved3;
struct {
unsigned int flag0 : 1;
unsigned int flag1 : 1;
unsigned int flag2 : 1;
unsigned int flag3 : 1;
unsigned int flag4 : 1;
unsigned int flag5 : 1;
unsigned int flag6 : 1;
unsigned int flag7 : 1;
unsigned int flag7a: 1;
unsigned int flag8 : 1;
unsigned int flag9 : 1;
unsigned int flag10 : 1;
unsigned int flag11 : 4;
unsigned int flag12 : 8;
unsigned int flag13 : 8;
} flags;
size_t reserved4[2];
union {
struct {
void *pdata;
void *pimag_data;
void *reserved5;
size_t reserved6[3];
} number_array;
} data;
};
#endif
typedef struct mxArray_tag mxArrayHack;
/* standard mex include; after hack */
#include "mex.h"
/* max length of directive string */
#define MAXDIRECTIVELEN 256
/* these are used for recording structure field names */
const char term_char = ';'; /*use this character to terminate a string containing the list of fields. Do this because it can't be in a valid field name*/
const size_t align_size = 8; /*the pointer alignment size, so if pdata is a valid pointer then &pdata[i*align_size] will also be. Ensure this is >= 4*/
/*
* The header_t object will be copied to shared memory in its entirety.
*
* Immediately after each copied header_t will be the matrix data values
* [size array, field names, (_at_most_ the four arrays [pr,pi,ir,jc] and in this order)].
*
* The data_t objects will never be copied to shared memory and serve only
* to abstract away mex calls and simplify the deep traversals in matlab.
*
*/
typedef struct data data_t;
typedef struct header header_t;
/* structure used to record all of the data addresses */
struct data {
mwSize *pSize; /* pointer to the size array */
void* pr; /* real data portion */
void* pi; /* imaginary data portion */
mwIndex *ir; /* row indexes, for sparse */
/* OR: may also be list of a structures fields,
each field name will be seperated by a null
character and terminated with a ";" */
mwIndex *jc; /* cumulative column counts, for sparse */
data_t *child_dat; /* array of children data structures, for cell */
header_t *child_hdr; /* array of corresponding children header structures, for cell */
};
/* captures fundamentals of the mxArray */
/* In the shared memory the storage order is [header, size array, field_names,
* real dat, image data, sparse index r, sparse index c] */
struct header {
bool isCell;
bool isSparse;
bool isComplex;
bool isStruct;
mxClassID classid; /* matlab class id */
size_t nDims; /* dimensionality of the matrix. The size array immediately follows the header */
size_t elemsiz; /* size of each element in pr and pi */
size_t nzmax; /* length of pr,pi */
size_t nFields; /* the number of fields. The field string immediately follows the size array */
size_t shmsiz; /* size of serialized object (header + size array + field names string) */
};
/* Remove shared memory references to input matrix (in-situ), recursively */
/* if needed. */
char* deepdetach (mxArray *mxInput);
/* Shallow copy matrix from shared memory into Matlab form. */
size_t shallowrestore (char *shm, mxArray** p_mxInput);
/* Recursively descend through Matlab matrix to assess how much space its */
/* serialization will require. */
size_t deepscan (header_t *hdr, data_t *dat, const mxArray* mxInput);
/* Descend through header and data structure and copy relevent data to */
/* shared memory. */
void deepcopy (header_t *hdr, data_t *dat, char *shared_mem, bool allocate_only);
/* Descend through header and data structure and free the memory. */
void deepfree (data_t *dat);
/* Pads the size to something that guarantees pointer alignment. */
__inline size_t pad_to_align(size_t size) {
if (size % align_size)
size += align_size - (size % align_size);
return size;
}
/* Function to find the number of bytes required to store all of the */
/* field names of a structure */
int FieldNamesSize(const mxArray * mxStruct);
/* Function to copy all of the field names to a character array */
/* Use FieldNamesSize() to allocate the required size of the array */
/* returns the number of bytes used in pList */
int CopyFieldNames(const mxArray * mxStruct, char* pList);
/* This function finds the number of fields contained within in a string */
/* the string is terminated by term_char, a character that can't be in a */
/* field name. pBytes is always an aligned number */
int NumFieldsFromString(const char* pString, size_t *pfields, size_t* pBytes);
/* Function to take point a each element of the char** array at a list of */
/* names contained in string */
/* ppCharArray must be allocated to length num_names */
/* names are seperated by null termination characters */
/* each name must start on an aligned address (see CopyFieldNames()) */
/* e.g. pCharArray[0] = name_1, pCharArray[1] = name_2 ... */
/* returns 0 if successful */
int PointCharArrayAtString(char ** pCharArray, char* pString, int nFields);
/* Function to find the bytes in the string starting from the end of the */
/* string returns < 0 on error */
int BytesFromStringEnd(const char* pString, size_t* pBytes);
// #ifdef SAFEMODE
/* A convenient function for safe assignment of memory to an mxArray */
void* safeCopy(void* pBuffer, mwSize Bytes) {
void* pSafeBuffer;
/* ensure Matlab knows it */
pSafeBuffer = mxMalloc(Bytes);
if (pSafeBuffer != NULL)
memcpy(pSafeBuffer, pBuffer, Bytes); /* and copy the data */
return pSafeBuffer;
}
// #endif
#endif
tomo/@shm/private/sharedmatrix.m
+239
View File
@@ -0,0 +1,239 @@
%SHAREDMATRIX Allows 2D (cell) matrix to be shared between Matlab processes.
% SHAREDMATRIX allows certain Matlab objects (see below) to be shared
% between multiple Matlab sessions, provided they have access to the same
% shared memory resources, i.e., the processes are on the same physical
% system. This program uses shared memory functions specified by POSIX and
% will not work in Windows nor any other non-POSIX environment, although it
% probably could be compiled under cygwin.
%
% WARNING: This program manipulates Matlab objects in a manner that is
% highly unstable. It is very likely that using this function will cause
% Matlab to crash and could potentially lead to data corruption or loss.
% This instability should be limited to the Matlab process itself and
% should not affect the stability of the system.
%
% Since this program is more complicated than the typical Matlab function
% please read this manual thoroughly! It is organized in a Q/A style to
% make referring to it easier.
%
% --- What exactly can be shared?
%
% This program allows multiple Matlab sessions to access one copy of the
% following Matlab data objects:
% - 2D matrix, non-sparse, non-empty
% - 2D matrix, sparse, non-empty
% - 2D cell array of above and/or empty matrices (at least one non-empty)
% - 2D cell array of any of above (recursive)
% In principle, the matrices can be of any type, i.e., UINT32, UINT64,
% etc., although only the DOUBLE type has been extensively tested. The
% matrices can be real or complex.
%
% --- What is meant by "shared?"
%
% A "shared object" is a Matlab matrix or cell array with references to a
% segment of "shared memory." Shared memory is special in that it can be
% made accessible to any program. For purposes of this program the shared
% data object should generally be regarded as read-only, however, by design
% there is no mechanism to force this.
%
% Writing to shared objects can have unpredictable results for two reasons:
% - If two or more sessions write to the same element at approximately
% the same time, the result is non-deterministic. This phenomenon is
% called "resource contention."
% - This program violates standard Matlab data reference conventions.
% Practically speaking this means that the copy-on-write mechanism
% will cause shared objects to become unshared or partially unshared
% or simply crash Matlab.
%
% If you do write to shared memory it is recommended that you share a
% non-sparse matrix and write only to internal elements, i.e., no dynamic
% resizing. Writes to a shared sparse matrix should not result in a change
% to nzmax. As the object becomes more complicated, i.e., sparse matrices
% followed by cell arrays, Matlab cross-referencing becomes more
% complicated and it becomes harder to trick Matlab into using shared
% memory.
%
% --- What are the typical use-cases for this program?
%
% Typically, this program is most useful for read-only sharing of a large
% matrix among several local Matlab worker sessions, perhaps in the body of
% a PARFOR or SPMD. This situation occurs, for example, when fitting a
% model to a large amount of training data or evaluating a model under
% different parametrizations.
%
% This program is also useful when repeatedly loading a large amount of data,
% i.e., from a .mat file. One could prevent this by loading the data into
% shared memory and attaching it as needed rather than reloading it. Since
% shared memory persists unless explicitly freed, it can easily be reattached
% to Matlab even if Matlab crashes or is exited. In this way there is
% essentially zero time to load large data.
%
% --- Enough already! How do I use this thing?!
%
% Suppose you have two running Matlab sessions, S0 and S1, and a large sparse
% matrix, X, which is loaded on S0.
%
% % S0:
% shmkey = 12345;
% shmsiz = sharedmatrix('clone',shmkey,X);
% clear X; % not required but no need to keep X locally
% % wait for S1 to finish, perhaps S1 is a spmd worker
% sharedmatrix('free',shmkey);
%
% % S1:
% shmkey = 12345; % must match the key!
% X = sharedmatrix('attach',shmkey);
% % do something with X
% sharedmatrix('detach',shmkey,X);
% clear X; % not required but good practice
%
% --- What are "directives?"
%
% This program has four modes of operation which are referred to as
% "directives." Directives indicate how to manipulate the memory, as
% follows:
% 'clone' copy data to shared memory
% 'attach' reconstitute shared memory as a local object
% 'detach' destroy the local object
% 'free' mark the shared memory segment for destruction
% All directives minimally require an integer value, or key, which uniquely
% identifies a particular shared memory segment.
%
% --- How do I copy a variable to shared memory?
%
% The "clone" directive copies the unshared Matlab object into shared
% memory. When in shared memory, the object is stored in a custom format
% optimized to use the least amount of memory possible.
%
% Required Arguments:
% (1) a unique integer "key" to identify a shared memory segment.
% (2) the variable to copy.
% Returns:
% (1) the size of the shared memory segment (bytes).
%
% --- How do I load shared memory into Matlab?
%
% The "attach" directive creates a Matlab object that is a shallow copy of
% shared memory data. Locally this object "owns" very little data as most
% is a reference to shared memory. The attached shared memory object is
% essentially a reference to data stored in shared memory.
%
% Required Arguments:
% (1) a unique integer "key" to identify a shared memory segment.
% Returns:
% (1) the attached shared memory object.
%
% --- How do I remove shared memory from Matlab?
%
% The "detach" directive removes the references the attached shared memory
% object makes to shared memory and replaces them with dummy data. This
% prevents the Matlab garbage collector from discovering any oddities.
%
% Required Arguments:
% (1) a unique integer "key" to identify a shared memory segment.
% (2) the variable to detach.
% Returns:
% (nil)
%
% --- How do I free shared memory from my system?
%
% The "free" directive marks the shared memory segment for deletion. Note:
% it is not actually deleted until every attached session explicitly
% detaches or is terminated. As soon as the last session detaches, the
% system will reclaim the allocated segment. The contact passphrase is
% ABEND, see second and third to last questions.
%
% Required Arguments:
% (1) a unique integer "key" to identify a shared memory segment
% Returns:
% (nil)
%
% --- How do I prevent crashes?
%
% Matlab will crash when its internal garbage collection is run on an
% attached shared object. This means that you cannot do the following:
% - CLEAR an attached shared object
% - use PACK when attached shared objects exist.
% In some circumstances, you may not be able to save a shared object,
% although this statement has not been thoroughly tested.
%
% The best way to prevent crashes is to attach the shared object only as it
% is needed and ALWAYS use the detach directive when done. Continually
% reattaching the same object without detaching will result in incorrect
% memory reporting and may cause the system to unnecessarily allocate
% additional resources, i.e., "OOM kill" other processes (out-of-memory).
%
% --- How do I manage shared memory outside of this program?
%
% Often this program will be insufficient for working with shared memory
% objects. A common situation is that Matlab will crash before memory can be
% appropriately detached or freed. Linux provides several commands which
% should augment this program.
%
% Useful commands/tips/tricks for managing shared memory in Linux:
% 1) To see current shared memory maximum (bytes):
% cat /proc/sys/kernel/shmmax
% cat /proc/sys/kernel/shmall
% 2) To do a one-time change shared memory maximum (bytes):
% sudo sysctl -w kernel.shmmax=25323843584 # 1TB
% sudo sysctl -w kernel.shmall=6182579 # 1TB/(pagesize=4096)
% 3) To permanently change the shared memory maximum (bytes):
% Edit /etc/sysctl.conf, i.e.,
% sudo vim /etc/sysctl.conf
% and add the following three lines to the end of the file,
% # default shared memory maximum
% #kernel.shmmax = 33554432
% kernel.shmmax = 25323843584 # 1TB
% kernel.shmall = 6182579 # 1TB/(pagesize=4096)
% then load the changes via,
% sudo sysctl -p /etc/sysctl.conf
% 4) To see the currently open shared memory resources:
% watch --interval=1 ipcs -m # or just: ipcs -m
% 5) To delete (destroy) a shared memory resource:
% ipcrm shm xxxxxx
% 6) To delete (destroy) all shared memory for current user:
% for id in `ipcs -m|grep "$USER"|cut -c12-19`;do ipcrm shm $id; done
%
% --- I found a bug, what can I do?
%
% First: calm self. This is a pretty wacky program so you shouldn't be
% surprised when you encounter a problem.
% Second: simplify the problem into a 10 or maybe 20 line .m script and zip
% it up with the necessary .mat data files and email it to the author
% at: "jvdillon {at} gmail {dot} com" with a description of the fault.
% I will not respond to any email that doesn't have the secret word in
% the subject, which you can find from reading this manual.
%
% This program was only tested on Matlab 7.8.0.347 (R2009a). It is
% *certain* that it will not work on older versions. Send me your
% $(MATLABROOT)/extern/include/matrix.h
% and I will try to make the necessary additions to this program.
%
% --- I looked at your code and it sucks!
%
% Well that isn't a question...but I hear ya. I am not a software engineer
% but I did badly need shared memory in Matlab (see "typical use-cases"
% #1). However, I would very much like to improve this program, so PLEASE
% share your complaints, criticisms, and/or code-fu. You can email me at:
% "jvdillon {at} gmail {dot} com" and I will reply as soon as I can. I
% will not respond to any email that doesn't have the secret word in the
% subject, which you can find from reading this manual.
%
% --- What's next?
%
% If this is useful to people I will write a semaphore interface for
% improved inter-Matlab communication.
%
%
%
% See also WHOSSHARED, PARFOR, SPMD, LOAD, PACK.
%
% Copyright (c) 2010,2011 Joshua V Dillon
% All rights reserved. (See file header for details.)
% Copyright 2010,2011 Joshua V Dillon
% $Revision: 0.9.0.0 $ $Date: 2010/08/27 9:24 $
% $Revision: 0.9.1.0 $ $Date: 2011/05/26 9:28 $
% Built-in function.