mirror of
https://github.com/c-sooyoung/fold_slice.git
synced 2026-09-17 22:59:07 +09:00
initial commit
This commit is contained in:
@@ -0,0 +1,210 @@
|
||||
function [p_out, fdb] = GPU_MS(p)
|
||||
|
||||
import math.*
|
||||
import utils.*
|
||||
import plotting.*
|
||||
import engines.GPU_MS.*
|
||||
|
||||
verbose(struct('prefix','GPU/CPU_MS-engine'))
|
||||
|
||||
%%%%%%%%%%%%%%%%%% WRAPPER FOR FOR GPU CODE %%%%%%%%%%%%%%%%%%%
|
||||
|
||||
if p.verbose_level > 2 && check_option(p, 'meta') && isfield(p.meta{1}, 'spec')
|
||||
verbose(1, 'Starting GPU_MS engines on scans:')
|
||||
for i = 1:length(p.meta)
|
||||
verbose(1,p.meta{i}.spec.S)
|
||||
end
|
||||
end
|
||||
|
||||
fdb = [];
|
||||
|
||||
%% load default settings
|
||||
param = initialize.get_defaults();
|
||||
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
%%%%% load values from p-struct to self-class and param structure %
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
|
||||
[self, param, p] = initialize.load_from_p(param, p);
|
||||
|
||||
global use_gpu gpu
|
||||
verbose_level = param.verbose_level;
|
||||
|
||||
%% INITIALIZE GPU IF AVAILABLE %%%
|
||||
gpu_id = -1;
|
||||
param= GPU_wrapper.initialize(param);
|
||||
use_gpu = param.use_gpu;
|
||||
|
||||
|
||||
if use_gpu
|
||||
gpu_id = gpu.Index;
|
||||
verbose(struct('prefix','GPU'))
|
||||
else
|
||||
verbose(struct('prefix','CPU'))
|
||||
end
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
%% %%%%%%%%%%% GPU SOLVER %%%%%%%%%%%%%%%%%%%
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
|
||||
time_start = tic;
|
||||
|
||||
try
|
||||
|
||||
%% initialization of the matlab engine
|
||||
|
||||
% rescale p -> for low resolution initial guess
|
||||
tmp = initialize.rescale_inputs(self, param.Np_p_presolve, true);
|
||||
if param.mirror_objects
|
||||
tmp = shared.flip_mirror_scan(tmp, true);
|
||||
param.share_probe = false; % probe will be flipped as well !!
|
||||
elseif param.align_shared_objects && size(tmp.object,1) > 1 && param.share_object
|
||||
tmp = shared.align_objects(tmp);
|
||||
end
|
||||
|
||||
% make final preparations & checks before engine is loaded
|
||||
[tmp,param] = initialize.check_inputs(tmp, param);
|
||||
|
||||
%disp(size(tmp.probe))
|
||||
|
||||
[tmp,cache] = initialize.init_solver(tmp, param);
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
%% WAIT IN QUEUE UNTIL THERE IS ENOUGH FREE GPU MEMORY TO RUN
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
|
||||
% Check RAM
|
||||
[mem_avail, mem_total] = utils.check_available_memory();
|
||||
verbose(1,'%.0f of %.0f GB RAM free', mem_avail/1e3, mem_total/1e3 );
|
||||
|
||||
while param.use_gpu
|
||||
if isinf(param.grouping)
|
||||
grouping_tmp = 5; % for inf grouping, use a small initial guess of the group size and refine later for given availible memory
|
||||
else
|
||||
grouping_tmp = param.grouping;
|
||||
end
|
||||
[required_mem , data_mem, object_mem, fft_mem] = GPU_wrapper.estimate_req_memory(tmp, param, grouping_tmp);
|
||||
|
||||
|
||||
% get an estimate of the free memory
|
||||
if param.check_gpu_load
|
||||
Ntest = 5; % [s]
|
||||
verbose(1,'Check GPU load')
|
||||
else
|
||||
Ntest = 1;
|
||||
end
|
||||
if utils.verbose >= 0 && gpu.TotalMemory - gpu.AvailableMemory > 1e9
|
||||
% if someone uses the GPU, report the user
|
||||
%utils.report_GPU_usage %disabled by YJ to avoid potential error
|
||||
end
|
||||
|
||||
for ii = 1:Ntest
|
||||
if param.check_gpu_load && verbose_level > 1; progressbar(ii, Ntest); end
|
||||
available_mem(ii) = gpu.AvailableMemory;
|
||||
if isnan(available_mem(ii)); verbose(0, 'GPU reset'); reset(gpu); continue; end
|
||||
if available_mem(ii) > 2*required_mem; break; end
|
||||
pause(1)
|
||||
end
|
||||
|
||||
if param.check_gpu_load && verbose_level > 1
|
||||
progressbar(Ntest, Ntest)
|
||||
fprintf('\b');
|
||||
end
|
||||
if ~isinf(param.grouping)
|
||||
verbose(1,'\n----MEMORY REPORT GPU ID:%i-----\nRequired total memory %3.2fGB\n - Required data memory %3.2fGB \n - Required object memory %3.2fGB \n - Required FFT memory %3.2fGB \n ============================ \navailable memory %3.2f/%3.2fGB',...
|
||||
gpu_id, required_mem / 1e9, data_mem / 1e9, object_mem / 1e9, fft_mem/1e9, min(available_mem)/ 1e9, gpu.TotalMemory/ 1e9)
|
||||
end
|
||||
if required_mem < min(available_mem)
|
||||
% ready to go ....
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
%%%%%%%%%%% call GPU solver %%%%%%%%%%%%%%
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
verbose(1,' === Starting %s solver === ', param.method)
|
||||
|
||||
try
|
||||
% call the solver
|
||||
[out, fourier_error, fdb.score] = ptycho_solver(tmp, param, cache);
|
||||
|
||||
catch ME
|
||||
if any(strcmpi(ME.identifier,{'id:parallel:gpu:array:OOMForOperation','id:MATLAB:LowGPUMem','MATLAB:LowGPUMem',...
|
||||
'parallel:gpu:array:OOM','parallel:gpu:device:UnknownCUDAError', ...
|
||||
'parallel:gpu:array:OOMForOperation', 'parallel:gpu:device:UnknownCUDAError', ...
|
||||
'parallel:gpu:array:FFTInternalError'}))
|
||||
warning('\n=== Failed due to GPU issue trying again ... === \nFile %s Line: %i \n id:%s msg:%s \n \nGPU ID:%i\nRequired memory %3.2fGB\navailable memory %3.2f/%3.2fGB\n', ...
|
||||
ME.stack(1).name, ME.stack(1).line, ME.identifier, ME.message, gpu_id, required_mem / 1e9, gpu.AvailableMemory/ 1e9, gpu.TotalMemory/ 1e9' )
|
||||
% try to return back to waiting queue
|
||||
|
||||
verbose(-1,'Reset GPU')
|
||||
try
|
||||
gpuDevice(gpu_id)
|
||||
end
|
||||
verbose(-1,'Return back to queue')
|
||||
required_mem = required_mem .* 1.2; % assume that the required memory was too low, -> increase
|
||||
% keyboard;
|
||||
continue
|
||||
else
|
||||
rethrow(ME)
|
||||
end
|
||||
end
|
||||
break
|
||||
elseif required_mem > gpu.TotalMemory * 0.9
|
||||
error('Too large memory requirements for selected GPU, try to reduce grouping')
|
||||
end
|
||||
% otherwise keep waiting
|
||||
wait_time = 5;
|
||||
warning('Low memory on GPU %i, waiting %is ...',gpu_id, wait_time)
|
||||
pause(wait_time)
|
||||
|
||||
end
|
||||
if ~param.use_gpu
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
%%%%%%%%%%% call CPU solver (backup) %%%%%%
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
[out, fourier_error, fdb.score] = ptycho_solver(tmp, param, cache);
|
||||
end
|
||||
|
||||
catch ME
|
||||
try
|
||||
warning('\nFile %s Line: %i id: %s msg:%s \n \nGPU ID:%i\nRequired memory %3.2fGB\navailable memory %3.2f/%3.2fGB\n', ...
|
||||
ME.stack(1).name, ME.stack(1).line,ME.identifier, ME.message, gpu_id, required_mem / 1e9, gpu.AvailableMemory/ 1e9, gpu.TotalMemory/ 1e9' )
|
||||
catch
|
||||
verbose(0,'File %s Line: %i \n id:%s msg:%s ', ME.stack(1).name, ME.stack(1).line, ME.identifier, ME.message)
|
||||
|
||||
end
|
||||
if verbose < 1
|
||||
rethrow(ME)
|
||||
else
|
||||
fprintf('verbose %i', verbose)
|
||||
keyboard
|
||||
end
|
||||
end
|
||||
|
||||
if check_option(p, 'clean_residua')
|
||||
warning('Cleaning residua, it may introduce errors in lowest spatial frequencies')
|
||||
out = engines.GPU.shared.clean_residua(out, cache);
|
||||
end
|
||||
if param.mirror_objects
|
||||
% flip the object, data and positions back
|
||||
out = shared.flip_mirror_scan(out, false);
|
||||
end
|
||||
%% upscale outputs back if needed and update valued in "self" structure
|
||||
|
||||
out.diffraction = self.diffraction;
|
||||
out.mask = self.mask;
|
||||
|
||||
out = initialize.rescale_inputs(out, p.asize, false);
|
||||
|
||||
% do not return the modified data, return the original
|
||||
p_out = initialize.save_to_p(out, param, p, fourier_error);
|
||||
|
||||
verbose(0,' === Finished %s solver === in %4.3gs', param.method, toc(time_start))
|
||||
|
||||
fdb.status.status = false;
|
||||
% reset verbosity back to the p-struct level
|
||||
verbose(p.verbose_level)
|
||||
|
||||
|
||||
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user