new yaml schema, new TODO

LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2026 Sooyoung Cheong
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

TODO.md

@@ -1,28 +1,24 @@
 # TODO:
-
 - fold_slice:
     - load diffractions / hdf5 files; change only param per job
     - restructure `FoldSlicePtychoEngine.__init__()` to load data but not params
         - write new `prepare_data.m`
     - start fold_slice from config.yaml instead of setup.txt
 - BO train_x/y transfer between engines for single job
-- multi-GPU dispatcher
-    - synchronous batched BO
-    - asynchronous BO
 - metric() function(s) for each ptycho engine
     - FRC score
 - separate `config` into `bo_config` and `ptycho_config`; let `BOEngine` have no knowledge of ptychography and vice versa.
 - add GPU version of ExamplePtychoEngine
 - change `PtychoEngine.metric()` to accept list of names and return dict
-
-- rename `BOEngine` to `Sampler`
-- organize results/
-    - possibly: `[dataset name]/[salient config]-[date]/`
-
-
-# TODAY:
-- separation of available GPUs and parallel BO batches
-- prepare next batch for efficient GPU use
 - `GridSampler`
     - `.__init__()` should create a grid
     - `.ask()` should remove those items from the grid
+- change job.sub to job.sh
+    - dynamic jobname
+    - pre-check result directory
+
+# TODAY:
+- change yaml schema, use stages in config-new.yaml
+- separation of available GPUs and parallel sample batches
+- prepare next batch for efficient GPU use
+

bo/__init__.py

@@ -1,3 +0,0 @@
-from .base import BOEngine
-from .random import RandomBOEngine
-from .sobo import SingleObjectiveBOEngine

config-new.yaml

@@ -1,16 +1,11 @@
-job:
-  type: "random+sobo"
-  random_iters: 100
-  sobo_iters: 1000
-
 io:
   input_data_path: '/home/swim/shared/Si_project/data/Si2V1_2.mat'
-  result_dir: '/home/swim/bo-ptycho/results/260809-Si/Si2V1_2'
+  result_dir: '/home/swim/bo-ptycho/results/test'
   verbosity: 1
 
 ptycho:
-  engine: 'fold_slice'
-  path: '/home/swim/fold_slice-stable'
+  engine: 'fake'
+  # path: '/home/swim/fold_slice-park'
   params:
     voltage: 200
     alpha_max: 30
@@ -19,16 +14,18 @@ ptycho:
     Nlayers: 20
     thickness: 250
     rbf: 37
-    Nprobe: 1
-    N_scan_x: 64
-    N_scan_y: 64
     tilt_x: 2
     tilt_y: 0 
     scan_step_size: 0.36
-    Niter: 100
-    Niter_save_results: 100
+
+    Niter: 10
+    Niter_save_results: 10
+
     CBED_size: 192
     ADU: 1
+    Nprobe: 1
+    N_scan_x: 64
+    N_scan_y: 64 
     extra_print_info: 'FIB'
     scan_number: 1
     gpu_id: 1
@@ -37,23 +34,38 @@ ptycho:
     probe_change_start: 1
     object_change_start: 1
     grouping: 64
-    probe_posiiton_search: 1
+    probe_position_search: 1
     regularize_layers: 0.2
     variable_probe: false
 
-bo:
-  mode: 'sobo'
-  acquisition: 'ucb'
-  metric: 'log_fourier'
+search:
+  metric: log_fourier
+  
   params:
-    alpha_max:
     defocus:
       radius: 100
-    rot_ang:
     Nlayers:
       radius: 5
       type: int
     thickness:
-      radius: 100
+      radius: 150
+  
   train_x:
   train_y:
+
+  stages:
+    # - sampler: fixed
+    
+    # - sampler: grid
+    #   defocus: 7
+    #   Nlayers: 11
+    #   thickness: 7
+
+    - sampler: random
+      samples: 16
+
+    - sampler: sobo
+      samples: 256
+      batch: 4
+      acquisition: 'ucb'
+      beta: 0.1

config.yaml

@@ -1,11 +1,11 @@
 job:
-  type: "random+sobo"
+  type: 'random+sobo'
   random_iters: 16
-  sobo_iters: 512
+  sobo_iters: 128
 
 io:
-  input_data_path: '/home/swim/shared/Si_project/data/Si30V3_2.mat'
-  result_dir: '/home/swim/bo-ptycho/results/Si30V3_2/260816'
+  input_data_path: '/home/swim/shared/Si_project/data/Si2V1_2.mat'
+  result_dir: '/home/swim/bo-ptycho/results/test'
   verbosity: 1
 
 ptycho:
@@ -14,14 +14,14 @@ ptycho:
   params:
     voltage: 200
     alpha_max: 30
-    defocus: 200
-    rot_ang: 0.1
-    Nlayers: 30
-    thickness: 650
-    rbf: 38
-    tilt_x: 5
-    tilt_y: 3 
-    scan_step_size: 0.35
+    defocus: -250
+    rot_ang: 0.3
+    Nlayers: 20
+    thickness: 250
+    rbf: 37
+    tilt_x: 2
+    tilt_y: 0 
+    scan_step_size: 0.36
 
     Niter: 100
     Niter_save_results: 100
@@ -43,16 +43,15 @@ ptycho:
     regularize_layers: 0.2
     variable_probe: false
 
+
 bo:
   batch: 4
   acquisition: 'ucb'
   beta: 0.1
   metric: 'log_fourier'
   params:
-    alpha_max:
     defocus:
       radius: 100
-    rot_ang:
     Nlayers:
       radius: 5
       type: int

examples/260805-Si/Si_02V1_2.sub

@@ -1,25 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=Si2V
-#SBATCH --nodes=1
-#SBATCH --ntasks=1
-#SBATCH --cpus-per-task=1
-#SBATCH --gres=gpu:rtx-6000ada:1
-#SBATCH --time=100:00:00
-#SBATCH --output=/home/swim/slurm-logs/job_%j.log
-
-echo "2V"
-pwd
-hostname
-date
-
-export PATH=/home/shared/MATLAB/R2021a/bin:$PATH
-export PATH=/usr/local/cuda-11.4/bin:$PATH
-export LD_LIBRARY_PATH=/usr/local/cuda-11.4/lib64:$LD_LIBRARY_PATH
-
-source /home/swim/bo-ptycho/venv/bin/activate
-
-cat examples/260805-Si/Si_02V1_2.yaml
-python -u main.py examples/260805-Si/Si_02V1_2.yaml
-
-date
-echo "SLURM JOB FINISHED"

examples/260805-Si/Si_02V1_2.yaml

@@ -1,59 +0,0 @@
-job:
-  type: "random+sobo"
-  random_iters: 30
-  sobo_iters: 300
-
-io:
-  input_data_path: '/home/swim/Si_project/data/Si2V1_2.mat'
-  result_dir: '/home/swim/bo-ptycho/results/260805-si/Si2V1_2'
-  verbosity: 1
-
-ptycho:
-  engine: 'fold_slice'
-  path: '/home/swim/fold_slice'
-  params:
-    voltage: 200
-    alpha_max: 30
-    defocus: -250
-    rot_ang: 0.3
-    Nlayers: 20
-    thickness: 250
-    rbf: 37
-    Nprobe: 1
-    N_scan_x: 64
-    N_scan_y: 64
-    tilt_x: 2
-    tilt_y: 0 
-    scan_step_size: 0.36
-    Niter: 100
-    Niter_save_results: 100
-    CBED_size: 192
-    ADU: 1
-    extra_print_info: 'FIB'
-    scan_number: 1
-    gpu_id: 1
-    roi_label: '0_Ndp64'
-    diff_pattern_blur: 1
-    probe_change_start: 1
-    object_change_start: 1
-    grouping: 64
-    probe_posiiton_search: 1
-    regularize_layers: 0.2
-    variable_probe: false
-
-bo:
-  mode: 'sobo'
-  acquisition: 'ucb'
-  metric: 'log_fourier'
-  params:
-    alpha_max:
-    defocus:
-      radius: 100
-    rot_ang:
-    Nlayers:
-      radius: 5
-      type: int
-    thickness:
-      radius: 100
-  train_x:
-  train_y:

examples/260805-Si/Si_05V1_2.sub

@@ -1,25 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=Si5V
-#SBATCH --nodes=1
-#SBATCH --ntasks=1
-#SBATCH --cpus-per-task=1
-#SBATCH --gres=gpu:rtx-6000ada:1
-#SBATCH --time=100:00:00
-#SBATCH --output=/home/swim/slurm-logs/job_%j.log
-
-echo "5V"
-pwd
-hostname
-date
-
-export PATH=/home/shared/MATLAB/R2021a/bin:$PATH
-export PATH=/usr/local/cuda-11.4/bin:$PATH
-export LD_LIBRARY_PATH=/usr/local/cuda-11.4/lib64:$LD_LIBRARY_PATH
-
-source /home/swim/bo-ptycho/venv/bin/activate
-
-cat examples/260805-Si/Si_05V1_2.yaml
-python -u main.py examples/260805-Si/Si_05V1_2.yaml
-
-date
-echo "SLURM JOB FINISHED"

examples/260805-Si/Si_05V1_2.yaml

@@ -1,59 +0,0 @@
-job:
-  type: "random+sobo"
-  random_iters: 30
-  sobo_iters: 300
-
-io:
-  input_data_path: '/home/swim/Si_project/data/Si5V1_2.mat'
-  result_dir: '/home/swim/bo-ptycho/results/260805-si/Si5V1_2'
-  verbosity: 1
-
-ptycho:
-  engine: 'fold_slice'
-  path: '/home/swim/fold_slice'
-  params:
-    voltage: 200
-    alpha_max: 30
-    defocus: -200
-    rot_ang: 1
-    Nlayers: 20
-    thickness: 250
-    rbf: 37
-    Nprobe: 1
-    N_scan_x: 64
-    N_scan_y: 64
-    tilt_x: 3
-    tilt_y: -1 
-    scan_step_size: 0.36
-    Niter: 100
-    Niter_save_results: 100
-    CBED_size: 192
-    ADU: 1
-    extra_print_info: 'FIB'
-    scan_number: 1
-    gpu_id: 1
-    roi_label: '0_Ndp64'
-    diff_pattern_blur: 1
-    probe_change_start: 1
-    object_change_start: 1
-    grouping: 64
-    probe_posiiton_search: 1
-    regularize_layers: 0.2
-    variable_probe: false
-
-bo:
-  mode: 'sobo'
-  acquisition: 'ucb'
-  metric: 'log_fourier'
-  params:
-    alpha_max:
-    defocus:
-      radius: 100
-    rot_ang:
-    Nlayers:
-      radius: 5
-      type: int
-    thickness:
-      radius: 150
-  train_x:
-  train_y:

examples/260805-Si/Si_08V1_2.sub

@@ -1,25 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=Si8V
-#SBATCH --nodes=1
-#SBATCH --ntasks=1
-#SBATCH --cpus-per-task=1
-#SBATCH --gres=gpu:rtx-6000ada:1
-#SBATCH --time=100:00:00
-#SBATCH --output=/home/swim/slurm-logs/job_%j.log
-
-echo "8V"
-pwd
-hostname
-date
-
-export PATH=/home/shared/MATLAB/R2021a/bin:$PATH
-export PATH=/usr/local/cuda-11.4/bin:$PATH
-export LD_LIBRARY_PATH=/usr/local/cuda-11.4/lib64:$LD_LIBRARY_PATH
-
-source /home/swim/bo-ptycho/venv/bin/activate
-
-cat examples/260805-Si/Si_08V1_2.yaml
-python -u main.py examples/260805-Si/Si_08V1_2.yaml
-
-date
-echo "SLURM JOB FINISHED"

examples/260805-Si/Si_08V1_2.yaml

@@ -1,59 +0,0 @@
-job:
-  type: "random+sobo"
-  random_iters: 30
-  sobo_iters: 300
-
-io:
-  input_data_path: '/home/swim/Si_project/data/Si8V1_2.mat'
-  result_dir: '/home/swim/bo-ptycho/results/260805-si/Si8V1_2'
-  verbosity: 1
-
-ptycho:
-  engine: 'fold_slice'
-  path: '/home/swim/fold_slice'
-  params:
-    voltage: 200
-    alpha_max: 30
-    defocus: -200
-    rot_ang: 1.3
-    Nlayers: 25
-    thickness: 350
-    rbf: 37
-    Nprobe: 1
-    N_scan_x: 64
-    N_scan_y: 64
-    tilt_x: 4
-    tilt_y: 0 
-    scan_step_size: 0.36
-    Niter: 100
-    Niter_save_results: 100
-    CBED_size: 192
-    ADU: 1
-    extra_print_info: 'FIB'
-    scan_number: 1
-    gpu_id: 1
-    roi_label: '0_Ndp64'
-    diff_pattern_blur: 1
-    probe_change_start: 1
-    object_change_start: 1
-    grouping: 64
-    probe_posiiton_search: 1
-    regularize_layers: 0.2
-    variable_probe: false
-
-bo:
-  mode: 'sobo'
-  acquisition: 'ucb'
-  metric: 'log_fourier'
-  params:
-    alpha_max:
-    defocus:
-      radius: 100
-    rot_ang:
-    Nlayers:
-      radius: 5
-      type: int
-    thickness:
-      radius: 150
-  train_x:
-  train_y:

examples/260805-Si/Si_30V3_2.sub

@@ -1,25 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=Si30V
-#SBATCH --nodes=1
-#SBATCH --ntasks=1
-#SBATCH --cpus-per-task=1
-#SBATCH --gres=gpu:rtx-6000ada:1
-#SBATCH --time=100:00:00
-#SBATCH --output=/home/swim/slurm-logs/job_%j.log
-
-echo "2V"
-pwd
-hostname
-date
-
-export PATH=/home/shared/MATLAB/R2021a/bin:$PATH
-export PATH=/usr/local/cuda-11.4/bin:$PATH
-export LD_LIBRARY_PATH=/usr/local/cuda-11.4/lib64:$LD_LIBRARY_PATH
-
-source /home/swim/bo-ptycho/venv/bin/activate
-
-cat examples/260805-Si/Si_30V3_2.yaml
-python -u main.py examples/260805-Si/Si_30V3_2.yaml
-
-date
-echo "SLURM JOB FINISHED"

examples/260805-Si/Si_30V3_2.yaml

@@ -1,59 +0,0 @@
-job:
-  type: "random+sobo"
-  random_iters: 30
-  sobo_iters: 300
-
-io:
-  input_data_path: '/home/swim/Si_project/data/Si30V3_2.mat'
-  result_dir: '/home/swim/bo-ptycho/results/260805-si/Si30V3_2'
-  verbosity: 1
-
-ptycho:
-  engine: 'fold_slice'
-  path: '/home/swim/fold_slice'
-  params:
-    voltage: 200
-    alpha_max: 30
-    defocus: 200
-    rot_ang: 0.1
-    Nlayers: 30
-    thickness: 650
-    rbf: 37
-    Nprobe: 1
-    N_scan_x: 64
-    N_scan_y: 64
-    tilt_x: 5
-    tilt_y: 3 
-    scan_step_size: 0.35
-    Niter: 100
-    Niter_save_results: 100
-    CBED_size: 192
-    ADU: 1
-    extra_print_info: 'FIB'
-    scan_number: 1
-    gpu_id: 1
-    roi_label: '0_Ndp64'
-    diff_pattern_blur: 1
-    probe_change_start: 1
-    object_change_start: 1
-    grouping: 64
-    probe_posiiton_search: 1
-    regularize_layers: 0.2
-    variable_probe: false
-
-bo:
-  mode: 'sobo'
-  acquisition: 'ucb'
-  metric: 'log_fourier'
-  params:
-    alpha_max:
-    defocus:
-      radius: 100
-    rot_ang:
-    Nlayers:
-      radius: 5
-      type: int
-    thickness:
-      radius: 150
-  train_x:
-  train_y:

examples/260809-Si/Si_02V1_2.sub

@@ -1,26 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=Si2V
-#SBATCH --nodes=1
-#SBATCH --ntasks=1
-#SBATCH --cpus-per-task=1
-#SBATCH --gres=gpu:rtx-6000ada:1
-#SBATCH --time=100:00:00
-#SBATCH --output=/home/swim/slurm-logs/job_%j.log
-
-echo "2V"
-pwd
-hostname
-date
-
-export PATH=/home/shared/MATLAB/R2021a/bin:$PATH
-export PATH=/usr/local/cuda-11.4/bin:$PATH
-export LD_LIBRARY_PATH=/usr/local/cuda-11.4/lib64:$LD_LIBRARY_PATH
-
-source /home/swim/bo-ptycho/venv/bin/activate
-
-YAML=examples/260809-Si/Si_02V1_2.yaml
-cat $YAML
-python -u main.py $YAML
-
-date
-echo "SLURM JOB FINISHED"

examples/260809-Si/Si_05V1_2.sub

@@ -1,26 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=Si5V
-#SBATCH --nodes=1
-#SBATCH --ntasks=1
-#SBATCH --cpus-per-task=1
-#SBATCH --gres=gpu:rtx-6000ada:1
-#SBATCH --time=100:00:00
-#SBATCH --output=/home/swim/slurm-logs/job_%j.log
-
-echo "5V"
-pwd
-hostname
-date
-
-export PATH=/home/shared/MATLAB/R2021a/bin:$PATH
-export PATH=/usr/local/cuda-11.4/bin:$PATH
-export LD_LIBRARY_PATH=/usr/local/cuda-11.4/lib64:$LD_LIBRARY_PATH
-
-source /home/swim/bo-ptycho/venv/bin/activate
-
-YAML=examples/260809-Si/Si_05V1_2.yaml
-cat $YAML
-python -u main.py $YAML
-
-date
-echo "SLURM JOB FINISHED"

examples/260809-Si/Si_08V1_2.sub

@@ -1,26 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=Si8V
-#SBATCH --nodes=1
-#SBATCH --ntasks=1
-#SBATCH --cpus-per-task=1
-#SBATCH --gres=gpu:rtx-6000ada:1
-#SBATCH --time=100:00:00
-#SBATCH --output=/home/swim/slurm-logs/job_%j.log
-
-echo "8V"
-pwd
-hostname
-date
-
-export PATH=/home/shared/MATLAB/R2021a/bin:$PATH
-export PATH=/usr/local/cuda-11.4/bin:$PATH
-export LD_LIBRARY_PATH=/usr/local/cuda-11.4/lib64:$LD_LIBRARY_PATH
-
-source /home/swim/bo-ptycho/venv/bin/activate
-
-YAML=examples/260809-Si/Si_08V1_2.yaml
-cat $YAML
-python -u main.py $YAML
-
-date
-echo "SLURM JOB FINISHED"

examples/260809-Si/Si_30V3_2.sub

@@ -1,26 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=Si30V
-#SBATCH --nodes=1
-#SBATCH --ntasks=1
-#SBATCH --cpus-per-task=1
-#SBATCH --gres=gpu:rtx-6000ada:1
-#SBATCH --time=100:00:00
-#SBATCH --output=/home/swim/slurm-logs/job_%j.log
-
-echo "30V"
-pwd
-hostname
-date
-
-export PATH=/home/shared/MATLAB/R2021a/bin:$PATH
-export PATH=/usr/local/cuda-11.4/bin:$PATH
-export LD_LIBRARY_PATH=/usr/local/cuda-11.4/lib64:$LD_LIBRARY_PATH
-
-source /home/swim/bo-ptycho/venv/bin/activate
-
-YAML=examples/260809-Si/Si_30V3_2.yaml
-cat $YAML
-python -u main.py $YAML
-
-date
-echo "SLURM JOB FINISHED"

examples/260812-Si/job.sub

@@ -1,26 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=Si2V
-#SBATCH --nodes=1
-#SBATCH --ntasks=1
-#SBATCH --cpus-per-task=1
-#SBATCH --gres=gpu:rtx-6000ada:4
-#SBATCH --time=100:00:00
-#SBATCH --output=/home/swim/slurm-logs/job_%j.log
-
-echo "2V"
-pwd
-hostname
-date
-
-export PATH=/home/shared/MATLAB/R2021a/bin:$PATH
-export PATH=/usr/local/cuda-11.4/bin:$PATH
-export LD_LIBRARY_PATH=/usr/local/cuda-11.4/lib64:$LD_LIBRARY_PATH
-
-source /home/swim/bo-ptycho/venv/bin/activate
-
-YAML=examples/260812-Si/2V.yaml
-cat $YAML
-python -u main.py $YAML
-
-date
-echo "SLURM JOB FINISHED"

examples/silicon-fib/config-2V.yaml

@@ -1,16 +1,16 @@
 job:
-  type: "random+sobo"
-  random_iters: 8
+  type: 'random+sobo'
+  random_iters: 16
   sobo_iters: 128
 
 io:
   input_data_path: '/home/swim/shared/Si_project/data/Si2V1_2.mat'
-  result_dir: '/home/swim/bo-ptycho/results/260812-Si/Si2V1_2'
+  result_dir: '/home/swim/bo-ptycho/results/Si2V1_2/260817'
   verbosity: 1
 
 ptycho:
   engine: 'fold_slice'
-  path: '/home/swim/fold_slice-stable'
+  path: '/home/swim/fold_slice-park'
   params:
     voltage: 200
     alpha_max: 30
@@ -19,16 +19,18 @@ ptycho:
     Nlayers: 20
     thickness: 250
     rbf: 37
-    Nprobe: 1
-    N_scan_x: 64
-    N_scan_y: 64
     tilt_x: 2
     tilt_y: 0 
     scan_step_size: 0.36
+
     Niter: 100
     Niter_save_results: 100
+
     CBED_size: 192
     ADU: 1
+    Nprobe: 1
+    N_scan_x: 64
+    N_scan_y: 64 
     extra_print_info: 'FIB'
     scan_number: 1
     gpu_id: 1
@@ -36,25 +38,24 @@ ptycho:
     diff_pattern_blur: 1
     probe_change_start: 1
     object_change_start: 1
-    grouping: 512
-    probe_posiiton_search: 1
+    grouping: 64
+    probe_position_search: 1
     regularize_layers: 0.2
     variable_probe: false
 
+
 bo:
   batch: 4
   acquisition: 'ucb'
   beta: 0.1
   metric: 'log_fourier'
   params:
-    alpha_max:
     defocus:
       radius: 100
-    rot_ang:
     Nlayers:
       radius: 5
       type: int
     thickness:
-      radius: 100
+      radius: 150
   train_x:
   train_y:

examples/silicon-fib/config-30V.yaml

@@ -1,34 +1,36 @@
 job:
   type: "random+sobo"
-  random_iters: 100
-  sobo_iters: 1000
+  random_iters: 16
+  sobo_iters: 512
 
 io:
   input_data_path: '/home/swim/shared/Si_project/data/Si30V3_2.mat'
-  result_dir: '/home/swim/bo-ptycho/results/260809-Si/Si30V3_2'
+  result_dir: '/home/swim/bo-ptycho/results/Si30V3_2/260816'
   verbosity: 1
 
 ptycho:
   engine: 'fold_slice'
-  path: '/home/swim/fold_slice-stable'
+  path: '/home/swim/fold_slice-park'
   params:
     voltage: 200
     alpha_max: 30
-    defocus: 175
+    defocus: 200
     rot_ang: 0.1
     Nlayers: 30
-    thickness: 680
-    rbf: 37
-    Nprobe: 1
-    N_scan_x: 64
-    N_scan_y: 64
+    thickness: 650
+    rbf: 38
     tilt_x: 5
     tilt_y: 3 
     scan_step_size: 0.35
+
     Niter: 100
     Niter_save_results: 100
+
     CBED_size: 192
     ADU: 1
+    Nprobe: 1
+    N_scan_x: 64
+    N_scan_y: 64 
     extra_print_info: 'FIB'
     scan_number: 1
     gpu_id: 1
@@ -37,23 +39,24 @@ ptycho:
     probe_change_start: 1
     object_change_start: 1
     grouping: 64
-    probe_posiiton_search: 1
+    probe_position_search: 1
     regularize_layers: 0.2
     variable_probe: false
 
 bo:
-  mode: 'sobo'
+  batch: 4
   acquisition: 'ucb'
+  beta: 0.1
   metric: 'log_fourier'
   params:
     alpha_max:
     defocus:
-      radius: 50
+      radius: 100
     rot_ang:
     Nlayers:
       radius: 5
       type: int
     thickness:
-      radius: 100
+      radius: 150
   train_x:
   train_y:

examples/silicon-fib/config-5V.yaml

@@ -1,16 +1,16 @@
 job:
   type: "random+sobo"
-  random_iters: 100
-  sobo_iters: 1000
+  random_iters: 16
+  sobo_iters: 128
 
 io:
   input_data_path: '/home/swim/shared/Si_project/data/Si5V1_2.mat'
-  result_dir: '/home/swim/bo-ptycho/results/260809-Si/Si5V1_2'
+  result_dir: '/home/swim/bo-ptycho/results/Si5V1_2/260815'
   verbosity: 1
 
 ptycho:
   engine: 'fold_slice'
-  path: '/home/swim/fold_slice-stable'
+  path: '/home/swim/fold_slice-park'
   params:
     voltage: 200
     alpha_max: 30
@@ -19,16 +19,18 @@ ptycho:
     Nlayers: 20
     thickness: 250
     rbf: 37
-    Nprobe: 1
-    N_scan_x: 64
-    N_scan_y: 64
     tilt_x: 3
     tilt_y: -1 
     scan_step_size: 0.36
+
     Niter: 100
     Niter_save_results: 100
+
     CBED_size: 192
     ADU: 1
+    Nprobe: 1
+    N_scan_x: 64
+    N_scan_y: 64 
     extra_print_info: 'FIB'
     scan_number: 1
     gpu_id: 1
@@ -37,13 +39,14 @@ ptycho:
     probe_change_start: 1
     object_change_start: 1
     grouping: 64
-    probe_posiiton_search: 1
+    probe_position_search: 1
     regularize_layers: 0.2
     variable_probe: false
 
 bo:
-  mode: 'sobo'
+  batch: 4
   acquisition: 'ucb'
+  beta: 0.1
   metric: 'log_fourier'
   params:
     alpha_max:

examples/silicon-fib/config-8V.yaml

@@ -1,16 +1,16 @@
 job:
   type: "random+sobo"
-  random_iters: 100
-  sobo_iters: 1000
+  random_iters: 16
+  sobo_iters: 128
 
 io:
-  input_data_path: '/home/swim/shared/Si_project/data/Si8V1_2.mat'
-  result_dir: '/home/swim/bo-ptycho/results/260809-Si/Si8V1_2'
+  input_data_path: '/home/swim/shared/Si_project/data/Si8V2_2.mat'
+  result_dir: '/home/swim/bo-ptycho/results/Si8V2_2/260815'
   verbosity: 1
 
 ptycho:
   engine: 'fold_slice'
-  path: '/home/swim/fold_slice-stable'
+  path: '/home/swim/fold_slice-park'
   params:
     voltage: 200
     alpha_max: 30
@@ -19,16 +19,18 @@ ptycho:
     Nlayers: 25
     thickness: 350
     rbf: 37
-    Nprobe: 1
-    N_scan_x: 64
-    N_scan_y: 64
     tilt_x: 4
     tilt_y: 0 
     scan_step_size: 0.36
+
     Niter: 100
     Niter_save_results: 100
+
     CBED_size: 192
     ADU: 1
+    Nprobe: 1
+    N_scan_x: 64
+    N_scan_y: 64 
     extra_print_info: 'FIB'
     scan_number: 1
     gpu_id: 1
@@ -37,13 +39,14 @@ ptycho:
     probe_change_start: 1
     object_change_start: 1
     grouping: 64
-    probe_posiiton_search: 1
+    probe_position_search: 1
     regularize_layers: 0.2
     variable_probe: false
 
 bo:
-  mode: 'sobo'
+  batch: 4
   acquisition: 'ucb'
+  beta: 0.1
   metric: 'log_fourier'
   params:
     alpha_max:

job.sub

@@ -1,5 +1,5 @@
 #!/bin/bash
-#SBATCH --job-name=Si30V
+#SBATCH --job-name=Si2V
 #SBATCH --nodes=1
 #SBATCH --ntasks=1
 #SBATCH --cpus-per-task=16

main.py

@@ -11,10 +11,15 @@ def main(config_yaml):
 
     result_dir = config["io"]["result_dir"]
     if os.path.exists(result_dir):
+        if sys.stdin.isatty():
+            answer = input(f"Will delete {result_dir}: [y/N]\n> ").strip().lower() == 'y'
+            if not answer:
+                print("Aborted.")
+                return         
         shutil.rmtree(result_dir)
+    
     os.makedirs(result_dir, exist_ok=True)
     shutil.copy(config_yaml, os.path.join(result_dir, os.path.basename(config_yaml)))
-
     pipelines.job_types[config['job']['type']](config)
     
 

pipelines/batched_sobo.py

@@ -2,7 +2,7 @@ import os
 import traceback
 import multiprocessing as mp
 
-import bo
+import samplers
 import ptycho
 
 def run_ptycho_worker(worker_id, gpu_token, job_config, metric, run_id, result_queue):
@@ -71,7 +71,7 @@ def sobo_pipeline(config):
 
 
     ############################ RANDOM SAMPLING ###############################
-    randombo = bo.RandomBOEngine(config)
+    randombo = samplers.RandomSampler(config)
 
     for j in range(RANDOM_ITERS):
         print(f"RANDOM sampling; iteration {j}")
@@ -81,7 +81,7 @@ def sobo_pipeline(config):
             randombo.tell(job_config, y_value)
 
     ############################  SOBO SAMPLING  ###############################
-    sobo = bo.SingleObjectiveBOEngine(config)
+    sobo = samplers.SOBOSampler(config)
     sobo.train_x = randombo.train_x
     sobo.train_y = randombo.train_y
 

pipelines/test.py

@@ -1,9 +1,6 @@
-import bo
+import samplers
 import ptycho
 
 
 def test_pipeline(config):
-
-    
-
     pass

readme.md

@@ -0,0 +1,26 @@
+# bo-ptycho
+
+This code is a (re)implementation of Bayesian Optimized ptychography in python for use at LEMON lab, used for
+
+- Optimization-based Thickness Estimation and FIB-induced Damage Characterization of TEM Sample via Multislice Electron Ptychography (manuscript)
+
+
+
+## Code usage
+
+Prerequisites:
+1. Create a python virtual environment with [requirements.txt](./requirements.txt)
+2. Download [fold_slice](#)
+2. Download [experimental data](#)
+
+Reconstruction and parameter optimization:
+1. Edit [config.yaml](./config.yaml)
+    - Example configurations can be found under `examples/silicon-fib/`
+    - Set io.input_data_path and io.result_dir as needed (WARNING: the result directory will be completely removed)
+    - Set ptycho.path to your fold_slice path
+2. Edit [job.sub](./job.sub)
+    - Set the SBATCH configurations as needed (especially `gres` and `output`)
+    - The number of GPUs should match bo.batch in config.yaml
+    - Set the virtual environment path
+    - Set the config.yaml path
+3. Submit the job via SLURM

samplers/__init__.py

@@ -0,0 +1,8 @@
+from .base import Sampler
+from .random import RandomSampler
+from .sobo import SOBOSampler
+
+samplers = {
+    'sobo':   SOBOSampler,
+    'random': RandomSampler,
+}

samplers/base.py

@@ -3,7 +3,7 @@ import os
 import numpy as np
 
 
-class BOEngine(ABC):
+class Sampler(ABC):
     def __init__(self, config):
         self.config = config
         self.params = [key for key, spec in config["bo"]["params"].items() if spec is not None]

samplers/random.py

@@ -1,9 +1,9 @@
 import copy
 import numpy as np
-from bo.base import BOEngine
+from samplers.base import Sampler
 
 
-class RandomBOEngine(BOEngine):
+class RandomSampler(Sampler):
 
     def __init__(self, config):
         super().__init__(config)

samplers/sobo.py

@@ -15,10 +15,10 @@ from botorch.sampling.normal import SobolQMCNormalSampler
 from botorch.utils.rounding import approximate_round
 
 
-from bo.base import BOEngine
+from samplers.base import Sampler
 
 
-class SingleObjectiveBOEngine(BOEngine):
+class SOBOSampler(Sampler):
     def __init__(self, config):
         super().__init__(config)
         self.acquisition = config['bo']['acquisition']