1000 iteration jobs

TODO.md

@@ -1,8 +1,14 @@
-- tag publish version for Si reconstruction
+TODO:
-- Clean up BO states
+
+- unify `BOEngine.__init__()`
 - BO train_x/y transfer between engines for single job
 - multi-GPU dispatcher
 - template job sequences / yamls
+    - mobo
 - metric() function(s) for each ptycho engine
+    - FRC score
 - fold_slice: load diffractions / hdf5 files; change only param per job
-    - Possibly restructure PtychoEngine.__init__() to load data but not params
+    - restructure `FoldSlicePtychoEngine.__init__()` to load data but not params
+        - write new `prepare_data.m`
+- separate `config` into `bo_config` and `ptycho_config`; let `BOEngine` have no knowledge of ptychography and vice versa.
+- add GPU version of ExamplePtychoEngine

examples/260809-Si/Si_02V1_2.sub

@@ -0,0 +1,26 @@
+#!/bin/bash
+#SBATCH --job-name=Si2V
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:rtx-6000ada:1
+#SBATCH --time=100:00:00
+#SBATCH --output=/home/swim/slurm-logs/job_%j.log
+
+echo "2V"
+pwd
+hostname
+date
+
+export PATH=/home/shared/MATLAB/R2021a/bin:$PATH
+export PATH=/usr/local/cuda-11.4/bin:$PATH
+export LD_LIBRARY_PATH=/usr/local/cuda-11.4/lib64:$LD_LIBRARY_PATH
+
+source /home/swim/bo-ptycho/venv/bin/activate
+
+YAML=examples/260809-Si/Si_02V1_2.yaml
+cat $YAML
+python -u main.py $YAML
+
+date
+echo "SLURM JOB FINISHED"

examples/260809-Si/Si_02V1_2.yaml

@@ -0,0 +1,59 @@
+job:
+  type: "random+sobo"
+  random_iters: 100
+  sobo_iters: 1000
+
+io:
+  input_data_path: '/home/swim/shared/Si_project/data/Si2V1_2.mat'
+  result_dir: '/home/swim/bo-ptycho/results/260809-Si/Si2V1_2'
+  verbosity: 1
+
+ptycho:
+  engine: 'fold_slice'
+  path: '/home/swim/fold_slice-stable'
+  params:
+    voltage: 200
+    alpha_max: 30
+    defocus: -250
+    rot_ang: 0.3
+    Nlayers: 20
+    thickness: 250
+    rbf: 37
+    Nprobe: 1
+    N_scan_x: 64
+    N_scan_y: 64
+    tilt_x: 2
+    tilt_y: 0 
+    scan_step_size: 0.36
+    Niter: 100
+    Niter_save_results: 100
+    CBED_size: 192
+    ADU: 1
+    extra_print_info: 'FIB'
+    scan_number: 1
+    gpu_id: 1
+    roi_label: '0_Ndp64'
+    diff_pattern_blur: 1
+    probe_change_start: 1
+    object_change_start: 1
+    grouping: 64
+    probe_posiiton_search: 1
+    regularize_layers: 0.2
+    variable_probe: false
+
+bo:
+  mode: 'sobo'
+  acquisition: 'ucb'
+  metric: 'log_fourier'
+  params:
+    alpha_max:
+    defocus:
+      radius: 100
+    rot_ang:
+    Nlayers:
+      radius: 5
+      type: int
+    thickness:
+      radius: 100
+  train_x:
+  train_y:

examples/260809-Si/Si_05V1_2.sub

@@ -0,0 +1,26 @@
+#!/bin/bash
+#SBATCH --job-name=Si5V
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:rtx-6000ada:1
+#SBATCH --time=100:00:00
+#SBATCH --output=/home/swim/slurm-logs/job_%j.log
+
+echo "5V"
+pwd
+hostname
+date
+
+export PATH=/home/shared/MATLAB/R2021a/bin:$PATH
+export PATH=/usr/local/cuda-11.4/bin:$PATH
+export LD_LIBRARY_PATH=/usr/local/cuda-11.4/lib64:$LD_LIBRARY_PATH
+
+source /home/swim/bo-ptycho/venv/bin/activate
+
+YAML=examples/260809-Si/Si_05V1_2.yaml
+cat $YAML
+python -u main.py $YAML
+
+date
+echo "SLURM JOB FINISHED"

examples/260809-Si/Si_05V1_2.yaml

@@ -0,0 +1,59 @@
+job:
+  type: "random+sobo"
+  random_iters: 100
+  sobo_iters: 1000
+
+io:
+  input_data_path: '/home/swim/shared/Si_project/data/Si5V1_2.mat'
+  result_dir: '/home/swim/bo-ptycho/results/260809-Si/Si5V1_2'
+  verbosity: 1
+
+ptycho:
+  engine: 'fold_slice'
+  path: '/home/swim/fold_slice-stable'
+  params:
+    voltage: 200
+    alpha_max: 30
+    defocus: -200
+    rot_ang: 1
+    Nlayers: 20
+    thickness: 250
+    rbf: 37
+    Nprobe: 1
+    N_scan_x: 64
+    N_scan_y: 64
+    tilt_x: 3
+    tilt_y: -1 
+    scan_step_size: 0.36
+    Niter: 100
+    Niter_save_results: 100
+    CBED_size: 192
+    ADU: 1
+    extra_print_info: 'FIB'
+    scan_number: 1
+    gpu_id: 1
+    roi_label: '0_Ndp64'
+    diff_pattern_blur: 1
+    probe_change_start: 1
+    object_change_start: 1
+    grouping: 64
+    probe_posiiton_search: 1
+    regularize_layers: 0.2
+    variable_probe: false
+
+bo:
+  mode: 'sobo'
+  acquisition: 'ucb'
+  metric: 'log_fourier'
+  params:
+    alpha_max:
+    defocus:
+      radius: 100
+    rot_ang:
+    Nlayers:
+      radius: 5
+      type: int
+    thickness:
+      radius: 150
+  train_x:
+  train_y:

examples/260809-Si/Si_08V1_2.sub

@@ -0,0 +1,26 @@
+#!/bin/bash
+#SBATCH --job-name=Si8V
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:rtx-6000ada:1
+#SBATCH --time=100:00:00
+#SBATCH --output=/home/swim/slurm-logs/job_%j.log
+
+echo "8V"
+pwd
+hostname
+date
+
+export PATH=/home/shared/MATLAB/R2021a/bin:$PATH
+export PATH=/usr/local/cuda-11.4/bin:$PATH
+export LD_LIBRARY_PATH=/usr/local/cuda-11.4/lib64:$LD_LIBRARY_PATH
+
+source /home/swim/bo-ptycho/venv/bin/activate
+
+YAML=examples/260809-Si/Si_08V1_2.yaml
+cat $YAML
+python -u main.py $YAML
+
+date
+echo "SLURM JOB FINISHED"

examples/260809-Si/Si_08V1_2.yaml

@@ -0,0 +1,59 @@
+job:
+  type: "random+sobo"
+  random_iters: 100
+  sobo_iters: 1000
+
+io:
+  input_data_path: '/home/swim/shared/Si_project/data/Si8V1_2.mat'
+  result_dir: '/home/swim/bo-ptycho/results/260809-Si/Si8V1_2'
+  verbosity: 1
+
+ptycho:
+  engine: 'fold_slice'
+  path: '/home/swim/fold_slice-stable'
+  params:
+    voltage: 200
+    alpha_max: 30
+    defocus: -200
+    rot_ang: 1.3
+    Nlayers: 25
+    thickness: 350
+    rbf: 37
+    Nprobe: 1
+    N_scan_x: 64
+    N_scan_y: 64
+    tilt_x: 4
+    tilt_y: 0 
+    scan_step_size: 0.36
+    Niter: 100
+    Niter_save_results: 100
+    CBED_size: 192
+    ADU: 1
+    extra_print_info: 'FIB'
+    scan_number: 1
+    gpu_id: 1
+    roi_label: '0_Ndp64'
+    diff_pattern_blur: 1
+    probe_change_start: 1
+    object_change_start: 1
+    grouping: 64
+    probe_posiiton_search: 1
+    regularize_layers: 0.2
+    variable_probe: false
+
+bo:
+  mode: 'sobo'
+  acquisition: 'ucb'
+  metric: 'log_fourier'
+  params:
+    alpha_max:
+    defocus:
+      radius: 100
+    rot_ang:
+    Nlayers:
+      radius: 5
+      type: int
+    thickness:
+      radius: 150
+  train_x:
+  train_y:

examples/260809-Si/Si_30V3_2.sub

@@ -0,0 +1,26 @@
+#!/bin/bash
+#SBATCH --job-name=Si30V
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:rtx-6000ada:1
+#SBATCH --time=100:00:00
+#SBATCH --output=/home/swim/slurm-logs/job_%j.log
+
+echo "30V"
+pwd
+hostname
+date
+
+export PATH=/home/shared/MATLAB/R2021a/bin:$PATH
+export PATH=/usr/local/cuda-11.4/bin:$PATH
+export LD_LIBRARY_PATH=/usr/local/cuda-11.4/lib64:$LD_LIBRARY_PATH
+
+source /home/swim/bo-ptycho/venv/bin/activate
+
+YAML=examples/260809-Si/Si_30V3_2.yaml
+cat $YAML
+python -u main.py $YAML
+
+date
+echo "SLURM JOB FINISHED"

examples/260809-Si/Si_30V3_2.yaml

@@ -0,0 +1,59 @@
+job:
+  type: "random+sobo"
+  random_iters: 100
+  sobo_iters: 1000
+
+io:
+  input_data_path: '/home/swim/shared/Si_project/data/Si30V3_2.mat'
+  result_dir: '/home/swim/bo-ptycho/results/260809-Si/Si30V3_2'
+  verbosity: 1
+
+ptycho:
+  engine: 'fold_slice'
+  path: '/home/swim/fold_slice-stable'
+  params:
+    voltage: 200
+    alpha_max: 30
+    defocus: 175
+    rot_ang: 0.1
+    Nlayers: 30
+    thickness: 680
+    rbf: 37
+    Nprobe: 1
+    N_scan_x: 64
+    N_scan_y: 64
+    tilt_x: 5
+    tilt_y: 3 
+    scan_step_size: 0.35
+    Niter: 100
+    Niter_save_results: 100
+    CBED_size: 192
+    ADU: 1
+    extra_print_info: 'FIB'
+    scan_number: 1
+    gpu_id: 1
+    roi_label: '0_Ndp64'
+    diff_pattern_blur: 1
+    probe_change_start: 1
+    object_change_start: 1
+    grouping: 64
+    probe_posiiton_search: 1
+    regularize_layers: 0.2
+    variable_probe: false
+
+bo:
+  mode: 'sobo'
+  acquisition: 'ucb'
+  metric: 'log_fourier'
+  params:
+    alpha_max:
+    defocus:
+      radius: 50
+    rot_ang:
+    Nlayers:
+      radius: 5
+      type: int
+    thickness:
+      radius: 100
+  train_x:
+  train_y:

pipelines/sobo.py

@@ -39,9 +39,9 @@ def sobo_pipeline(config):
         print(f"SOBO sampling; iteration {RANDOM_ITERS + j}")
         job_config = sobo.ask()
         ptycho_engine = PTYCHOENGINE(job_config)
-        ptycho_engine.run(run_id=f"bo-{j:03d}")
+        ptycho_engine.run(run_id=f"bo-{RANDOM_ITERS + j:03d}")
         y_value = ptycho_engine.metric(METRIC)
         sobo.tell(job_config, y_value)
         with open(bo_txt, "a") as f:
             p = [f'{job_config['ptycho']['params'][key]:.2f}' for key in sobo.params]
-            f.write(f"{j: 8d}\t{y_value:.4f}\t{"\t".join(p)}\n")
+            f.write(f"{RANDOM_ITERS + j: 8d}\t{y_value:.4f}\t{"\t".join(p)}\n")

ptycho/__init__.py

@@ -1,5 +1,4 @@
-from .base import PtychoEngine
+from .example import ExamplePtychoEngine
-from .ptycho_example import ExamplePtychoEngine
 from .fold_slice import FoldSlicePtychoEngine
 
 engines = {

ptycho/example.py

@@ -8,7 +8,6 @@ class ExamplePtychoEngine(PtychoEngine):
     def __init__(self, config):
         super().__init__(config)
 
-    # run single ptychography job based on `config`
     def run(self, run_id="") -> None:
         print(f"[{run_id}] [ExamplePtychoEngine] Sleeping for 0.1 second.")
         time.sleep(0.1)

ptycho/fold_slice.py

@@ -81,9 +81,9 @@ class FoldSlicePtychoEngine(PtychoEngine):
         self._output = loadmat(mat_path)
 
         log_fourier_error = self._log_fourier_metric()
-        os.makedirs(os.path.join(self.config['io']['result_dir'], "mat"), exist_ok=True)
+        # os.makedirs(os.path.join(self.config['io']['result_dir'], "mat"), exist_ok=True)
         os.makedirs(os.path.join(self.config['io']['result_dir'], "tiff"), exist_ok=True)
-        shutil.copy(mat_path, os.path.join(self.config['io']['result_dir'], "mat", f"{log_fourier_error:.4f}_{run_id}.mat"))
+        # shutil.copy(mat_path, os.path.join(self.config['io']['result_dir'], "mat", f"{log_fourier_error:.4f}_{run_id}.mat")) # saving .mat files takes a lot of space (expect 20+ GB for 64*64 scan size, 300 iterations)
         shutil.copy(image_path, os.path.join(self.config['io']['result_dir'], "tiff", f"{log_fourier_error:.4f}_{run_id}.tiff"))