Julie dev

.gitignore

@@ -2,6 +2,7 @@
 /local_staging
 /data
 /dask-worker-space
+/.dask_logs
 
 # notebooks bcz eww (force add them if you must?)
 *.ipynb

mti_nma/bin/make_9grid_mode_video.py

@@ -14,8 +14,8 @@
 
 
 def create_gif_from_png_dir(
-        pngdir="local_staging/nma_nuc/post_processing_vtk/9grid_frames/",
-        save_file_path="local_staging/nma_nuc/post_processing_vtk/9grid.gif"):
+        pngdir="/home/juliec/projects/mti_nma/local_staging/nma/nma_data/post_processing_vtk/9grid_frames/",
+        save_file_path="/home/juliec/projects/mti_nma/local_staging/nma/nma_data/post_processing_vtk/9grid.gif"):
 
     # Create empty array 
     images = []

mti_nma/steps/avgshape/avgshape.py

@@ -106,11 +106,20 @@ def run(self, sh_df=None, struct="Nuc", **kwargs):
         save_displacement_map(grid_avg, avg_data_dir / f"avgshape_dmap_{struct}.tif")
 
         # Save mesh as image
-        save_voxelization(mesh_avg, avg_data_dir / f"avgshape_{struct}.tif")
+        domain = save_voxelization(mesh_avg, avg_data_dir / f"avgshape_{struct}.tif")
 
         # Save mesh as stl file for blender import
         save_mesh_as_stl(mesh_avg, avg_data_dir / f"avgshape_{struct}.stl")
 
+        # Remesh voxelization
+        remesh_avg, _, _ = shtools.get_mesh_from_image((domain>0).astype(np.uint8))
+
+        # Save remesh as PLY
+        shtools.save_polydata(
+            mesh=remesh_avg,
+            filename=str(avg_data_dir / f"avgshape_remesh_{struct}.ply")
+        )
+
         # Save avg coeffs to csv file
         coeffs_df_avg.to_csv(
             str(avg_data_dir / f"avgshape_{struct}.csv")

mti_nma/steps/avgshape/avgshape_utils.py

@@ -145,13 +145,13 @@ def save_mesh_as_obj(polydata, fname):
 def save_voxelization(polydata, fname):
 
     domain, _ = cytoparam.voxelize_meshes([polydata])
-
     with writers.ome_tiff_writer.OmeTiffWriter(fname, overwrite_file=True) as writer:
         writer.save(
             255*domain,
             dimension_order='ZYX',
             image_name=fname.stem
         )
+    return domain
 
 def save_displacement_map(grid, fname):
 

mti_nma/steps/load_data/load_data.py

@@ -30,23 +30,27 @@ def run(
         self,
         struct="Nuc",
         nsamples=0,
-        distributed_executor_address: Optional[str]=None,
+        distributed_executor_address: Optional[str] = None,
         debug=False,
         **kwargs
     ):
 
-        struct_dir = self.project_local_staging_dir / f"single_{struct}"
+        struct_dir = self.step_local_staging_dir / f"single_{struct}"
         struct_dir.mkdir(parents=True, exist_ok=True)
-        
+
         df = download_data(
             save_dir=struct_dir / "singlecell_data",
             nsamples=nsamples,
             struct=struct,
             distributed_executor_address=distributed_executor_address
         )
-
+
+        print("Data downloaded")
+
         self.manifest = df
         manifest_save_path = struct_dir / "manifest.csv"
         self.manifest.to_csv(manifest_save_path)
-
+
+        print(f"Manifest saved at {manifest_save_path}")
+
         return self.manifest

mti_nma/steps/load_data/load_data_tools.py

@@ -1,65 +1,64 @@
 import quilt3
-import numpy as np
-import pandas as pd
 from pathlib import Path
-from skimage import measure as skmeasure
 from aicsimageio import AICSImage, writers
-from typing import Dict, List, Optional, Union
+from typing import Optional
 from aics_dask_utils import DistributedHandler
 
+
 def _keep_nucleus_only(fpath):
     seg = AICSImage(fpath).data.squeeze()
     with writers.ome_tiff_writer.OmeTiffWriter(fpath, overwrite_file=True) as writer:
         writer.save(
             seg[0],
-            dimension_order = 'ZYX',
-            image_name = fpath.stem,
+            dimension_order='ZYX',
+            image_name=fpath.stem,
         )
 
+
 def _fetch_data(index, row, pkg, save_dir):
     ''' Downlaods single cell seg only. '''
-    fpath = save_dir/row["crop_seg"]
+    fpath = save_dir / row["crop_seg"]
     pkg[row["crop_seg"]].fetch(fpath)
     _keep_nucleus_only(fpath)
 
+
 def download_data(
     save_dir: Path,
     struct: str,
-    nsamples: int=0,
+    nsamples: int = 0,
     distributed_executor_address: Optional[str] = None
 ):
-   
+
     # >> struct==cell not yet implemented
-
-    package_name="aics/hipsc_single_cell_image_dataset"
-    registry="s3://allencell"
-    data_save_loc="quilt_data"
+
+    package_name = "aics/hipsc_single_cell_image_dataset"
+    registry = "s3://allencell"
 
     pkg = quilt3.Package.browse(package_name, registry)
     df = pkg["metadata.csv"]()
     df = df.set_index('CellId', drop=True)
 
     print(f"Dataset size: {df.shape[0]} elements.")
 
-    df = df.loc[df.cell_stage=='M0']
-    
+    df = df.loc[df.cell_stage == 'M0']
+
     print(f"Dataset size after removing mitotic cells: {df.shape[0]} elements.")
-    
+
     if nsamples > 0:
         df = df.sample(n=nsamples, random_state=42)
         print(f"Test dataset size: {df.shape[0]} elements.")
 
     save_dir.mkdir(parents=True, exist_ok=True)
-    
+
     nrows = df.shape[0]
     with DistributedHandler(distributed_executor_address) as handler:
         handler.batched_map(
             _fetch_data,
             *zip(*list(df.iterrows())),
-            [pkg]*nrows,
-            [save_dir]*nrows
+            [pkg] * nrows,
+            [save_dir] * nrows
         )
-    
+
     # Rename columns according to the rest of the repo    
     columns_to_keep = {
         'FOVId': 'FOVId',
@@ -68,11 +67,10 @@ def download_data(
         'crop_raw': 'RawFilePath',
         'crop_seg': 'SegFilePath'
     }
-        
+
     df = df[[k for k in columns_to_keep.keys()]]
-    df = df.rename(columns = columns_to_keep)
-
-    df['SegFilePath'] = save_dir / df['SegFilePath']
-
-    return df
+    df = df.rename(columns=columns_to_keep)
+
+    df['SegFilePath'] = df['SegFilePath'].apply(lambda x: f"{save_dir}/{x}")
 
+    return df

mti_nma/steps/nma/nma.py

@@ -41,13 +41,11 @@ class Nma(Step):
     def __init__(
         self,
         direct_upstream_tasks: Optional[List["Step"]] = [],
-        filepath_columns=["w_FilePath", "v_FilePath", "vmag_FilePath", "fig_FilePath"],
-        **kwargs
+        filepath_columns=["w_FilePath", "v_FilePath", "vmag_FilePath", "fig_FilePath"]
     ):
         super().__init__(
             direct_upstream_tasks=direct_upstream_tasks,
-            filepath_columns=filepath_columns,
-            **kwargs
+            filepath_columns=filepath_columns
         )
 
     @log_run_params
@@ -104,19 +102,24 @@ def run(
         # If no dataframe is passed in, load manifest from previous step
         if avg_df is None:
             avg_df = pd.read_csv(
-                self.step_local_staging_dir.parent / "avgshape_"
-                f"{struct}" / "manifest.csv"
+                self.step_local_staging_dir.parent / "avgshape" /
+                f"avgshape_{struct}" / "manifest.csv"
             )
 
         # Create directory to hold NMA results
         nma_data_dir = self.step_local_staging_dir / "nma_data"
         nma_data_dir.mkdir(parents=True, exist_ok=True)
 
         reader = vtk.vtkPLYReader()
-        reader.SetFileName(str(avg_df["AvgShapeFilePath"].iloc[0]))
+        # filename = str(avg_df["AvgShapeFilePath"].iloc[0])
+        filename = str(self.step_local_staging_dir / "quadmesh.ply")
+        reader.SetFileName(filename)
         reader.Update()
         polydata = reader.GetOutput()
 
+        print("Loaded data")
+        print(polydata.GetNumberOfCells())
+
         verts, faces = get_vtk_verts_faces(polydata)
         w, v = run_nma(verts, faces)
         draw_whist(w)
@@ -126,7 +129,10 @@ def run(
         n = polydata.GetNumberOfPoints()
         writer = vtk.vtkPolyDataWriter()
 
+        print("about to generate mode visualizations")
+
         for id_mode in range(9):
+            print(id_mode)
 
             # 1st Get eigenvector of interest as a Nx3 array
             arr_eigenvec = v.T[id_mode, :].reshape(3, -1).T
@@ -168,6 +174,8 @@ def run(
                     nma_data_dir / f"avgshape_{struct}_M{id_mode}_T{id_theta:03d}.vtk"))
                 writer.Write()
 
+        print("Saving figures")
+
         fig_path = nma_data_dir / f"w_fig_{struct}.pdf"
         plt.savefig(fig_path, format="pdf")
         w_path = nma_data_dir / f"eigvals_{struct}.npy"
@@ -187,6 +195,8 @@ def run(
             "Structure": struct
         }, index=[0])
 
+        print("Starting blender")
+
         # If no blender path passed: use default for mac and throw error otherwise
         if path_blender is None:
             if platform == "darwin":
@@ -223,7 +233,9 @@ def run(
                 self.manifest[f"mode_{mode}_FilePath"] = output_path
                 self.filepath_columns.append(output_path)
 
+        print("Done - saving manifest")
         # Save manifest as csv
         self.manifest.to_csv(
             self.step_local_staging_dir / f"manifest.csv", index=False
         )
+        print("Manifest saved")

mti_nma/steps/shparam/shparam.py

@@ -142,12 +142,12 @@ def run(
         # If no dataframe is passed in, load manifest from previous step
         if sc_df is None:
             sc_df = pd.read_csv(
-                self.step_local_staging_dir.parent / "single_" 
-                f"{struct}" / "manifest.csv", index_col='CellId'
+                self.step_local_staging_dir.parent / "loaddata" /
+                f"single_{struct}" / "manifest.csv", index_col='CellId'
             )
 
         # Create directory to save data for this step in local staging
-        struct_dir = self.project_local_staging_dir / f"shparam_{struct}"
+        struct_dir = self.step_local_staging_dir / f"shparam_{struct}"
         struct_dir.mkdir(parents=True, exist_ok=True)
         sh_data_dir = struct_dir / "shparam_data"
         sh_data_dir.mkdir(parents=True, exist_ok=True)

mti_nma/tests/local_staging/nma/init_parameters.json

@@ -0,0 +1 @@
+{"step_name": "nma", "filepath_columns": ["w_FilePath", "v_FilePath", "vmag_FilePath", "fig_FilePath"], "metadata_columns": [], "direct_upstream_tasks": [], "config": {"quilt_storage_bucket": "s3://allencell-internal-quilt", "quilt_package_owner": "aics", "quilt_package_name": "mti_nma", "project_local_staging_dir": "local_staging", "nma": {"step_local_staging_dir": "local_staging/nma"}}, "__version__": "0.1.6"}

mti_nma/tests/local_staging/nma/run_parameters.json

@@ -0,0 +1 @@
+{"mode_list": [0, 1, 2, 3, 4, 5], "avg_df": "sphere", "struct": "sphere", "norm_vecs": true, "n_revs": 4, "n_frames": 64, "path_blender": "/allen/aics/modeling/jacksonb/applications/...blender-2.82-linux64/blender", "distributed_executor_address": null}