diff --git a/.gitignore b/.gitignore index 71a0c2f..907d647 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /local_staging /data /dask-worker-space +/.dask_logs # notebooks bcz eww (force add them if you must?) *.ipynb diff --git a/mti_nma/bin/make_9grid_mode_video.py b/mti_nma/bin/make_9grid_mode_video.py index 696763b..6705235 100644 --- a/mti_nma/bin/make_9grid_mode_video.py +++ b/mti_nma/bin/make_9grid_mode_video.py @@ -14,8 +14,8 @@ def create_gif_from_png_dir( - pngdir="local_staging/nma_nuc/post_processing_vtk/9grid_frames/", - save_file_path="local_staging/nma_nuc/post_processing_vtk/9grid.gif"): + pngdir="/home/juliec/projects/mti_nma/local_staging/nma/nma_data/post_processing_vtk/9grid_frames/", + save_file_path="/home/juliec/projects/mti_nma/local_staging/nma/nma_data/post_processing_vtk/9grid.gif"): # Create empty array images = [] diff --git a/mti_nma/steps/avgshape/avgshape.py b/mti_nma/steps/avgshape/avgshape.py index dae785e..16a98c6 100644 --- a/mti_nma/steps/avgshape/avgshape.py +++ b/mti_nma/steps/avgshape/avgshape.py @@ -106,11 +106,20 @@ def run(self, sh_df=None, struct="Nuc", **kwargs): save_displacement_map(grid_avg, avg_data_dir / f"avgshape_dmap_{struct}.tif") # Save mesh as image - save_voxelization(mesh_avg, avg_data_dir / f"avgshape_{struct}.tif") + domain = save_voxelization(mesh_avg, avg_data_dir / f"avgshape_{struct}.tif") # Save mesh as stl file for blender import save_mesh_as_stl(mesh_avg, avg_data_dir / f"avgshape_{struct}.stl") + # Remesh voxelization + remesh_avg, _, _ = shtools.get_mesh_from_image((domain>0).astype(np.uint8)) + + # Save remesh as PLY + shtools.save_polydata( + mesh=remesh_avg, + filename=str(avg_data_dir / f"avgshape_remesh_{struct}.ply") + ) + # Save avg coeffs to csv file coeffs_df_avg.to_csv( str(avg_data_dir / f"avgshape_{struct}.csv") diff --git a/mti_nma/steps/avgshape/avgshape_utils.py b/mti_nma/steps/avgshape/avgshape_utils.py index 13e8900..0674343 100644 --- a/mti_nma/steps/avgshape/avgshape_utils.py +++ b/mti_nma/steps/avgshape/avgshape_utils.py @@ -145,13 +145,13 @@ def save_mesh_as_obj(polydata, fname): def save_voxelization(polydata, fname): domain, _ = cytoparam.voxelize_meshes([polydata]) - with writers.ome_tiff_writer.OmeTiffWriter(fname, overwrite_file=True) as writer: writer.save( 255*domain, dimension_order='ZYX', image_name=fname.stem ) + return domain def save_displacement_map(grid, fname): diff --git a/mti_nma/steps/load_data/load_data.py b/mti_nma/steps/load_data/load_data.py index 2b58d94..90fe104 100644 --- a/mti_nma/steps/load_data/load_data.py +++ b/mti_nma/steps/load_data/load_data.py @@ -30,23 +30,27 @@ def run( self, struct="Nuc", nsamples=0, - distributed_executor_address: Optional[str]=None, + distributed_executor_address: Optional[str] = None, debug=False, **kwargs ): - struct_dir = self.project_local_staging_dir / f"single_{struct}" + struct_dir = self.step_local_staging_dir / f"single_{struct}" struct_dir.mkdir(parents=True, exist_ok=True) - + df = download_data( save_dir=struct_dir / "singlecell_data", nsamples=nsamples, struct=struct, distributed_executor_address=distributed_executor_address ) - + + print("Data downloaded") + self.manifest = df manifest_save_path = struct_dir / "manifest.csv" self.manifest.to_csv(manifest_save_path) - + + print(f"Manifest saved at {manifest_save_path}") + return self.manifest diff --git a/mti_nma/steps/load_data/load_data_tools.py b/mti_nma/steps/load_data/load_data_tools.py index 2ca30d3..07153ce 100644 --- a/mti_nma/steps/load_data/load_data_tools.py +++ b/mti_nma/steps/load_data/load_data_tools.py @@ -1,39 +1,38 @@ import quilt3 -import numpy as np -import pandas as pd from pathlib import Path -from skimage import measure as skmeasure from aicsimageio import AICSImage, writers -from typing import Dict, List, Optional, Union +from typing import Optional from aics_dask_utils import DistributedHandler + def _keep_nucleus_only(fpath): seg = AICSImage(fpath).data.squeeze() with writers.ome_tiff_writer.OmeTiffWriter(fpath, overwrite_file=True) as writer: writer.save( seg[0], - dimension_order = 'ZYX', - image_name = fpath.stem, + dimension_order='ZYX', + image_name=fpath.stem, ) + def _fetch_data(index, row, pkg, save_dir): ''' Downlaods single cell seg only. ''' - fpath = save_dir/row["crop_seg"] + fpath = save_dir / row["crop_seg"] pkg[row["crop_seg"]].fetch(fpath) _keep_nucleus_only(fpath) + def download_data( save_dir: Path, struct: str, - nsamples: int=0, + nsamples: int = 0, distributed_executor_address: Optional[str] = None ): - + # >> struct==cell not yet implemented - - package_name="aics/hipsc_single_cell_image_dataset" - registry="s3://allencell" - data_save_loc="quilt_data" + + package_name = "aics/hipsc_single_cell_image_dataset" + registry = "s3://allencell" pkg = quilt3.Package.browse(package_name, registry) df = pkg["metadata.csv"]() @@ -41,25 +40,25 @@ def download_data( print(f"Dataset size: {df.shape[0]} elements.") - df = df.loc[df.cell_stage=='M0'] - + df = df.loc[df.cell_stage == 'M0'] + print(f"Dataset size after removing mitotic cells: {df.shape[0]} elements.") - + if nsamples > 0: df = df.sample(n=nsamples, random_state=42) print(f"Test dataset size: {df.shape[0]} elements.") save_dir.mkdir(parents=True, exist_ok=True) - + nrows = df.shape[0] with DistributedHandler(distributed_executor_address) as handler: handler.batched_map( _fetch_data, *zip(*list(df.iterrows())), - [pkg]*nrows, - [save_dir]*nrows + [pkg] * nrows, + [save_dir] * nrows ) - + # Rename columns according to the rest of the repo columns_to_keep = { 'FOVId': 'FOVId', @@ -68,11 +67,10 @@ def download_data( 'crop_raw': 'RawFilePath', 'crop_seg': 'SegFilePath' } - + df = df[[k for k in columns_to_keep.keys()]] - df = df.rename(columns = columns_to_keep) - - df['SegFilePath'] = save_dir / df['SegFilePath'] - - return df + df = df.rename(columns=columns_to_keep) + + df['SegFilePath'] = df['SegFilePath'].apply(lambda x: f"{save_dir}/{x}") + return df diff --git a/mti_nma/steps/nma/nma.py b/mti_nma/steps/nma/nma.py index 45b430f..8948f6e 100644 --- a/mti_nma/steps/nma/nma.py +++ b/mti_nma/steps/nma/nma.py @@ -41,13 +41,11 @@ class Nma(Step): def __init__( self, direct_upstream_tasks: Optional[List["Step"]] = [], - filepath_columns=["w_FilePath", "v_FilePath", "vmag_FilePath", "fig_FilePath"], - **kwargs + filepath_columns=["w_FilePath", "v_FilePath", "vmag_FilePath", "fig_FilePath"] ): super().__init__( direct_upstream_tasks=direct_upstream_tasks, - filepath_columns=filepath_columns, - **kwargs + filepath_columns=filepath_columns ) @log_run_params @@ -104,8 +102,8 @@ def run( # If no dataframe is passed in, load manifest from previous step if avg_df is None: avg_df = pd.read_csv( - self.step_local_staging_dir.parent / "avgshape_" - f"{struct}" / "manifest.csv" + self.step_local_staging_dir.parent / "avgshape" / + f"avgshape_{struct}" / "manifest.csv" ) # Create directory to hold NMA results @@ -113,10 +111,15 @@ def run( nma_data_dir.mkdir(parents=True, exist_ok=True) reader = vtk.vtkPLYReader() - reader.SetFileName(str(avg_df["AvgShapeFilePath"].iloc[0])) + # filename = str(avg_df["AvgShapeFilePath"].iloc[0]) + filename = str(self.step_local_staging_dir / "quadmesh.ply") + reader.SetFileName(filename) reader.Update() polydata = reader.GetOutput() + print("Loaded data") + print(polydata.GetNumberOfCells()) + verts, faces = get_vtk_verts_faces(polydata) w, v = run_nma(verts, faces) draw_whist(w) @@ -126,7 +129,10 @@ def run( n = polydata.GetNumberOfPoints() writer = vtk.vtkPolyDataWriter() + print("about to generate mode visualizations") + for id_mode in range(9): + print(id_mode) # 1st Get eigenvector of interest as a Nx3 array arr_eigenvec = v.T[id_mode, :].reshape(3, -1).T @@ -168,6 +174,8 @@ def run( nma_data_dir / f"avgshape_{struct}_M{id_mode}_T{id_theta:03d}.vtk")) writer.Write() + print("Saving figures") + fig_path = nma_data_dir / f"w_fig_{struct}.pdf" plt.savefig(fig_path, format="pdf") w_path = nma_data_dir / f"eigvals_{struct}.npy" @@ -187,6 +195,8 @@ def run( "Structure": struct }, index=[0]) + print("Starting blender") + # If no blender path passed: use default for mac and throw error otherwise if path_blender is None: if platform == "darwin": @@ -223,7 +233,9 @@ def run( self.manifest[f"mode_{mode}_FilePath"] = output_path self.filepath_columns.append(output_path) + print("Done - saving manifest") # Save manifest as csv self.manifest.to_csv( self.step_local_staging_dir / f"manifest.csv", index=False ) + print("Manifest saved") diff --git a/mti_nma/steps/shparam/shparam.py b/mti_nma/steps/shparam/shparam.py index 2e7aec6..24c9b5a 100644 --- a/mti_nma/steps/shparam/shparam.py +++ b/mti_nma/steps/shparam/shparam.py @@ -142,12 +142,12 @@ def run( # If no dataframe is passed in, load manifest from previous step if sc_df is None: sc_df = pd.read_csv( - self.step_local_staging_dir.parent / "single_" - f"{struct}" / "manifest.csv", index_col='CellId' + self.step_local_staging_dir.parent / "loaddata" / + f"single_{struct}" / "manifest.csv", index_col='CellId' ) # Create directory to save data for this step in local staging - struct_dir = self.project_local_staging_dir / f"shparam_{struct}" + struct_dir = self.step_local_staging_dir / f"shparam_{struct}" struct_dir.mkdir(parents=True, exist_ok=True) sh_data_dir = struct_dir / "shparam_data" sh_data_dir.mkdir(parents=True, exist_ok=True) diff --git a/mti_nma/tests/local_staging/nma/init_parameters.json b/mti_nma/tests/local_staging/nma/init_parameters.json new file mode 100644 index 0000000..a56aeab --- /dev/null +++ b/mti_nma/tests/local_staging/nma/init_parameters.json @@ -0,0 +1 @@ +{"step_name": "nma", "filepath_columns": ["w_FilePath", "v_FilePath", "vmag_FilePath", "fig_FilePath"], "metadata_columns": [], "direct_upstream_tasks": [], "config": {"quilt_storage_bucket": "s3://allencell-internal-quilt", "quilt_package_owner": "aics", "quilt_package_name": "mti_nma", "project_local_staging_dir": "local_staging", "nma": {"step_local_staging_dir": "local_staging/nma"}}, "__version__": "0.1.6"} \ No newline at end of file diff --git a/mti_nma/tests/local_staging/nma/run_parameters.json b/mti_nma/tests/local_staging/nma/run_parameters.json new file mode 100644 index 0000000..76c4974 --- /dev/null +++ b/mti_nma/tests/local_staging/nma/run_parameters.json @@ -0,0 +1 @@ +{"mode_list": [0, 1, 2, 3, 4, 5], "avg_df": "sphere", "struct": "sphere", "norm_vecs": true, "n_revs": 4, "n_frames": 64, "path_blender": "/allen/aics/modeling/jacksonb/applications/...blender-2.82-linux64/blender", "distributed_executor_address": null} \ No newline at end of file