Dont transfer entire models for data generation

dingo/pipe/data_generation.py

@@ -1,5 +1,6 @@
 import os
 import sys
+import json
 
 import dingo.pipe.create_injections  # Needed for delta-function time priors.
 
@@ -19,7 +20,6 @@
 import lalsimulation as LS
 import numpy as np
 
-from dingo.core.posterior_models.build_model import build_model_from_kwargs
 from dingo.gw.data.event_dataset import EventDataset
 from dingo.gw.domains import UniformFrequencyDomain, build_domain_from_model_metadata
 from dingo.gw.injection import Injection
@@ -41,6 +41,7 @@ def __init__(self, args, unknown_args, create_data=True):
             command_line_args=args.__dict__,
             unknown_command_line_args=unknown_args,
             injection_parameters=None,
+            model=json.loads(args.model_metadata_str),
             # bilby_version=bilby.__version__,
             # bilby_pipe_version=get_version_information(),
         )
@@ -271,17 +272,7 @@ def interferometers(self, interferometers):
     def _inject_dingo_signal(self, args):
         """Generate a GW signal using the dingo.gw.injection class and add it to the
         interferometer strain data. Also compute SNRs and store them."""
-        try:
-            model = build_model_from_kwargs(
-                filename=args.model, device="meta", load_training_info=False
-            )
-        except RuntimeError:
-            # 'meta' is not supported by older version of python / torch
-            model = build_model_from_kwargs(
-                filename=args.model, device="cpu", load_training_info=False
-            )
-
-        injection = Injection.from_posterior_model_metadata(model.metadata)
+        injection = Injection.from_posterior_model_metadata(self.meta_data["model"])
         injection.use_base_domain = True  # Do not generate MFD signals.
         injection.t_ref = self.trigger_time
         injection._initialize_transform()
@@ -339,17 +330,7 @@ def save_hdf5(self):
         This method will also save the PSDs as .txt files in the data directory
         for easy reading by pesummary and Bilby.
         """
-
-        try:
-            model = build_model_from_kwargs(
-                filename=self.model, device="meta", load_training_info=False
-            )
-        except RuntimeError:
-            # 'meta' is not supported by older version of python / torch
-            model = build_model_from_kwargs(
-                filename=self.model, device="cpu", load_training_info=False
-            )
-        domain = build_domain_from_model_metadata(model.metadata, base=True)
+        domain = build_domain_from_model_metadata(self.meta_data["model"], base=True)
         assert isinstance(domain, UniformFrequencyDomain)
 
         if self.save_bilby_data_dump:

dingo/pipe/main.py

@@ -192,7 +192,8 @@ def fill_in_arguments_from_model(args, perform_arg_checks=True):
         importance_sampling_updates = {
             k.replace("-", "_"): v for k, v in importance_sampling_updates.items()
         }
-    return {**changed_args, **importance_sampling_updates}, model_args
+    model_metadata_str = json.dumps(model_metadata, separators=(",", ":"))
+    return {**changed_args, **importance_sampling_updates}, model_metadata_str
 
 
 class MainInput(BilbyMainInput):
@@ -618,7 +619,9 @@ def main():
     parser = create_parser(top_level=True)
     args, unknown_args = parse_args(get_command_line_arguments(), parser)
 
-    importance_sampling_updates, model_args = fill_in_arguments_from_model(args)
+    importance_sampling_updates, model_metadata_str = fill_in_arguments_from_model(args)
+    args.model_metadata_str = model_metadata_str
+
     inputs = MainInput(args, unknown_args, importance_sampling_updates)
     write_complete_config_file(parser, args, inputs)
 

dingo/pipe/nodes/generation_node.py

@@ -56,19 +56,6 @@ def __init__(self, inputs, trigger_time, idx, dag, parent=None, importance_sampl
                 if (value := getattr(self.inputs, attr)) is not None:
                     input_files_to_transfer.append(os.path.abspath(str(value)))
 
-            # modifying the paths for OSDF networks
-            network_files = []
-            for s in [self.inputs.model, self.inputs.model_init]:
-                if s is None:
-                    continue
-                if "osdf" in s:
-                    # stripping osdf prefix as it is not needed
-                    network_files.append(f"igwn+osdf://{s.replace('/osdf', '')}")
-                else:
-                    network_files.append(s)
-
-            input_files_to_transfer.extend(network_files)
-
             if self.transfer_container:
                 input_files_to_transfer.append(self.inputs.container)
             for value in [

dingo/pipe/parser.py

@@ -1605,4 +1605,10 @@ def create_parser(top_level=True, usage=None):
         ),
     )
 
+    metadata_parser = parser.add_argument_group(
+        "Optional model metadata",
+        description="Machine-written only posterior model metadata.",
+    )
+    metadata_parser.add_argument("--model-metadata-str", type=str, default="{}")
+
     return parser