From acd6883a9e6c5effe5a4aea426a1b913b5968194 Mon Sep 17 00:00:00 2001 From: Stephen Green Date: Thu, 2 Jul 2026 14:46:21 +0200 Subject: [PATCH 01/17] Migrate documentation from Sphinx/MyST to Quarto Convert the docs in place on the docs-quarto branch (prose preserved verbatim): - Rename all 16 pages .md -> .qmd and convert MyST syntax to Quarto: admonitions -> callouts, {cite:p}/{footcite} -> [@..], {eq} -> @eq- refs, cross-links, tables, figures. - Convert the 5 tutorial notebooks' markdown cells (list-form source so Quarto renders callouts); drop inline autoclass embeds now covered by the API reference; convert the mermaid caption to a Quarto fig-cap. - Add _quarto.yml: ReadTheDocs-style docked left sidebar + minimal top navbar, cosmo + custom.scss theme (tidy links/code blocks), mermaid neutral, bibliography. - gen_api.py auto-discovers the package (the sphinx-apidoc analogue) and writes a comprehensive quartodoc reference: 81 modules / 91 pages. - Replace the Sphinx build instructions in installation.qmd with Quarto ones. Sphinx (conf.py, dingo.*.rst) is left intact; a render allowlist makes Quarto ignore it during the migration. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/source/.gitignore | 7 + docs/source/_quarto.yml | 341 ++++++++++++++++++ docs/source/{asimov.md => asimov.qmd} | 0 .../{code_design.md => code_design.qmd} | 10 +- docs/source/custom.scss | 40 ++ docs/source/{dingo_pipe.md => dingo_pipe.qmd} | 31 +- ...e_gnpe_model.md => example_gnpe_model.qmd} | 6 +- ...ple_injection.md => example_injection.qmd} | 4 +- ...ple_npe_model.md => example_npe_model.qmd} | 16 +- ...npe_model.md => example_toy_npe_model.qmd} | 26 +- docs/source/gen_api.py | 79 ++++ docs/source/generating_waveforms.ipynb | 17 +- docs/source/{gnpe.md => gnpe.qmd} | 25 +- docs/source/index.md | 108 ------ docs/source/index.qmd | 46 +++ docs/source/{inference.md => inference.qmd} | 25 +- .../{installation.md => installation.qmd} | 40 +- docs/source/network_architecture.ipynb | 13 +- docs/source/noise_dataset.ipynb | 13 +- docs/source/{overview.md => overview.qmd} | 24 +- docs/source/{quickstart.md => quickstart.qmd} | 4 +- docs/source/{result.md => result.qmd} | 15 +- docs/source/{sbi.md => sbi.qmd} | 8 +- docs/source/{training.md => training.qmd} | 15 +- docs/source/training_transforms.ipynb | 82 +---- docs/source/waveform_dataset.ipynb | 29 +- 26 files changed, 664 insertions(+), 360 deletions(-) create mode 100644 docs/source/.gitignore create mode 100644 docs/source/_quarto.yml rename docs/source/{asimov.md => asimov.qmd} (100%) rename docs/source/{code_design.md => code_design.qmd} (93%) create mode 100644 docs/source/custom.scss rename docs/source/{dingo_pipe.md => dingo_pipe.qmd} (85%) rename docs/source/{example_gnpe_model.md => example_gnpe_model.qmd} (94%) rename docs/source/{example_injection.md => example_injection.qmd} (89%) rename docs/source/{example_npe_model.md => example_npe_model.qmd} (96%) rename docs/source/{example_toy_npe_model.md => example_toy_npe_model.qmd} (88%) create mode 100644 docs/source/gen_api.py rename docs/source/{gnpe.md => gnpe.qmd} (90%) delete mode 100644 docs/source/index.md create mode 100644 docs/source/index.qmd rename docs/source/{inference.md => inference.qmd} (88%) rename docs/source/{installation.md => installation.qmd} (66%) rename docs/source/{overview.md => overview.qmd} (70%) rename docs/source/{quickstart.md => quickstart.qmd} (97%) rename docs/source/{result.md => result.qmd} (94%) rename docs/source/{sbi.md => sbi.qmd} (88%) rename docs/source/{training.md => training.qmd} (97%) diff --git a/docs/source/.gitignore b/docs/source/.gitignore new file mode 100644 index 000000000..3a90a14fe --- /dev/null +++ b/docs/source/.gitignore @@ -0,0 +1,7 @@ +/.quarto/ +**/*.quarto_ipynb + +# Rendered site + quartodoc-generated API reference +/_site/ +/reference/ +/objects.json diff --git a/docs/source/_quarto.yml b/docs/source/_quarto.yml new file mode 100644 index 000000000..39b5b8a66 --- /dev/null +++ b/docs/source/_quarto.yml @@ -0,0 +1,341 @@ +project: + type: website + output-dir: _site + # In-place migration on branch `docs-quarto`. Sphinx files (conf.py, dingo.*.rst, + # build/, jupyter_execute/) still live in this directory; the explicit render list + # below is what makes Quarto ignore them and build only the converted pages. + render: + - index.qmd + - installation.qmd + - overview.qmd + - quickstart.qmd + - example_toy_npe_model.qmd + - example_npe_model.qmd + - example_gnpe_model.qmd + - example_injection.qmd + - sbi.qmd + - code_design.qmd + - generating_waveforms.ipynb + - waveform_dataset.ipynb + - training_transforms.ipynb + - noise_dataset.ipynb + - network_architecture.ipynb + - training.qmd + - inference.qmd + - gnpe.qmd + - result.qmd + - dingo_pipe.qmd + - asimov.qmd + - reference/ + +website: + title: "Dingo" + description: "Neural posterior estimation for gravitational-wave inference" + site-url: "https://dingo-gw.github.io/dingo" + repo-url: "https://github.com/dingo-gw/dingo" + # Top navbar: reserved for site-level links (Home, API, source). Docs navigation + # lives in the left sidebar below, ReadTheDocs-style. + navbar: + background: primary + search: true + left: + - text: "Home" + file: index.qmd + - text: "API reference" + file: reference/index.qmd + right: + - icon: github + href: https://github.com/dingo-gw/dingo + aria-label: Dingo on GitHub + # Left sidebar: the full documentation tree, shown on every docs page. + sidebar: + style: docked + collapse-level: 2 + contents: + - text: "Home" + file: index.qmd + - section: "Getting started" + contents: + - installation.qmd + - overview.qmd + - quickstart.qmd + - section: "Examples" + contents: + - example_toy_npe_model.qmd + - example_npe_model.qmd + - example_gnpe_model.qmd + - example_injection.qmd + - section: "User guide" + contents: + - sbi.qmd + - code_design.qmd + - generating_waveforms.ipynb + - waveform_dataset.ipynb + - training_transforms.ipynb + - noise_dataset.ipynb + - network_architecture.ipynb + - training.qmd + - inference.qmd + - gnpe.qmd + - result.qmd + - dingo_pipe.qmd + - asimov.qmd + - section: "API reference" + contents: + - reference/index.qmd + +bibliography: refs.bib + +format: + html: + theme: [cosmo, custom.scss] + toc: true + code-copy: true + code-overflow: wrap + highlight-style: github + code-block-bg: true + code-block-border-left: "#2478cc" + mermaid: + theme: neutral + +execute: + enabled: false + +# ---- API reference (quartodoc) ---- +# The `sections` below are AUTOGENERATED by gen_api.py, which walks the dingo package +# and lists every public module (the quartodoc analogue of `sphinx-apidoc`). Each module +# is documented with all its public members. Regenerate with `python gen_api.py`. +quartodoc: + package: dingo + dir: reference + title: "API reference" + style: pkgdown + sidebar: reference/_sidebar.yml + sections: + # BEGIN AUTOGEN SECTIONS (run `python gen_api.py` to regenerate; do not edit by hand) + - title: "dingo.asimov.asimov" + contents: + - name: asimov.asimov + children: linked + - title: "dingo.core.dataset" + contents: + - name: core.dataset + children: linked + - title: "dingo.core.density" + contents: + - name: core.density.interpolation + children: linked + - name: core.density.nde_settings + children: linked + - name: core.density.unconditional_density_estimation + children: linked + - title: "dingo.core.likelihood" + contents: + - name: core.likelihood + children: linked + - title: "dingo.core.multiprocessing" + contents: + - name: core.multiprocessing + children: linked + - title: "dingo.core.nn" + contents: + - name: core.nn.cfnets + children: linked + - name: core.nn.enets + children: linked + - name: core.nn.nsf + children: linked + - title: "dingo.core.posterior_models" + contents: + - name: core.posterior_models.base_model + children: linked + - name: core.posterior_models.build_model + children: linked + - name: core.posterior_models.cflow_base + children: linked + - name: core.posterior_models.flow_matching + children: linked + - name: core.posterior_models.normalizing_flow + children: linked + - name: core.posterior_models.score_matching + children: linked + - title: "dingo.core.result" + contents: + - name: core.result + children: linked + - title: "dingo.core.samplers" + contents: + - name: core.samplers + children: linked + - title: "dingo.core.transforms" + contents: + - name: core.transforms + children: linked + - title: "dingo.core.utils" + contents: + - name: core.utils.backward_compatibility + children: linked + - name: core.utils.condor_utils + children: linked + - name: core.utils.gnpeutils + children: linked + - name: core.utils.logging_utils + children: linked + - name: core.utils.misc + children: linked + - name: core.utils.plotting + children: linked + - name: core.utils.pt_to_hdf5 + children: linked + - name: core.utils.torchutils + children: linked + - name: core.utils.trainutils + children: linked + - title: "dingo.gw.SVD" + contents: + - name: gw.SVD + children: linked + - title: "dingo.gw.conversion" + contents: + - name: gw.conversion.spin_conversion + children: linked + - title: "dingo.gw.data" + contents: + - name: gw.data.data_download + children: linked + - name: gw.data.data_preparation + children: linked + - name: gw.data.event_dataset + children: linked + - title: "dingo.gw.dataset" + contents: + - name: gw.dataset.evaluate_multibanded_domain + children: linked + - name: gw.dataset.generate_dataset + children: linked + - name: gw.dataset.generate_dataset_dag + children: linked + - name: gw.dataset.utils + children: linked + - name: gw.dataset.waveform_dataset + children: linked + - title: "dingo.gw.domains" + contents: + - name: gw.domains.base + children: linked + - name: gw.domains.base_frequency_domain + children: linked + - name: gw.domains.build_domain + children: linked + - name: gw.domains.multibanded_frequency_domain + children: linked + - name: gw.domains.time_domain + children: linked + - name: gw.domains.uniform_frequency_domain + children: linked + - title: "dingo.gw.download_strain_data" + contents: + - name: gw.download_strain_data + children: linked + - title: "dingo.gw.gwutils" + contents: + - name: gw.gwutils + children: linked + - title: "dingo.gw.importance_sampling" + contents: + - name: gw.importance_sampling.diagnostics + children: linked + - name: gw.importance_sampling.importance_weights + children: linked + - title: "dingo.gw.inference" + contents: + - name: gw.inference.gw_samplers + children: linked + - name: gw.inference.inference_utils + children: linked + - name: gw.inference.visualization + children: linked + - title: "dingo.gw.injection" + contents: + - name: gw.injection + children: linked + - title: "dingo.gw.likelihood" + contents: + - name: gw.likelihood + children: linked + - title: "dingo.gw.ls_cli" + contents: + - name: gw.ls_cli + children: linked + - title: "dingo.gw.noise" + contents: + - name: gw.noise.asd_dataset + children: linked + - name: gw.noise.asd_estimation + children: linked + - name: gw.noise.generate_dataset + children: linked + - name: gw.noise.generate_dataset_dag + children: linked + - name: gw.noise.synthetic.asd_parameterization + children: linked + - name: gw.noise.synthetic.asd_sampling + children: linked + - name: gw.noise.synthetic.generate_dataset + children: linked + - name: gw.noise.synthetic.utils + children: linked + - name: gw.noise.utils + children: linked + - title: "dingo.gw.prior" + contents: + - name: gw.prior + children: linked + - title: "dingo.gw.result" + contents: + - name: gw.result + children: linked + - title: "dingo.gw.temporary_debug_utils" + contents: + - name: gw.temporary_debug_utils + children: linked + - title: "dingo.gw.training" + contents: + - name: gw.training.train_builders + children: linked + - name: gw.training.train_pipeline + children: linked + - name: gw.training.train_pipeline_condor + children: linked + - name: gw.training.utils + children: linked + - title: "dingo.gw.transforms" + contents: + - name: gw.transforms.detector_transforms + children: linked + - name: gw.transforms.general_transforms + children: linked + - name: gw.transforms.gnpe_transforms + children: linked + - name: gw.transforms.inference_transforms + children: linked + - name: gw.transforms.noise_transforms + children: linked + - name: gw.transforms.parameter_transforms + children: linked + - name: gw.transforms.utils + children: linked + - name: gw.transforms.waveform_transforms + children: linked + - title: "dingo.gw.waveform_generator" + contents: + - name: gw.waveform_generator.frame_utils + children: linked + - name: gw.waveform_generator.waveform_generator + children: linked + - name: gw.waveform_generator.wfg_utils + children: linked + # END AUTOGEN SECTIONS + +metadata-files: + - reference/_sidebar.yml diff --git a/docs/source/asimov.md b/docs/source/asimov.qmd similarity index 100% rename from docs/source/asimov.md rename to docs/source/asimov.qmd diff --git a/docs/source/code_design.md b/docs/source/code_design.qmd similarity index 93% rename from docs/source/code_design.md rename to docs/source/code_design.qmd index 42e4c278e..63b3f2620 100644 --- a/docs/source/code_design.md +++ b/docs/source/code_design.qmd @@ -17,15 +17,15 @@ In addition to saving the user-provided settings at each step, Dingo also saves ### Random seeds -```{admonition} To-do +::: {.callout-note title="To-do"} Implement this. -``` +::: ### Unique identifiers for models -```{admonition} To-do +::: {.callout-note title="To-do"} Implement this. -``` +::: ## Code re-use @@ -39,7 +39,7 @@ We follow the [PyTorch guidelines](https://pytorch.org/tutorials/beginner/basics ### Data structures -Dingo uses several dataset classes, all of which inherit from {py:class}`dingo.core.dataset.DingoDataset`. This provides a common IO (to save/load from HDF5 as well as dictionaries). It also stores the settings dictionary as an attribute. +Dingo uses several dataset classes, all of which inherit from `dingo.core.dataset.DingoDataset`. This provides a common IO (to save/load from HDF5 as well as dictionaries). It also stores the settings dictionary as an attribute. ## Command-line scripts diff --git a/docs/source/custom.scss b/docs/source/custom.scss new file mode 100644 index 000000000..13cfabf6d --- /dev/null +++ b/docs/source/custom.scss @@ -0,0 +1,40 @@ +/*-- scss:defaults --*/ +$link-color: #2478cc; +$code-bg: #f7f8fa; + +/*-- scss:rules --*/ + +// Links: drop the heavy permanent underline; underline only on hover. +a { + text-decoration: none; +} +a:hover { + text-decoration: underline; +} + +// Code blocks (incl. the example config listings): soft border + rounded corners +// so ini/yaml examples read as tidy panels rather than bare text. +div.sourceCode { + border: 1px solid #e3e6ea; + border-radius: 6px; + margin-block: 1rem; + overflow: hidden; +} +div.sourceCode pre { + padding: 0.8em 1em; + margin-bottom: 0; +} +pre > code.sourceCode { + font-size: 0.85em; +} + +// Mermaid: center the flowchart and cap its width so it isn't oversized. +.cell-output-display svg, +figure.quarto-figure svg, +div.mermaid svg, +pre.mermaid svg { + max-width: 720px; + height: auto; + display: block; + margin-inline: auto; +} diff --git a/docs/source/dingo_pipe.md b/docs/source/dingo_pipe.qmd similarity index 85% rename from docs/source/dingo_pipe.md rename to docs/source/dingo_pipe.qmd index 9d9b8478a..1839a6040 100644 --- a/docs/source/dingo_pipe.md +++ b/docs/source/dingo_pipe.qmd @@ -6,10 +6,9 @@ dingo_pipe GW150914.ini ``` dingo_pipe then executes various commands for [preparing data](#data-generation), [sampling from networks](#sampling), [importance sampling](#importance-sampling), and [plotting](#plotting). It can execute commands locally or on a cluster using a DAG. This documentation will only describe the relevant differences compared to bilby_pipe, and we refer the reader to the bilby_pipe documentation for additional information. -```{code-block} ini ---- -caption: Example `GW150914.ini` file. This is also available in the examples/ directory. ---- +Example `GW150914.ini` file (also available in the `examples/` directory): + +```ini ################################################################################ ## Job submission arguments ################################################################################ @@ -70,33 +69,33 @@ plot-log-probs = true The main difference compared to a bilby_pipe `.ini` file is that one specifies trained Dingo models rather than data conditioning and prior settings. The reason for this is that such settings have already been incorporated into training of the model. It is therefore not possible to change them when sampling from the Dingo model. Understandably, this could cause inconvenience if one is interested in a different prior or data conditioning settings. As a solution, Dingo enables the changing of such settings during importance sampling, which applies the new settings for likelihood evaluations. -```{important} +::: {.callout-important} For dingo_pipe it is necessary to specify a trained Dingo model *instead* of sampler settings such as prior and data conditioning. -``` +::: ## Data generation The first step is to download and prepare gravitational-wave data. In the example, dingo_pipe (using bilby_pipe routines) downloads the event and PSD data at the time of GW150914. It then prepares the data based on conditioning settings in the specified Dingo model. If other conflicting conditioning settings are provided (e.g., `sampling_frequency = 2048.0`), dingo_pipe stores these in the dictionary `importance_sampling_updates` (which can also be specified explicitly). These settings are ignored for now, and only applied later for calculating the likelihood in importance sampling. -The prepared event data and ASD are stored in a {py:class}`dingo.gw.data.event_dataset.EventDataset`, which is then saved to disk in HDF5 format. +The prepared event data and ASD are stored in a `dingo.gw.data.event_dataset.EventDataset`, which is then saved to disk in HDF5 format. -```{note} +::: {.callout-note} Dingo models are typically trained using Welch PSDs. For this reason we do not recommend using a BayesWave PSD for initial sampling. Rather, a BayesWave PSD should be specified within the `importance_sampling_updates` dictionary, so that it will be used during importance sampling. -``` +::: ### Injections Injections mirror the [approach of `bilby_pipe`](https://lscsoft.docs.ligo.org/bilby_pipe/master/injections.html) with a few modifications: -* By default, injections are generated using `bilby_pipe`. To use signal waveforms generated by the Dingo code (using the [Injections class](inference.md#injections)) set `dingo-injection = True`. Note that Dingo and Bilby injections might differ depending on the `spin_conversion_phase` setting, which is only available in Dingo. +* By default, injections are generated using `bilby_pipe`. To use signal waveforms generated by the Dingo code (using the [Injections class](inference.qmd#injections)) set `dingo-injection = True`. Note that Dingo and Bilby injections might differ depending on the `spin_conversion_phase` setting, which is only available in Dingo. * Setting `zero-noise = True` will create a zero-noise injection, which is out-of-distribution with respect to Dingo training data. It should therefore be used with caution. * When running many injections `n-simulation > 1`, one may want to produce a PP plot. This can be done automatically by setting `plot-pp = true`. ## Sampling -The next step is sampling from the Dingo model. The model is loaded into a [GWSampler](dingo.gw.inference.gw_samplers.GWSampler) or [GWSamplerGNPE](dingo.gw.inference.gw_samplers.GWSamplerGNPE) object. (If using [GNPE](gnpe) it is necessary to specify a `model-init`.) The Sampler `context` is then set from the EventDataset prepared in the previous step. `num-samples` samples are then generated in batches of size `batch-size`. The samples (and context) are stored in a [Result](dingo.gw.result.Result) object and saved in HDF5 format. +The next step is sampling from the Dingo model. The model is loaded into a `GWSampler` or `GWSamplerGNPE` object. (If using [GNPE](gnpe.qmd) it is necessary to specify a `model-init`.) The Sampler `context` is then set from the EventDataset prepared in the previous step. `num-samples` samples are then generated in batches of size `batch-size`. The samples (and context) are stored in a `Result` object and saved in HDF5 format. -If using GNPE, one can optionally specify `num-gnpe-iterations` (it defaults to 30). Importantly, obtaining the log probability when using GNPE requires an [extra step of training an unconditional flow](result.md#density-recovery). This is done using the `recover-log-prob` flag, which defaults to `True`. The default density recovery settings can be overwritten by providing a `density-recovery-settings` dictionary in the `.ini` file. +If using GNPE, one can optionally specify `num-gnpe-iterations` (it defaults to 30). Importantly, obtaining the log probability when using GNPE requires an [extra step of training an unconditional flow](result.qmd#density-recovery). This is done using the `recover-log-prob` flag, which defaults to `True`. The default density recovery settings can be overwritten by providing a `density-recovery-settings` dictionary in the `.ini` file. Since sampling uses GPU hardware, there is an additional key `sampling-requirements` for HTCondor requirements during the sampling stage. This is intended for specifying GPU requirements such as memory or CUDA version. @@ -106,13 +105,13 @@ For importance sampling, the Result saved in the previous step is loaded. Since If `prior-dict-updates` is specified in the `.ini` file, then this will be used for the importance sampling prior. One example where this is useful is for the luminosity distance prior. Indeed, Dingo tends to train better using a uniform prior over luminosity distance, but physically one would prefer a uniform in volume prior. By specifying `prior-dict-updates` this change can be made in importance sampling. -```{caution} +::: {.callout-caution} If extending the prior support during importance sampling, be sure that the posterior does not rail up against the prior boundary being extended. -``` +::: By default, dingo_pipe assumes that it is necessary to sample the phase synthetically, so it will do so before importance sampling. This can be turned off by passing an empty dictionary to `importance-sampling-settings`. Note that importance sampling itself can be switched off by setting the `importance-sample` flag to False (it defaults to True). -Importance sampling (including synthetic phase sampling) is an expensive step, so dingo_pipe allows for parallelization: this step is split over `n-parallel` jobs, each of which uses `request-cpus-importance-sampling` processes. In the backend, this makes use of the Result [split()](dingo.core.result.Result.split) and [merge()](dingo.core.result.Result.merge) methods. +Importance sampling (including synthetic phase sampling) is an expensive step, so dingo_pipe allows for parallelization: this step is split over `n-parallel` jobs, each of which uses `request-cpus-importance-sampling` processes. In the backend, this makes use of the Result `split()` and `merge()` methods. ### Calibration marginalization @@ -132,7 +131,7 @@ spline-calibration-curves ## Plotting -The standard Result [plots](result.md#plotting) are turned on using the `plot-corner`, `plot-weights`, and `plot-log-probs` flags. +The standard Result [plots](result.qmd#plotting) are turned on using the `plot-corner`, `plot-weights`, and `plot-log-probs` flags. PP plots are produced using the `plot-pp` flag. Generally this should be done when analyzing many injections (`n-simulation > 1`). Dingo will produce two PP plots, one using unweighted and another using weighted samples (from importance sampling, labeled "IS"). The PP plot for unweighted samples is useful for gauging Dingo network performance, whereas the weighted PP plot tests consistency of the likelihood. diff --git a/docs/source/example_gnpe_model.md b/docs/source/example_gnpe_model.qmd similarity index 94% rename from docs/source/example_gnpe_model.md rename to docs/source/example_gnpe_model.qmd index c7f7b5991..7d0316b8a 100644 --- a/docs/source/example_gnpe_model.md +++ b/docs/source/example_gnpe_model.qmd @@ -1,8 +1,8 @@ # GNPE model (production) This tutorial has the highest profile settings and is the one typically used for production use. -The main difference from the [NPE](example_npe_model.md) tutorial is that here we are now using [GNPE](gnpe.md) -(group neural posterior estimation). The data generation is exactly the same as the [previous](example_npe_model.md) +The main difference from the [NPE](example_npe_model.qmd) tutorial is that here we are now using [GNPE](gnpe.qmd) +(group neural posterior estimation). The data generation is exactly the same as the [previous](example_npe_model.qmd) tutorial, but we repeat it here, for completeness. The file structure is similar to the NPE example, except now there are two @@ -114,7 +114,7 @@ dingo_train --settings_file train_settings.yaml --train_dir training/main_train_ ``` Notice the `data.gnpe_time_shifts` section. The `kernel` describes how much to blur the GNPE proxies and is specified in -seconds. To read more about this see [GNPE](gnpe.md). +seconds. To read more about this see [GNPE](gnpe.qmd). Step 4 Doing Inference diff --git a/docs/source/example_injection.md b/docs/source/example_injection.qmd similarity index 89% rename from docs/source/example_injection.md rename to docs/source/example_injection.qmd index 1f0ec73d8..0d78242c0 100644 --- a/docs/source/example_injection.md +++ b/docs/source/example_injection.qmd @@ -2,8 +2,8 @@ A simple example is creating an injection consistent with what the network was trained on, and then running Dingo on it. First one can instantiate -the {py:class}`dingo.gw.injection.Injection` using the metadata from the -{py:class}`dingo.core.models.posterior_model.PosteriorModel` (the trained network). An ASD dataset also needs to be specified, +the `dingo.gw.injection.Injection` using the metadata from the +`dingo.core.models.posterior_model.PosteriorModel` (the trained network). An ASD dataset also needs to be specified, one can take the fiducial asd dataset the network was trained on. ``` diff --git a/docs/source/example_npe_model.md b/docs/source/example_npe_model.qmd similarity index 96% rename from docs/source/example_npe_model.md rename to docs/source/example_npe_model.qmd index e340ea3b9..28395c3c1 100644 --- a/docs/source/example_npe_model.md +++ b/docs/source/example_npe_model.qmd @@ -1,9 +1,9 @@ # NPE Model (production) We will now do a tutorial with higher profile settings. Note these are not the -full production settings used for runs since we are not using [GNPE](gnpe.md), but -they should lead to decent results. Go to [this](example_gnpe_model.md) tutorial for the full production network. The -steps are the essentially same as [the toy example](example_toy_model.md) but with higher level settings. It is +full production settings used for runs since we are not using [GNPE](gnpe.qmd), but +they should lead to decent results. Go to [this](example_gnpe_model.qmd) tutorial for the full production network. The +steps are the essentially same as [the toy example](example_toy_npe_model.qmd) but with higher level settings. It is recommended to run this on a cluster or GPU machine. We can repeat the same first few steps from the previous tutorial with a couple @@ -123,10 +123,10 @@ but the settings are increased to production values. To run the training do dingo_train --settings_file train_settings.yaml --train_dir training ``` -```{tip} +::: {.callout-tip} If running on a machine with multiple GPUs make sure to specify the GPU by running export `CUDA_VISIBILE_DEVICES=GPU_NUM` before running `dingo_train` -``` +::: The main difference from the toy example in the network architecture is the size of the embedding network which is described in `model.embedding_net_kwargs.hidden_dims` and the @@ -135,18 +135,18 @@ number of neural spline flow transforms described in number/size of the layers in the embedding network. Notice, we are not inferring the phase parameter here as it is not listed below `inference_parameters`. However, -we do recover the phase in post processing. To see why and how this is done see [synthetic phase](result.md) +we do recover the phase in post processing. To see why and how this is done see [synthetic phase](result.qmd) Also notice there are now two training stages `stage_0` and `stage_1`. In `stage_0`, a fixed ASD is used and the reduced basis layer is frozen. Then in `stage_1`, all ASDs are used and the reduced basis layer is unfrozen. The main difference in the local settings is that the `device` is set to `CUDA`. -```{important} +::: {.callout-important} It is recommended to have at least 40 GB of GPU memory on the device. If there is not enough memory on the machine, first try halving the `batch_size`. In this case one should also multiply the learning rate, `lr`, by $\frac{1}{\sqrt{2}}$. If there is still not enough memory, consider reducing the number of hidden dimensions. -``` +::: Step 4 Doing Inference ---------------------- diff --git a/docs/source/example_toy_npe_model.md b/docs/source/example_toy_npe_model.qmd similarity index 88% rename from docs/source/example_toy_npe_model.md rename to docs/source/example_toy_npe_model.qmd index e5bde8988..064aa6a40 100644 --- a/docs/source/example_toy_npe_model.md +++ b/docs/source/example_toy_npe_model.qmd @@ -2,10 +2,10 @@ The goal of the following tutorial is to take a user from start to finish analyzing GW150914 using dingo. -```{caution} +::: {.callout-caution} This is only a toy example which is useful for testing on a local machine. This is NOT meant be used for production gravitational wave analyses. -``` +::: There are 4 main steps: @@ -14,7 +14,7 @@ There are 4 main steps: 3. Train the network 4. Do inference -In this tutorial as well as the [npe model](example_npe_model) and [gnpe model](example_gnpe_model) the following file structure will +In this tutorial as well as the [npe model](example_npe_model.qmd) and [gnpe model](example_gnpe_model.qmd) the following file structure will be employed ``` @@ -64,7 +64,7 @@ dingo_generate_dataset --settings waveform_dataset_settings.yaml --out_file trai ``` which will create a -{py:class}`dingo.gw.waveform_generator.waveform_generator.WaveformGenerator` +`dingo.gw.waveform_generator.waveform_generator.WaveformGenerator` object and store it at the location provided with `--out_file`. For convenience, here is the waveform dataset file @@ -104,10 +104,10 @@ The file `waveform_dataset_settings.yaml` contains four sections: `domain`, `waveform_generator`, `intrinsic_prior`, and `compression`. The domain section defines the settings for storing the waveform. Note the `type` attribute; this does not refer to the native domain of the waveform model, but -rather to the internal {py:class}`dingo.gw.domains.Domain` class. This allows the use +rather to the internal `dingo.gw.domains.Domain` class. This allows the use of time domain waveform models, which are transformed into Fourier domain before being passed to the network. Currently, only -the {py:class}`dingo.gw.domains.FrequencyDomain` class is supported for training the +the `dingo.gw.domains.FrequencyDomain` class is supported for training the network. It is sometimes advisable to generate waveforms with a higher `f_max` and then truncate them at a lower `f_max` for training due to issues with generating short waveforms for some of the waveform models implemented in LALSuite's LALSimulation package @@ -117,8 +117,8 @@ for some of the waveform models implemented in LALSuite's LALSimulation package The `waveform_generator` section specifies the `approximant` attribute. At present any waveform model, aka `approximant`, that is callable through LALSimulation's `SimInspiralFD` API can be used to generate waveforms for dingo via the -{py:class}`dingo.gw.waveform_generator.waveform_generator.WaveformGenerator` module (see -[generating_waveforms](generating_waveforms.md)). +`dingo.gw.waveform_generator.waveform_generator.WaveformGenerator` module (see +[generating_waveforms](generating_waveforms.ipynb)). The `intrinsic_prior` section is based on Bilby's prior module. Default values can be found in `dingo.gw.prior`. @@ -141,7 +141,7 @@ To generate an ASD dataset run dingo_generate_asd_dataset --settings_file asd_dataset_settings.yaml --data_dir training_data/asd_dataset ``` -This command will generate an {py:class}`dingo.gw.noise.asd_dataset.ASDDataset` object in the form of an .hdf5 file, which will be used later for training. The reason for specifying a folder instead of a file, as in the waveform dataset example, is because some temporary data is downloaded to create Welch estimates of the ASD. This data can be removed later, but it is sometimes useful for understanding how the ASDs were estimated. For convenience here is a copy of the `asd_dataset_settings.yaml` file. +This command will generate an `dingo.gw.noise.asd_dataset.ASDDataset` object in the form of an .hdf5 file, which will be used later for training. The reason for specifying a folder instead of a file, as in the waveform dataset example, is because some temporary data is downloaded to create Welch estimates of the ASD. This data can be removed later, but it is sometimes useful for understanding how the ASDs were estimated. For convenience here is a copy of the `asd_dataset_settings.yaml` file. ```yaml dataset_settings: @@ -159,7 +159,7 @@ detectors: observing_run: O1 ``` -The `asd_dataset_settings.yaml` file includes several attributes. `f_s` is the sampling frequency in Hz, `time_psd` is the length of time used for an ASD estimate, and `T` is the duration of each ASD segment. Thus, the value of `time_psd`/`T` gives the number of segments analyzed to estimate one ASD. To avoid spectral leakage, a window is applied to each segment. We use the standard window used in LVK analyses, a Tukey window with a roll off of $\alpha=0.4$. The next attribute, `num_psds_max=1`, defines the number of ASDs stored in the ASD dataset. For now, we will use only one. See the next [tutorial](example_npe_model.md) for a more advanced setup. +The `asd_dataset_settings.yaml` file includes several attributes. `f_s` is the sampling frequency in Hz, `time_psd` is the length of time used for an ASD estimate, and `T` is the duration of each ASD segment. Thus, the value of `time_psd`/`T` gives the number of segments analyzed to estimate one ASD. To avoid spectral leakage, a window is applied to each segment. We use the standard window used in LVK analyses, a Tukey window with a roll off of $\alpha=0.4$. The next attribute, `num_psds_max=1`, defines the number of ASDs stored in the ASD dataset. For now, we will use only one. See the next [tutorial](example_npe_model.qmd) for a more advanced setup. Step 3 Training the network --------------------------- @@ -170,7 +170,7 @@ To train the network, first the paths to the correct datasets must be specified dingo_train --settings_file train_settings.yaml --train_dir training ``` -While this file contains numerous settings that are discussed in [training](training.md), we will cover the most significant ones here. Again here is the file. +While this file contains numerous settings that are discussed in [training](training.qmd), we will cover the most significant ones here. Again here is the file. ```yaml @@ -265,7 +265,7 @@ Step 4 Doing Inference ---------------------- The final step is to do inference, for example on GW150914. To do this we will use -[dingo_pipe](dingo_pipe.md). For a local run execute: +[dingo_pipe](dingo_pipe.qmd). For a local run execute: ``` dingo_pipe GW150914.ini @@ -313,7 +313,7 @@ plot-weights = true plot-log-probs = true ``` -This will generate files which are described in [dingo_pipe](dingo_pipe.md). To see the results, take a look in `outdir_GW150914`. We set the flag `importance-sample = False` in the INI file, which disables importance sampling for this simple example. Generally one would omit this (it defaults to True). +This will generate files which are described in [dingo_pipe](dingo_pipe.qmd). To see the results, take a look in `outdir_GW150914`. We set the flag `importance-sample = False` in the INI file, which disables importance sampling for this simple example. Generally one would omit this (it defaults to True). We can load and manipulate the data with the following code. For example, here we create a cornerplot diff --git a/docs/source/gen_api.py b/docs/source/gen_api.py new file mode 100644 index 000000000..0fc111880 --- /dev/null +++ b/docs/source/gen_api.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Auto-generate the quartodoc API `sections` in _quarto.yml. + +The quartodoc analogue of running ``sphinx-apidoc``: walk the ``dingo`` package, +find every public module that defines classes or functions, and list them (grouped +by sub-package) so quartodoc documents all their members. New modules appear +automatically; no hand-maintained object list. + +Run from ``docs/source``:: + + python gen_api.py + +then ``quartodoc build`` && ``quarto render`` as usual. +""" +import os +import re + +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.abspath(os.path.join(HERE, "..", "..")) +PKG_DIR = os.path.join(REPO, "dingo") +QUARTO_YML = os.path.join(HERE, "_quarto.yml") + +# Sub-packages to skip entirely (CLI/orchestration layers, not a library API). +SKIP_TOP = {"pipe"} + + +def discover(): + """Return {group: [module.dotted.path, ...]} for public modules with an API.""" + groups = {} + for root, dirs, files in os.walk(PKG_DIR): + dirs[:] = sorted( + d for d in dirs if not d.startswith((".", "_")) and d != "tests" + ) + for fname in sorted(files): + if not fname.endswith(".py") or fname.startswith(("_", "test")): + continue + path = os.path.join(root, fname) + rel = os.path.relpath(path, PKG_DIR)[:-3].replace(os.sep, ".") # core.nn.enets + parts = rel.split(".") + if parts[0] in SKIP_TOP: + continue + src = open(path, encoding="utf-8", errors="ignore").read() + if not re.search(r"^(class|def)\s+[A-Za-z]", src, re.M): + continue # no public classes/functions -> nothing to document + group = ".".join(parts[:2]) if len(parts) > 1 else parts[0] + groups.setdefault(group, []).append(rel) + return groups + + +def build_sections(groups): + lines = [] + for group in sorted(groups): + lines.append(f' - title: "dingo.{group}"') + lines.append(" contents:") + for mod in sorted(groups[group]): + lines.append(f" - name: {mod}") + lines.append(" children: linked") + return "\n".join(lines) + + +def main(): + groups = discover() + sections = build_sections(groups) + text = open(QUARTO_YML, encoding="utf-8").read() + new = re.sub( + r"( # BEGIN AUTOGEN SECTIONS[^\n]*\n).*?( # END AUTOGEN SECTIONS)", + lambda m: m.group(1) + sections + "\n" + m.group(2), + text, + flags=re.S, + ) + if new == text or "BEGIN AUTOGEN" not in new: + raise SystemExit("AUTOGEN markers not found in _quarto.yml") + open(QUARTO_YML, "w", encoding="utf-8").write(new) + n_mods = sum(len(v) for v in groups.values()) + print(f"Wrote {n_mods} modules across {len(groups)} groups to _quarto.yml") + + +if __name__ == "__main__": + main() diff --git a/docs/source/generating_waveforms.ipynb b/docs/source/generating_waveforms.ipynb index a4df018d9..4330c545e 100644 --- a/docs/source/generating_waveforms.ipynb +++ b/docs/source/generating_waveforms.ipynb @@ -96,7 +96,14 @@ "metadata": { "tags": [] }, - "source": "```{note}\nThe standard deviation of white noise in each frequency bin is stored in `domain.noise_std`. In frequency domain, this is given by $\\sqrt{1/4\\delta f}$. See [data conditioning](ref:window-factor) for details.\n```\n\nVarious class methods also act on data, to perform operations such as zeroing below `f_min`, truncating above `f_max`, or applying a time shift:\n\n```{eval-rst}\n.. autoclass:: dingo.gw.domains.UniformFrequencyDomain\n :members:\n```" + "source": [ + "::: {.callout-note}\n", + "The standard deviation of white noise in each frequency bin is stored in `domain.noise_std`. In frequency domain, this is given by $\\sqrt{1/4\\delta f}$. See [data conditioning](ref:window-factor) for details.\n", + "\n", + ":::\n", + "\n", + "Various class methods also act on data, to perform operations such as zeroing below `f_min`, truncating above `f_max`, or applying a time shift:" + ] }, { "attachments": {}, @@ -301,11 +308,7 @@ "source": [ "Note that the waveform is nonzero slightly below `f_min`. This simply arises from the model implementation in `LALSimulation`. When training networks, input data will be truncated below `f_min`.\n", "\n", - "The complete specification of the `WaveformGenerator` class is given as\n", - "```{eval-rst}\n", - ".. autoclass:: dingo.gw.waveform_generator.WaveformGenerator\n", - " :members:\n", - "```" + "The complete specification of the `WaveformGenerator` class is given as" ] }, { @@ -350,4 +353,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/docs/source/gnpe.md b/docs/source/gnpe.qmd similarity index 90% rename from docs/source/gnpe.md rename to docs/source/gnpe.qmd index 189b8009e..8a971d7b9 100644 --- a/docs/source/gnpe.md +++ b/docs/source/gnpe.qmd @@ -1,6 +1,6 @@ # GNPE -GNPE (Gibbs- or Group-Equivariant Neural Posterior Estimation) is an algorithm that can generate significantly improved results by incorporating known physical symmetries into NPE.{footcite:p}`Dax:2021myb` The aim is to simplify the data seen by the network by using the symmetries to transform certain parameters to "standardized" values. This simplifies the learning task of the network. At inference time, the standardizing transform is initially unknown, so we use Gibbs sampling to simultaneously learn the transform (along with the rest of the parameters) *and* apply it to simplify the data. +GNPE (Gibbs- or Group-Equivariant Neural Posterior Estimation) is an algorithm that can generate significantly improved results by incorporating known physical symmetries into NPE. [@Dax:2021myb] The aim is to simplify the data seen by the network by using the symmetries to transform certain parameters to "standardized" values. This simplifies the learning task of the network. At inference time, the standardizing transform is initially unknown, so we use Gibbs sampling to simultaneously learn the transform (along with the rest of the parameters) *and* apply it to simplify the data. For gravitational waves, we use GNPE to standardize the times of arrival of the signal in the individual interferometers. (This corresponds to translations of the time of arrival at geocenter, and approximate sky rotations.) In frequency domain, time translations correspond to multiplication of the data by $e^{-2\pi i f \Delta t}$, and a standard NPE network would have to learn to interpret such transformations consistent with the prior from the data. We found this to be a challenging learning task, which limited inference performance on the other parameters. Instead, GNPE leverages our knowledge of the time translations to build a network that is only required to interpret a much narrower window of arrival times. @@ -25,12 +25,7 @@ It is based on two ideas: and repeating until the chain is converged. The stationary distribution of the Markov chain is then $p(x, y)$. -```{figure} gibbs_figure.jpg ---- -height: 300px ---- -Illustration of Gibbs sampling for a distribution $p(x, y)$. -``` +![Illustration of Gibbs sampling for a distribution $p(x, y)$.](gibbs_figure.jpg) Gibbs sampling can be combined with NPE by first introducing blurred "proxy" versions of a subset of parameters, which we denote $\hat\theta$ i.e., $\hat\theta \sim p(\hat\theta | \theta)$ where $p(\hat\theta | \theta)$ is defined by a blurring kernel. For example, for GWs we take $\hat t_I = t_I + \epsilon_I$, where $\epsilon_I \sim \text{Unif}(-1~\text{ms}, 1~\text{ms})$. We then train a network to model the posterior, but now conditioned also on $\hat \theta$, i.e., $p(\theta | d, \hat\theta)$. We can then apply Gibbs sampling to obtain samples from the joint distribution $p(\theta, \hat \theta | d)$, since we are able to sample individually from the conditional distributions: @@ -49,12 +44,12 @@ So far we have described how Gibbs sampling together with NPE can simplify data For gravitational waves, the overall time translation symmetry (in each detector) of the time of coalescence at geocenter is an exact symmetry, so we fully enforce this. The sky rotation, however, corresponds to an approximate symmetry: it shifts the time of coalescence in each detector, but a subleading effect is to change angle of incidence on a detector and hence the combination of polarizations observed. For this latter symmetry, we simply do not drop the proxy dependence. -```{tip} +::: {.callout-tip} GNPE is a generic method to incorporate symmetries into NPE: * **Any** symmetry (exact or approximate) connecting data and parameters * **Any** architecture, as it just requires (at most) conditioning on the proxy variables -``` +::: As far as we are aware, GNPE is the only way to incorporate symmetries connecting data and parameters into architectures such as normalizing flows. @@ -108,14 +103,8 @@ The number of Gibbs iterations is also specified here (typically 30 is appropria ## The `GNPESampler` class The inference script above uses the `GWSamplerGNPE` class, which is based on `GNPESampler`, -```{eval-rst} -.. autoclass:: dingo.core.samplers.GNPESampler - :members: - :inherited-members: - :show-inheritance: -``` +see the [`GNPESampler`](reference/core.samplers.GNPESampler.qmd) API reference for the full list of methods. In addition to storing a `PosteriorModel`, a `GNPESampler` also stores a second `Sampler` instance, which is based on the initialization network. When `run_sampler()` is called, it first generates samples from the initialization network, perturbs them with the kernel to obtain proxy samples, and then performs `num_iterations` Gibbs steps to obtain the final samples. -```{eval-rst} -.. footbibliography:: -``` \ No newline at end of file +::: {#refs} +::: \ No newline at end of file diff --git a/docs/source/index.md b/docs/source/index.md deleted file mode 100644 index 9a254e776..000000000 --- a/docs/source/index.md +++ /dev/null @@ -1,108 +0,0 @@ - - -Dingo -===== - -**Dingo (Deep Inference for Gravitational-wave Observations)** is a Python program for analyzing gravitational wave data using neural posterior -estimation. It dramatically speeds up inference of astrophysical source parameters from -data measured at gravitational-wave observatories. Dingo aims to enable the routine -use of the most advanced theoretical models in analysing data, to make rapid predictions -for multi-messenger counterparts, and to do so in the context of sensitive detectors with -high event rates. - -The basic approach of Dingo is to *train a neural network to represent the Bayesian -posterior*, conditioned on data. This enables **amortized inference**: when new data are -observed, they can be plugged in and results obtained in a small amount of time. Tasks -handled by Dingo include - -* [building training datasets](waveform_dataset.ipynb); -* [training](training.md) normalizing flows to estimate the posterior density; -* [performing inference](inference.md) on real or simulated data; and -* verifying and correcting model results using [importance sampling](result.md#importance-sampling). - -As training a network from scratch can be expensive, we intend to also distribute trained networks that can be used directly for inference. These can be used with [dingo_pipe](dingo_pipe.md) to automate analysis of gravitational wave events. - -```{eval-rst} -.. toctree:: - :caption: Getting started - :maxdepth: 1 - - installation - overview - quickstart - -.. toctree:: - :caption: Examples - :maxdepth: 1 - - example_toy_npe_model - example_npe_model - example_gnpe_model - example_injection - -.. toctree:: - :caption: Advanced guide - :maxdepth: 1 - - sbi - code_design - generating_waveforms - waveform_dataset - training_transforms - noise_dataset - network_architecture - training - inference - gnpe - result - dingo_pipe - asimov - -.. toctree:: - :caption: API - :maxdepth: 1 - - modules -``` - -References ----------- - -Dingo is based on a series of papers developing neural posterior estimation for gravitational waves, starting from proof of concept {cite:p}`Green:2020hst`, to inclusion of all 15 parameters and analysis of real data {cite:p}`Green:2020dnx`, noise conditioning and full amortization {cite:p}`Dax:2021tsq`, and group-equivariant NPE {cite:p}`Dax:2021myb`. Dingo results are augmented with importance sampling in {cite:p}`Dax:2022pxd`. Finally, training with forecasted noise (needed for training *prior* to an observing run) is described in {cite:p}`Wildberger:2022agw`. - -```{eval-rst} -.. bibliography:: -``` - -If you use Dingo in your work, we ask that you please cite at least {cite:p}`Dax:2021tsq`. - -Contributors to the code are listed in [AUTHORS.md](https://github.com/dingo-gw/dingo/blob/main/AUTHORS.md). We thank Vivien Raymond -and Rory Smith for acting as LIGO-Virgo-KAGRA (LVK) code reviewers. Dingo makes use of -many LVK software tools, including [Bilby](https://lscsoft.docs.ligo.org/bilby/), -[bilby_pipe](https://lscsoft.docs.ligo.org/bilby_pipe/master/index.html), and -[LALSimulation](https://lscsoft.docs.ligo.org/lalsuite/lalsimulation/), as well as third -party tools such as [PyTorch](https://pytorch.org) and -[nflows](https://github.com/bayesiains/nflows). - -Contact -------- - -For questions or comments please contact -[Maximilian Dax](mailto:maximilian.dax@tuebingen.mpg.de) or -[Stephen Green](mailto:stephen.green2@nottingham.ac.uk). - -Indices and tables ------------------- - -```{eval-rst} -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` -``` \ No newline at end of file diff --git a/docs/source/index.qmd b/docs/source/index.qmd new file mode 100644 index 000000000..8a15c362e --- /dev/null +++ b/docs/source/index.qmd @@ -0,0 +1,46 @@ +Dingo +===== + +**Dingo (Deep Inference for Gravitational-wave Observations)** is a Python program for analyzing gravitational wave data using neural posterior +estimation. It dramatically speeds up inference of astrophysical source parameters from +data measured at gravitational-wave observatories. Dingo aims to enable the routine +use of the most advanced theoretical models in analysing data, to make rapid predictions +for multi-messenger counterparts, and to do so in the context of sensitive detectors with +high event rates. + +The basic approach of Dingo is to *train a neural network to represent the Bayesian +posterior*, conditioned on data. This enables **amortized inference**: when new data are +observed, they can be plugged in and results obtained in a small amount of time. Tasks +handled by Dingo include + +* [building training datasets](waveform_dataset.ipynb); +* [training](training.qmd) normalizing flows to estimate the posterior density; +* [performing inference](inference.qmd) on real or simulated data; and +* verifying and correcting model results using [importance sampling](result.qmd#importance-sampling). + +As training a network from scratch can be expensive, we intend to also distribute trained networks that can be used directly for inference. These can be used with [dingo_pipe](dingo_pipe.qmd) to automate analysis of gravitational wave events. + +References +---------- + +Dingo is based on a series of papers developing neural posterior estimation for gravitational waves, starting from proof of concept [@Green:2020hst], to inclusion of all 15 parameters and analysis of real data [@Green:2020dnx], noise conditioning and full amortization [@Dax:2021tsq], and group-equivariant NPE [@Dax:2021myb]. Dingo results are augmented with importance sampling in [@Dax:2022pxd]. Finally, training with forecasted noise (needed for training *prior* to an observing run) is described in [@Wildberger:2022agw]. + +::: {#refs} +::: + +If you use Dingo in your work, we ask that you please cite at least [@Dax:2021tsq]. + +Contributors to the code are listed in [AUTHORS.md](https://github.com/dingo-gw/dingo/blob/main/AUTHORS.md). We thank Vivien Raymond +and Rory Smith for acting as LIGO-Virgo-KAGRA (LVK) code reviewers. Dingo makes use of +many LVK software tools, including [Bilby](https://lscsoft.docs.ligo.org/bilby/), +[bilby_pipe](https://lscsoft.docs.ligo.org/bilby_pipe/master/index.html), and +[LALSimulation](https://lscsoft.docs.ligo.org/lalsuite/lalsimulation/), as well as third +party tools such as [PyTorch](https://pytorch.org) and +[nflows](https://github.com/bayesiains/nflows). + +Contact +------- + +For questions or comments please contact +[Maximilian Dax](mailto:maximilian.dax@tuebingen.mpg.de) or +[Stephen Green](mailto:stephen.green2@nottingham.ac.uk). \ No newline at end of file diff --git a/docs/source/inference.md b/docs/source/inference.qmd similarity index 88% rename from docs/source/inference.md rename to docs/source/inference.qmd index 767137d80..9e29279c8 100644 --- a/docs/source/inference.md +++ b/docs/source/inference.qmd @@ -1,20 +1,15 @@ # Inference With a trained network, inference can be performed on injections or real data. For -injections, see the [discussion in the examples](example_injection.md). For real data, we -recommend to use [dingo_pipe](dingo_pipe.md). +injections, see the [discussion in the examples](example_injection.qmd). For real data, we +recommend to use [dingo_pipe](dingo_pipe.qmd). ## The `Sampler` class Inference uses the `Sampler` class, or more specifically, the `GWSampler` class, which inherits from it. -```{eval-rst} -.. autoclass:: dingo.gw.inference.gw_samplers.GWSampler - :members: - :inherited-members: - :show-inheritance: -``` +See the [`GWSampler`](reference/gw.inference.gw_samplers.GWSampler.qmd) API reference for the full list of methods. This is instantiated based on a `PosteriorModel`. To draw samples, the `context` property must first be set to the data to be analyzed. For gravitational waves this should be a dictionary with the following keys: @@ -36,16 +31,12 @@ The `GWSampler.metadata` attribute contains all settings that went into producin Injections (i.e., simulated data) are produced using the `Injection` class. It includes options for fixed or random parameters (drawn from a prior), and it returns injections in a format that can be directly set as `GWSampler.context`. -```{eval-rst} -.. autoclass:: dingo.gw.injection.Injection - :members: - :show-inheritance: -``` +See the [`Injection`](reference/gw.injection.Injection.qmd) API reference for the full list of methods. -```{hint} +::: {.callout-tip} The convenience class method `from_posterior_model_metadata()` instantiates an `Injection` with all of the settings that went into the posterior model. To this class pass the PosteriorModel.metadata dictionary. It should produce injections that perfectly match the characteristics of the training data (waveform approximant, data conditioning, noise characteristics, etc.). This can be very useful for testing a trained model. -``` +::: -```{important} +::: {.callout-important} Repeated calls to `Injection.injection()`, even with the same parameters, will produce injections with different noise realizations (which therefore lead to different posteriors). For repeated analyses of the *exact same* injection (e.g., with different models or codes) it is necessary to either save the injection for re-use or fix a random seed. -``` \ No newline at end of file +::: \ No newline at end of file diff --git a/docs/source/installation.md b/docs/source/installation.qmd similarity index 66% rename from docs/source/installation.md rename to docs/source/installation.qmd index f46167369..13ebd8be6 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.qmd @@ -77,36 +77,26 @@ pip install -e ".[dev,wandb]" ### Documentation -To build the documentation, first generate the API documentation using [autodoc](https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html): +The documentation is built with [Quarto](https://quarto.org) and +[`quartodoc`](https://machow.github.io/quartodoc/), which generates the API reference from +the package docstrings. First [install Quarto](https://quarto.org/docs/download/), then +install `quartodoc`: ```sh -cd docs -sphinx-apidoc -o source ../dingo +pip install quartodoc ``` -This will create `dingo.*.rst` and `modules.rst` files in `source/`. These correspond to -the various modules and are constructed from docstrings. -To finally compile the documentation, run +The documentation sources live in `docs/source`. From that directory, first generate the +API reference pages from the docstrings: ```sh -make html +quartodoc build ``` -This creates a directory `build/` containing HTML documentation. The main index is at `build/html/index.html`. - -To use the autodoc feature, which works for pycharm and numpy docstrings, insert in a .rst file, e.g., - -``` -.. autofunction:: dingo.core.utils.trainutils.write_history` -``` - -This will render as - -```{eval-rst} -.. autofunction:: dingo.core.utils.trainutils.write_history +This writes the `reference/` pages. Then render the site: +```sh +quarto render ``` - -#### Cleanup - -To remove generated docs, execute +The HTML is written to `docs/source/_site`, with the main index at +`docs/source/_site/index.html`. While editing, run ```sh -make clean -rm source/dingo.* source/modules.rst +quarto preview ``` +to serve the documentation locally with live reload. diff --git a/docs/source/network_architecture.ipynb b/docs/source/network_architecture.ipynb index 97b2025ea..52d2d4159 100644 --- a/docs/source/network_architecture.ipynb +++ b/docs/source/network_architecture.ipynb @@ -7,10 +7,10 @@ "source": [ "# Neural network architecture\n", "\n", - "Dingo employs different network architectures for posterior estimation, e.g. [Neural posterior estimation](https://arxiv.org/abs/1605.06376) and [Flow Matching Posterior estimation](https://arxiv.org/pdf/2305.17161), see [here](sbi.md) for an introduction. A central object is the conditional neural density estimator, a deep neural network trained to represent the Bayesian posterior. This section describes the neural network architecture developed in {cite:p}`Dax:2021tsq`, and subsequently used in {cite:p}`Dax:2021myb`, {cite:p}`Dax:2022pxd` and {cite:p}`Wildberger:2022agw`.\n", + "Dingo employs different network architectures for posterior estimation, e.g. [Neural posterior estimation](https://arxiv.org/abs/1605.06376) and [Flow Matching Posterior estimation](https://arxiv.org/pdf/2305.17161), see [here](sbi.qmd) for an introduction. A central object is the conditional neural density estimator, a deep neural network trained to represent the Bayesian posterior. This section describes the neural network architecture developed in [@Dax:2021tsq], and subsequently used in [@Dax:2021myb], [@Dax:2022pxd] and [@Wildberger:2022agw].\n", "\n", "## Neural spline flow with SVD compression\n", - "The NPE architecture consists of two compenents, the embedding network which compresses the high-dimensionl data to a lower dimensional feature vector, and the conditional normalizing flow which estimates the Bayesian posterior based on this feature vector. Both components are trained jointly and end-to-end with the objective descriped [here](sbi.md). The network can be build with" + "The NPE architecture consists of two compenents, the embedding network which compresses the high-dimensionl data to a lower dimensional feature vector, and the conditional normalizing flow which estimates the Bayesian posterior based on this feature vector. Both components are trained jointly and end-to-end with the objective descriped [here](sbi.qmd). The network can be build with" ] }, { @@ -74,9 +74,10 @@ "id": "18525738", "metadata": {}, "source": [ - "```{note}\n", - "Not all of these arguments have to be set in the configuration file when training dingo. For example, the `input_dims` argument is automatically filled in based on the specified domain information and number of detectors. Similarly, the `context_dim` of the flow (see below) is filled in based on the `output_dim` of the embedding network and the number of [GNPE](gnpe.md) proxies. See the [Dingo examples](https://github.com/dingo-gw/dingo/tree/main/examples) for the corresponding configuration files and training commands.\n", - "```" + "::: {.callout-note}\n", + "Not all of these arguments have to be set in the configuration file when training dingo. For example, the `input_dims` argument is automatically filled in based on the specified domain information and number of detectors. Similarly, the `context_dim` of the flow (see below) is filled in based on the `output_dim` of the embedding network and the number of [GNPE](gnpe.qmd) proxies. See the [Dingo examples](https://github.com/dingo-gw/dingo/tree/main/examples) for the corresponding configuration files and training commands.\n", + "\n", + ":::" ] }, { @@ -116,7 +117,7 @@ "id": "fc24eb0d", "metadata": {}, "source": [ - "This creates a neural spline flow with `input_dim=15` parameters, conditioned on a `129` dimensional context vector, corresponding to the `128` dimensional output of the embedding network and one [GNPE](gnpe.md) proxy variable. The neural spline flow consists of `num_flow_steps=30` layers, for which the transformation is specified with `base_transform_kwargs`." + "This creates a neural spline flow with `input_dim=15` parameters, conditioned on a `129` dimensional context vector, corresponding to the `128` dimensional output of the embedding network and one [GNPE](gnpe.qmd) proxy variable. The neural spline flow consists of `num_flow_steps=30` layers, for which the transformation is specified with `base_transform_kwargs`." ] }, { diff --git a/docs/source/noise_dataset.ipynb b/docs/source/noise_dataset.ipynb index f1beb7571..f95637f7a 100644 --- a/docs/source/noise_dataset.ipynb +++ b/docs/source/noise_dataset.ipynb @@ -14,9 +14,11 @@ "$$\n", "\n", "Dingo assumes this noise to be stationary and Gaussian, thus it is independent in each frequency bin, with variance given by some power spectral density (PSD).\n", - "```{important}\n", + "\n", + "::: {.callout-important}\n", "Similar to extrinsic parameters, detector noise is repeatedly sampled **during training** and added to the simulated signal. This augments the training set with new noise realizations for each epoch, reducing overfitting. \n", - "```\n", + "\n", + ":::\n", "\n", "Although noise is *mostly* stationary and Guassian during an LVK observing run, the PSD in each detector does tend to drift from event to event. In a usual likelihood-based PE run, this is taken into account by estimating the PSD at the time of the event (either using [Welch's method](https://en.wikipedia.org/wiki/Welch%27s_method) on signal-free data surrounding the event, or at the same time as the event using [BayesWave](https://git.ligo.org/lscsoft/bayeswave)), and using this in the likelihood integral.\n", "\n", @@ -33,13 +35,6 @@ "\n", "The `ASDDataset` class stores a set of ASD samples for several detectors, allowing for sampling during training.\n", "\n", - "```{eval-rst}\n", - ".. autoclass:: dingo.gw.ASD_dataset.noise_dataset.ASDDataset\n", - " :members:\n", - " :inherited-members:\n", - " :show-inheritance:\n", - "```\n", - "\n", "As with the noise realizations, a random ASD is chosen from the dataset when preparing each sample during training. This augments the training set compared to fixing the noise ASD for each sample prior to training.\n", "\n", "Similarly to the `WaveformDataset`, the `ASDDataset` is just a container. Dingo includes routines for building such a dataset from observational data.\n", diff --git a/docs/source/overview.md b/docs/source/overview.qmd similarity index 70% rename from docs/source/overview.md rename to docs/source/overview.qmd index ae4388584..e2c5f4886 100644 --- a/docs/source/overview.md +++ b/docs/source/overview.qmd @@ -1,26 +1,25 @@ # Overview -Dingo performs gravitational-wave (GW) parameter estimation using [**neural posterior estimation**](sbi.md). The basic idea is to train a neural network (a normalizing flow) to represent the Bayesian posterior distribution $p(\theta|d)$ for GW parameters $\theta$ given observed data $d$. Training can take some time (typically, a week for a production-level model) but once trained, inference is very fast (just a few seconds). +Dingo performs gravitational-wave (GW) parameter estimation using [**neural posterior estimation**](sbi.qmd). The basic idea is to train a neural network (a normalizing flow) to represent the Bayesian posterior distribution $p(\theta|d)$ for GW parameters $\theta$ given observed data $d$. Training can take some time (typically, a week for a production-level model) but once trained, inference is very fast (just a few seconds). ## Basic workflow The basic workflow for using Dingo is as follows: 1. **Prepare training data.** This consists of pairs of intrinsic parameters and [waveform polarizations](waveform_dataset.ipynb), as well as [noise PSDs](noise_dataset.ipynb). Training parameters are drawn from the prior distribution, and [waveforms are simulated](generating_waveforms.ipynb) using a waveform model. -2. **Train a model.** [Build a neural network](network_architecture.ipynb) and [simulate data sets](training_transforms.ipynb) (noisy waveforms in detectors). [Train the model](training.md) to infer parameters based on the data. -3. **[Perform inference](dingo_pipe.md) on new data** using the trained model. +2. **Train a model.** [Build a neural network](network_architecture.ipynb) and [simulate data sets](training_transforms.ipynb) (noisy waveforms in detectors). [Train the model](training.qmd) to infer parameters based on the data. +3. **[Perform inference](dingo_pipe.qmd) on new data** using the trained model. In many cases, a user may have downloaded a pre-trained model. If so, there is no need to carry out the first two steps, and one may instead skip to **step 3**. ## Command-line interface -In most cases, we expect Dingo to be called from the command line. Dingo commands begin with the prefix `dingo_`. There can be a large number of configurations options for many tasks, so in such cases, rather than specify all settings as arguments, Dingo commands take a single YAML or INI file containing all settings. As described in the [quickstart tutorial](quickstart.md), it is best to begin with settings files provided in the [examples/](https://github.com/dingo-gw/dingo/tree/main/examples) folder, modifying them as necessary. +In most cases, we expect Dingo to be called from the command line. Dingo commands begin with the prefix `dingo_`. There can be a large number of configurations options for many tasks, so in such cases, rather than specify all settings as arguments, Dingo commands take a single YAML or INI file containing all settings. As described in the [quickstart tutorial](quickstart.qmd), it is best to begin with settings files provided in the [examples/](https://github.com/dingo-gw/dingo/tree/main/examples) folder, modifying them as necessary. ### Summary of commands Here we provide a list of key user commands along with brief descriptions. The commands for carrying out the main tasks above are -```{table} | Command | Description | |---|---| @@ -28,44 +27,39 @@ Here we provide a list of key user commands along with brief descriptions. The c |`dingo_generate_ASD_dataset`| Generate a training dataset of detector noise ASDs. | |`dingo_train`| Build and train a neural network. | |`dingo_pipe`| Perform inference on data (real or simulated), starting from an INI file. | -``` Building a training dataset and training a model can be very expensive tasks. We therefore expect these to be frequently run on clusters, and for this reason provided [HTCondor](https://htcondor.readthedocs.io/en/latest/) versions of these commands (note that `dingo_pipe` is already HTCondor-compatible): -```{table} | Command | Description | |---|---| |`dingo_generate_dataset_dag`| HTCondor version of `dingo_generate_dataset`. | |`dingo_train_condor`| HTCondor version of `dingo_train`. | -``` Finally, there are several utility commands that are useful for working with Dingo-produced files: -```{table} | Command | Description | |---|---| |`dingo_ls`| Inspect a file produced by Dingo and print a summary.| |`dingo_append_training_stage`| Modify the training plan of a model checkpoint.| |`dingo_pt_to_hdf5`| Convert a trained Dingo model from a PyTorch pickle .pt file to HDF5.| -``` -```{hint} +::: {.callout-tip} The `dingo_ls` command is very useful for inspecting Dingo files. It will print all settings that went in to producing the file, as well as some derived quantities. -``` +::: ### File types As noted above, most Dingo commands take a YAML file to specify configuration options (except for `dingo_pipe`, which uses an INI file, as is standard for LVK parameter estimation). When run, these commands generate data, which is usually stored in HDF5 files. One exception is when training a neural network. This saves the network weights using the PyTorch `.pt` format. However, primarily for LVK use, `dingo_pt_to_hdf5` can convert the weights of a trained model to a HDF5 file. -```{important} +::: {.callout-important} In all cases, Dingo will save the YAML file settings within the final output file. This is needed for downstream tasks and for maintaining reproducibility. -``` +::: ## GNPE -A slightly more complicated workflow occurs when using [](gnpe.md). GNPE is an algorithm that combines physical symmetries with Gibbs sampling to significantly improve results. When using GNPE, however, it is necessary to train **two networks**---one main (conditional) network that will be repeatedly sampled during Gibbs sampling and one smaller network used to initialize the Gibbs sampler. At inference time, `dingo_pipe` must be pointed to **both** of these networks. See the section on [GNPE usage](gnpe.md#usage) for further details. +A slightly more complicated workflow occurs when using [](gnpe.qmd). GNPE is an algorithm that combines physical symmetries with Gibbs sampling to significantly improve results. When using GNPE, however, it is necessary to train **two networks**---one main (conditional) network that will be repeatedly sampled during Gibbs sampling and one smaller network used to initialize the Gibbs sampler. At inference time, `dingo_pipe` must be pointed to **both** of these networks. See the section on [GNPE usage](gnpe.qmd#usage) for further details. diff --git a/docs/source/quickstart.md b/docs/source/quickstart.qmd similarity index 97% rename from docs/source/quickstart.md rename to docs/source/quickstart.qmd index 8ac4a7447..04c6a70aa 100644 --- a/docs/source/quickstart.md +++ b/docs/source/quickstart.qmd @@ -107,7 +107,7 @@ rather to get a feel for the Dingo pipeline. The production settings contain tes settings. Note that depending on the waveform model and event, these may need to occasionally be tuned. `train_settings_init_toy.yaml` and `train_settings_init_production.yaml` train flows to estimate the time of coalescence in the individual detectors. These two -networks are needed to use [GNPE](gnpe.md). This is the preferred and +networks are needed to use [GNPE](gnpe.qmd). This is the preferred and most tested way of using Dingo. Alternatively, the `train_settings_no_gnpe_toy.yaml` and @@ -119,7 +119,7 @@ new ideas or doing a less expensive training. ## Inference Once a Dingo model is trained, inference for real events can be performed using -[dingo_pipe](dingo_pipe.md). There are 3 main inference steps, downloading the data, +[dingo_pipe](dingo_pipe.qmd). There are 3 main inference steps, downloading the data, running Dingo on this data and finally running importance sampling. The basic idea is to create a .ini file which contains the filepaths of the Dingo networks trained above and the segment of data to analyze. An example .ini file can be diff --git a/docs/source/result.md b/docs/source/result.qmd similarity index 94% rename from docs/source/result.md rename to docs/source/result.qmd index 261c2ab6a..1c2dae63c 100644 --- a/docs/source/result.md +++ b/docs/source/result.qmd @@ -2,12 +2,7 @@ The `Result` class stores the output of a `Sampler` run, namely a collection of samples. It contains several methods for operating on the samples, including for **importance sampling**, **plotting**, and **density recovery**: -```{eval-rst} -.. autoclass:: dingo.gw.result.Result - :members: - :inherited-members: - :show-inheritance: -``` +See the [`Result`](reference/gw.result.Result.qmd) API reference for the full list of methods. Following a sampler run, a `Result` can be obtained using `Sampler.to_result()`. Since `Result` inherits from `DingoDataset` it also possesses `to_file()` and `to_dictionary()` methods for saving samples and associated metadata (including context data, namely event data and ASDs). @@ -21,15 +16,15 @@ The `Result` class contains methods to enable *recovery* of the probability dens 2. Train an *unconditional* density estimator $q(\hat\theta)$ to model the proxy parameters. This is done by (1) using `parameter_subset()` to produce a new `Result` containing just the proxies, and (2) using `train_unconditional_flow()` on this subset. 3. Generate new samples $(\theta, \hat\theta) \sim q(\theta, \hat\theta | d) = q(\theta | d, \hat\theta) q(\hat\theta)$. This can be accomplished using `GNPESampler.sample()` with `num_iterations = 1` and setting the initial sampler to be the unconditional flow trained in the previous step. Since this does not involve multiple iterations, the density is obtained as well, so importance sampling can be performed. -```{note} +::: {.callout-note} Density recovery can also be achieved using an unconditional density estimator for $\theta$ (trained on samples $\{\theta_i\}_{i=1}^N$ from GNPE). Since $\theta$ typically comprises 14 parameters (versus 2 or 3 for $\hat\theta$) it is usually more straightforward to learn the proxies. -``` +::: ## Synthetic phase It is often challenging for Dingo to learn to model the `phase` parameter $\phi_c$. For this reason, we usually marginalize over it in training by excluding it from the list of `inference_parameters`. The phase is, however, required for importance sampling unless using also a phase-marginalized likelihood (which is approximate except under special circumstances). -The Dingo `gw.Result` class includes a method `sample_synthetic_phase()` which produces a $\phi_c$ sample from a $\phi_c$-marginalized sample. It does so by evaluating the likelihood on a $\phi_c$-grid and then sampling from the associated 1D distribution. The `log_prob` value for the sample is also corrected to reflect the sampled $\phi_c$. Speed is ensured by caching waveform modes and evaluating the polarizations for different $\phi_c$. For further details, see the Supplemental Material of {cite:p}`Dax:2022pxd`. +The Dingo `gw.Result` class includes a method `sample_synthetic_phase()` which produces a $\phi_c$ sample from a $\phi_c$-marginalized sample. It does so by evaluating the likelihood on a $\phi_c$-grid and then sampling from the associated 1D distribution. The `log_prob` value for the sample is also corrected to reflect the sampled $\phi_c$. Speed is ensured by caching waveform modes and evaluating the polarizations for different $\phi_c$. For further details, see the Supplemental Material of [@Dax:2022pxd]. This method should be run *after* recovering the density, since in particular it applies a correction to the density. @@ -56,7 +51,7 @@ num_processes ## Importance sampling -Once samples are in the right form---including all relevant parameters *and* the log probability---importance sampling is carried out using the `importance_sample()` method. It allows to specify options for using a marginalized likelihood. (Time and phase marginalization are separately supported; see the documentation of {py:class}`dingo.gw.likelihood.StationaryGaussianGWLikelihood`.) +Once samples are in the right form---including all relevant parameters *and* the log probability---importance sampling is carried out using the `importance_sample()` method. It allows to specify options for using a marginalized likelihood. (Time and phase marginalization are separately supported; see the documentation of `dingo.gw.likelihood.StationaryGaussianGWLikelihood`.) As with the synthetic phase, importance sampling allows for parallelization. diff --git a/docs/source/sbi.md b/docs/source/sbi.qmd similarity index 88% rename from docs/source/sbi.md rename to docs/source/sbi.qmd index 6c215bf7a..1b4ae91e6 100644 --- a/docs/source/sbi.md +++ b/docs/source/sbi.qmd @@ -12,7 +12,7 @@ Normalizing flows provide a means to represent complicated probability distribut $$ q(\theta | d) = \mathcal{N}(0, 1)^D(f_d^{-1}(\theta)) \left| \det f_d^{-1} \right|, -$$ (eq:flow) +$$ {#eq-flow} where $D$ is the dimensionality of the parameter space. @@ -25,7 +25,7 @@ A normalizing flow must satisfy the following properties: 1. **Invertibility,** so that one can evaluate $f_d^{-1}(\theta)$ for any $\theta$. 2. **Simple Jacobian determinant,** so that one can quickly evaluate $\det f_d^{-1}(\theta)$. -With these properties, one can quickly evaluate the right-hand side of {eq}`eq:flow` to obtain the density. Various types of normalizing flow have been constructed to satisfy these properties, typically as a composition of relatively simple transforms $f^{(j)}$. These relatively simple transforms are then parametrized by the output of a neural network. To sample $\theta \sim q(\theta|d)$, one samples $u \sim \mathcal N(0,1)^D$ and applies the flow in the forward direction. +With these properties, one can quickly evaluate the right-hand side of @eq-flow to obtain the density. Various types of normalizing flow have been constructed to satisfy these properties, typically as a composition of relatively simple transforms $f^{(j)}$. These relatively simple transforms are then parametrized by the output of a neural network. To sample $\theta \sim q(\theta|d)$, one samples $u \sim \mathcal N(0,1)^D$ and applies the flow in the forward direction. For each flow step, Dingo uses a conditional coupling transform, meaning that half of the components are held fixed, and the other half transform elementwise, conditional on the untransformed components and the data, @@ -55,7 +55,7 @@ $$ L &= \int dd\, p(d) \int d\theta\, p(\theta | d) \left[ - \log q(\theta | d) \right]\\ &= \int d\theta\, p(\theta) \int dd\, p(d|\theta)\left[ - \log q(\theta | d) \right]. \end{align} -$$ (eq:loss) +$$ {#eq-loss} On the second line we used Bayes' theorem $p(d) p(\theta | d) = p(\theta) p(d | \theta)$ to re-order the integrations. The loss may finally be approximated on a mini-batch of samples, @@ -69,7 +69,7 @@ where the samples are drawn ancestrally in a two-step process: We then take the gradient of $L$ with respect to network parameters and minimize using the [Adam](https://pytorch.org/docs/stable/generated/torch.optim.Adam.html) optimizer. -Importantly, the process to generate training samples incorporates the **same information** as a standard (likelihood-based) sampler would use. Namely, the prior is incorporated by sampling parameters from it, and the likelihood is incorporated by simulating data. Bayes' theorem is incorporated in going from line 1 to line 2 in {eq}`eq:loss`. For gravitational waves, the likelihood is taken to be the probability that the residual when subtracting a signal $h(\theta)$ from $d$ is stationary Gaussian noise (with the measured PSD $S_{\text{n}}(f)$ in the detector). Likewise, to simulate data we generate a waveform $h(\theta^{(i)})$ and add a random noise realization $n \sim \mathcal N(0, S_\text{n}(f))$. Ultimately, however, the SBI approach is more flexible, since in principle one could add non-stationary or non-Gaussian noise, and train the network to reproduce the posterior, despite not having a tractable likelihood. See the section on training data for additional details of training for gravitational wave inference. +Importantly, the process to generate training samples incorporates the **same information** as a standard (likelihood-based) sampler would use. Namely, the prior is incorporated by sampling parameters from it, and the likelihood is incorporated by simulating data. Bayes' theorem is incorporated in going from line 1 to line 2 in @eq-loss. For gravitational waves, the likelihood is taken to be the probability that the residual when subtracting a signal $h(\theta)$ from $d$ is stationary Gaussian noise (with the measured PSD $S_{\text{n}}(f)$ in the detector). Likewise, to simulate data we generate a waveform $h(\theta^{(i)})$ and add a random noise realization $n \sim \mathcal N(0, S_\text{n}(f))$. Ultimately, however, the SBI approach is more flexible, since in principle one could add non-stationary or non-Gaussian noise, and train the network to reproduce the posterior, despite not having a tractable likelihood. See the section on training data for additional details of training for gravitational wave inference. Intuitively, one way to understand NPE is simply that we are doing supervised deep learning---inferring parameter labels from examples---but allowing for the flexibility to produce a probabilistic answer. With this flexibility, the network learns to produce the Bayesian posterior. diff --git a/docs/source/training.md b/docs/source/training.qmd similarity index 97% rename from docs/source/training.md rename to docs/source/training.qmd index e48413fee..eec03efb5 100644 --- a/docs/source/training.md +++ b/docs/source/training.qmd @@ -6,10 +6,9 @@ Both of these scripts take as main argument a settings file, which specifies opt ## Settings file -```{code-block} yaml ---- -caption: Example `train_settings.yaml` file. This is also available in the examples/ folder. The specific settings listed will train a production-size network, taking about a week on an NVIDIA A100. Consider reducing some model hyperparameters for experimentation. ---- +Example `train_settings.yaml` file (also available in the `examples/` folder). The specific settings listed will train a production-size network, taking about a week on an NVIDIA A100. Consider reducing some model hyperparameters for experimentation. + +```yaml data: waveform_dataset_path: /path/to/waveform_dataset.hdf5 # Contains intrinsic waveforms train_fraction: 0.95 @@ -143,9 +142,9 @@ scheduler batch_size : Number of training samples per mini-batch. For a training dataset of size $N$, then each epoch will consist of $N / \text{batch_size}$ batches. Generally training will be faster for a larger batch size, but will require additional iterations. -```{important} +::: {.callout-important} The stage-training framework allows for separate pre-training and fine-tuning stages. We found that having a pre-training stage where we freeze certain network weights and fix the noise ASD improves overall training results. -``` +::: ### `local` @@ -257,6 +256,6 @@ optional arguments: --value VALUE ``` -```{warning} +::: {.callout-warning} Modifications to model metadata can easily break things. Do not use this unless completely sure what you are doing! -``` +::: diff --git a/docs/source/training_transforms.ipynb b/docs/source/training_transforms.ipynb index 24073f3d4..794c5d589 100644 --- a/docs/source/training_transforms.ipynb +++ b/docs/source/training_transforms.ipynb @@ -16,8 +16,7 @@ "For Dingo, the flowchart below indicates the sequence of transforms applied to a sample from a `WaveformDataset`.\n", "\n", "```{mermaid}\n", - " :caption: Flowchart for Dingo data-preprocessing pipeline for training, starting from a sample from a `WaveformDataset`. Transforms with rounded corners include an element of randomness, whereas trapezoidal items are deterministic.\n", - " \n", + "%%| fig-cap: Flowchart for Dingo data-preprocessing pipeline for training, starting from a sample from a `WaveformDataset`. Transforms with rounded corners include an element of randomness, whereas trapezoidal items are deterministic.\n", "flowchart TB\n", " sample[Sample from WaveformDataset]\n", " sample-->extrinsic([SampleExtrinsicParameters])\n", @@ -47,97 +46,51 @@ " output-->E[End]\n", "```\n", "\n", - "```{important}\n", + "::: {.callout-important}\n", "Some pre-processing transforms include an element of randomness. This serves to augment the training data and reduce overfitting.\n", - "```\n", + "\n", + ":::\n", "\n", "### Extrinsic parameters\n", "\n", "The starting point for this chain of transforms is a sample `sample` with `parameters` and `polarizations` sub-dictionaries. The first transform samples the extrinsic parameters, and adds a new sub-dictionary `extrinsic_parameters` to `sample`. Extrinsic parameters include sky position (right ascension, declination), polarization, time of coalescense, and luminosity distance (the latter two of which are also considered intrinsic parameters).\n", "\n", - "```{eval-rst}\n", - ".. autoclass:: dingo.gw.transforms.SampleExtrinsicParameters\n", - " :members:\n", - "```\n", - "\n", "### Detector waveforms\n", "\n", "The next sequence of transforms applies the extrinsic parameters to `sample[\"polarizations\"]` to produce detector waveforms in `sample[\"waveform\"]`. First it calculates the arrival time $t_I$ of the waveform in each detector, based on the time of coalescense at geocenter and the sky position, and stores this in `sample[\"extrinsic_parameters\"]`,\n", - "```{eval-rst}\n", - ".. autoclass:: dingo.gw.transforms.GetDetectorTimes\n", - " :members:\n", - "```\n", "\n", "(ref:ref-time)=\n", - "```{important}\n", + "\n", + "::: {.callout-important}\n", "Dingo models are trained for a **fixed set of detectors.** This must be selected prior to training, and a new model must be trained if one wishes to analyze data in a different set of detectors. Thus, e.g., separate models must be trained for HL and HLV configurations.\n", - "```\n", "\n", - "```{note}\n", + ":::\n", + "\n", + "::: {.callout-note}\n", "During training, Dingo **fixes the orientation of the Earth** (and corresponding interferometer positions and orientations) to that at a fixed reference time `ref_time`. This is so that the model does not have to learn about the rotation of the Earth. This is corrected in post-processing by shifting the inferred right ascension by the difference between the true and reference sidereal times.\n", - "```\n", "\n", - "Optionally, the times $t_I$ are perturbed to give new \"proxy times\" as part of the [](gnpe.md) algorithm.\n", + ":::\n", "\n", - "```{eval-rst}\n", - ".. autoclass:: dingo.gw.transforms.GNPECoalescenceTimes\n", - " :members:\n", - "```\n", + "Optionally, the times $t_I$ are perturbed to give new \"proxy times\" as part of the [](gnpe.qmd) algorithm.\n", "\n", "Finally, the detector waveforms $h_I$ are calculated from the extrinsic parameters. (In the backend, these transforms use the Bilby interferometer libraries.) The contents of the `extrinsic_parameters` sub-dictionary are then moved into `sample[\"parameters\"]`; this was essentially a holding place for parameters not yet applied to the waveform.\n", "\n", - "```{eval-rst}\n", - ".. autoclass:: dingo.gw.transforms.ProjectOntoDetectors\n", - " :members:\n", - "```\n", - "\n", "### Noise\n", "\n", "Once the detector waveforms have been obtained, noise $n_I$ must be added to simulate realistic data. First, noise ASDs are selected randomly for each detector from an `ASDDataset` for the relevant observing run. This is stored in `sample[\"asds\"]`. For details see [](noise_dataset.ipynb#asd-dataset).\n", "\n", - "```{eval-rst}\n", - ".. autoclass:: dingo.gw.transforms.SampleNoiseASD\n", - " :members:\n", - "```\n", - "\n", "The waveform is then whitened based on the PSD, and furthermore scaled by the standard deviation of white noise. This is so that each input to the network will have unit variance, which is important for successful training.\n", "\n", - "```{eval-rst}\n", - ".. autoclass:: dingo.gw.transforms.WhitenAndScaleStrain\n", - " :members:\n", - "```\n", - "\n", "For whitened waveforms, noise is white, so finally this is randomly sampled and added to `sample[\"waveform\"]`.\n", "\n", - "```{eval-rst}\n", - ".. autoclass:: dingo.gw.transforms.AddWhiteNoiseComplex\n", - " :members:\n", - "```\n", - "\n", "### Output\n", "\n", "The final set of transforms prepares the sample for input to the neural network. First, the desired inference parameters are selected. By taking only a subset of `parameters`, one can train a marginalized posterior model. These parameters are also standardized to have zero mean and unit variance to improve training. (Standardization will be undone in post-processing after inference.) The parameters will then be repackaged into a `numpy.ndarray`, so that parameter labels are implicit based on ordering.\n", "\n", - "```{eval-rst}\n", - ".. autoclass:: dingo.gw.transforms.SelectStandardizeRepackageParameters\n", - " :members:\n", - "```\n", - "\n", "The `waveform` and `asds` dictionaries are also repackaged into a single array of shape suitable for input to the network. In particular, the complex frequency domain strain data are decomposed into real and imaginary parts.\n", "\n", - "\n", - "```{eval-rst}\n", - ".. autoclass:: dingo.gw.transforms.RepackageStrainsAndASDS\n", - " :members:\n", - "```\n", - "\n", "Finally, the `samples` dictionary of arrays is unpacked to a tuple of arrays for parameters and data.\n", "\n", - "```{eval-rst}\n", - ".. autoclass:: dingo.gw.transforms.UnpackDict\n", - " :members:\n", - "```\n", - "\n", "When used with a torch `DataLoader`, the final numpy arrays are automatically transformed into torch tensors.\n", "\n", "\n", @@ -145,16 +98,11 @@ "\n", "The following function will set the `transform` property of a `WaveformDataset` to the above transform sequence:\n", "\n", - "```{eval-rst}\n", - ".. autofunction:: dingo.gw.training.set_train_transforms\n", - "```\n", + "The various options are specified by passing an appropriate `data_settings` dictionary. In practice, these settings will be specified along with other [training settings](training.qmd).\n", "\n", - "The various options are specified by passing an appropriate `data_settings` dictionary. In practice, these settings will be specified along with other [training settings](training).\n", + "Sample `data_settings` dictionary for configuring a sequence of training transforms. This dictionary includes several options not needed for `set_train_transforms`, but which are needed as part of other training settings.\n", "\n", - "```{code-block} yaml\n", - "---\n", - "caption: Sample `data_settings` dictionary for configuring a sequence of training transforms. This dictionary includes several options not needed for `set_train_transforms`, but which are needed as part of other training settings.\n", - "---\n", + "```yaml\n", "waveform_dataset_path: /path/to/waveform_dataset.hdf5 # Contains intrinsic waveforms\n", "train_fraction: 0.95\n", "domain_update:\n", @@ -199,7 +147,7 @@ ": Reference time for the interferometer locations and orientations. See the [important note](ref:ref-time) above.\n", "\n", "gnpe_time_shifts (optional)\n", - ": GNPE kernel and additional options. See [](gnpe.md).\n", + ": GNPE kernel and additional options. See [](gnpe.qmd).\n", "\n", "inference_parameters\n", ": Parameters to infer with the model. At present they must be a subset of `sample[\"parameters\"]`. By specifying a strict subset, this can be used to marginalize over parameters. The `default` setting points to `dingo.gw.prior.default_inference_parameters`:" diff --git a/docs/source/waveform_dataset.ipynb b/docs/source/waveform_dataset.ipynb index 09a82914e..80e2310ae 100644 --- a/docs/source/waveform_dataset.ipynb +++ b/docs/source/waveform_dataset.ipynb @@ -25,13 +25,6 @@ "\n", "The `WaveformDataset` is a storage container for waveform polarizations and parameters, which can used to serve samples to a neural network during training:\n", "\n", - "```{eval-rst}\n", - ".. autoclass:: dingo.gw.dataset.WaveformDataset\n", - " :members:\n", - " :inherited-members:\n", - " :show-inheritance:\n", - "```\n", - "\n", "`WaveformDataset` subclasses `dingo.core.dataset.DingoDataset` and `torch.utils.data.Dataset`. The former provides generic functionality for saving and loading datasets as HDF5 files and dictionaries, and is used in several components of Dingo. The latter allows the `WaveformDataset` to be used with a PyTorch `DataLoader`. In general, we follow the PyTorch design framework for training, including [Datasets, DataLoaders,](https://pytorch.org/tutorials/beginner/basics/data_tutorial.html) and [Transforms](https://pytorch.org/tutorials/beginner/basics/transforms_tutorial.html)." ] }, @@ -505,9 +498,10 @@ "id": "1bc66ea3-86d6-4e26-83cc-b3fadcbe1166", "metadata": {}, "source": [ - "```{note}\n", + "::: {.callout-note}\n", "The sample is represented as a *nested dictionary*. This is a standard format for Dingo.\n", - "```" + "\n", + ":::" ] }, { @@ -523,9 +517,7 @@ "* Not reproducible. The dataset contains no metadata describing its construction (e.g., waveform approximant, domain, prior, ...).\n", "\n", "The `generate_dataset` function automates all of these advanced features:\n", - "```{eval-rst}\n", - ".. autofunction:: dingo.gw.dataset.generate_dataset.generate_dataset\n", - "```\n", + "\n", "This function is in turn wrapped by the command-line functions `dingo_generate_dataset` and `dingo_generate_dataset_dag`. These take a `.yaml` file with the same contents as the settings dictionary.\n", "\n", "#### Configuration\n", @@ -586,9 +578,10 @@ " spin_conversion_phase (optional)\n", " : Value for `phiRef` when converting PE spins to Cartesian spins via `bilby_to_lalsimulation_spins`. When set to `None` (default), this uses the `phase`\n", " parameter. When set to 0.0, `phase` only refers to the azimuthal observation angle, allowing for it to be treated as an extrinsic parameter.\n", - " ```{important}\n", - " It is necessary to set this to 0.0 if planning to train a `phase`-marginalized network, and then reconstruct the `phase` synthetically.\n", - " ```\n", + "\n", + "::: {.callout-important}\n", + "It is necessary to set this to 0.0 if planning to train a `phase`-marginalized network, and then reconstruct the `phase` synthetically.\n", + ":::\n", "\n", "intrinsic_prior\n", ": Specify the prior over intrinsic parameters. Intrinsic parameters here refer to those parameters that are needed to generate waveform polarizations. Extrinsic parameters here refer to those parameters that can be sampled and applied rapidly during training. As shown in the example, it is also possible to specify `default` priors, which is convenient for certain parameters. These are listed in `dingo.gw.prior.default_intrinsic_dict`.\n", @@ -611,9 +604,11 @@ " \n", " Dataset compression is implemented internally by setting the `WaveformGenerator.transform` operator, so that elements are compressed immediately after generation (avoiding the need to store many uncompressed waveforms in memory). Likewise, decompression is implemented by setting the `WaveformDataset.decompression_transform` operator to apply the inverse transformation. This will act on samples to decompress them when accessed through `WaveformDataset.__getitem__()`.\n", " \n", - "```{important}\n", + "\n", + "::: {.callout-important}\n", "The automated dataset constructors store the configuration settings in `WaveformDataset.settings`. This is so that the settings can be accessed by more downstream tasks, and for reference.\n", - "```" + "\n", + ":::" ] }, { From d58161f43b5a2d45e4abe721217a3262975b5e1e Mon Sep 17 00:00:00 2001 From: Stephen Green Date: Thu, 2 Jul 2026 15:33:51 +0200 Subject: [PATCH 02/17] Polish Quarto docs: RTD sidebar, auto API reference, rendering fixes - Left docked sidebar (ReadTheDocs-style) with the full docs tree; top navbar minimal (Home / API reference / GitHub). - Auto-generate the quartodoc API reference via gen_api.py (the sphinx-apidoc analogue): every public module, regenerated on demand (81 modules / 91 pages). - Notebooks: convert markdown-cell sources to nbformat list-form so Quarto renders the callouts (they were collapsing); restore dropped code-block captions; convert the mermaid :caption: to a Quarto fig-cap. - Code fences: retag unlabeled ``` blocks as default (directory trees), sh (commands), or python, so they render as styled panels instead of plain text. - Fix 9 enumerations that were missing the blank line pandoc requires. - custom.scss: lighter link underlines + center/cap the mermaid flowchart. - installation.qmd: document the gen_api.py -> quartodoc build -> quarto render flow. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/source/_quarto.yml | 3 --- docs/source/asimov.qmd | 22 +++++++++++----------- docs/source/code_design.qmd | 1 + docs/source/custom.scss | 18 +----------------- docs/source/dingo_pipe.qmd | 1 + docs/source/example_gnpe_model.qmd | 18 +++++++++--------- docs/source/example_injection.qmd | 4 ++-- docs/source/example_npe_model.qmd | 18 +++++++++--------- docs/source/example_toy_npe_model.qmd | 14 +++++++------- docs/source/installation.qmd | 11 +++++++---- docs/source/quickstart.qmd | 12 ++++++------ docs/source/result.qmd | 1 + docs/source/sbi.qmd | 4 ++++ docs/source/training.qmd | 2 ++ 14 files changed, 61 insertions(+), 68 deletions(-) diff --git a/docs/source/_quarto.yml b/docs/source/_quarto.yml index 39b5b8a66..34b8774ca 100644 --- a/docs/source/_quarto.yml +++ b/docs/source/_quarto.yml @@ -91,10 +91,7 @@ format: theme: [cosmo, custom.scss] toc: true code-copy: true - code-overflow: wrap highlight-style: github - code-block-bg: true - code-block-border-left: "#2478cc" mermaid: theme: neutral diff --git a/docs/source/asimov.qmd b/docs/source/asimov.qmd index b7f2bd2c3..5ab9b0135 100644 --- a/docs/source/asimov.qmd +++ b/docs/source/asimov.qmd @@ -20,7 +20,7 @@ this could be the network you want to use to analyze the event. The first step is to start a project which can be done with -``` +```sh mkdir project_tutorial cd project_tutorial asimov init "init message" @@ -28,7 +28,7 @@ asimov init "init message" If you type `ls -a` this should now show the following directory structure -``` +```default asimov.log .asimov checkouts logs results working ``` @@ -41,7 +41,7 @@ While the project has now been started, currently there are no settings applied the project. To apply some settings, download the `project_dingo.yaml` file from `https://github.com/dingo-gw/dingo/tree/main/examples/asimov` and apply them with: -``` +```sh asimov apply -f project_dingo.yaml ``` @@ -50,7 +50,7 @@ Let's add the GW150914 event to the ledger. You can download the `event_dingo.ya file from `https://github.com/dingo-gw/dingo/tree/main/examples/asimov` and apply it with -``` +```sh asimov apply -f ../../event_dingo.yaml ``` @@ -60,14 +60,14 @@ download a sample `analysis_dingo.yaml` file from You will need to change this yaml file to point to the location of your network .pt files. -``` +```sh asimov apply -f ../../analysis_dingo.yaml -e GW150914_095045 ``` We have now applied all the settings to GW150914 and are ready to begin the asimov run! Start by creating the .ini file using -``` +```sh asimov manage build ``` @@ -79,7 +79,7 @@ the dag files are created using `dingo_pipe` based off the .ini files created using the `asimov manage build` command. Next the dag files will be submitted to the cluster. To do this run -``` +```sh asimov manage submit ``` @@ -87,7 +87,7 @@ Now you should be all done! Check the output of the `working` directory to ensure that log files are being created in accordance with starting a `dingo_pipe` run. You can run -``` +```sh asimov monitor ``` @@ -103,7 +103,7 @@ you will need to install a few extra packages which are not shipped with default DINGO. To do this, run -``` +```sh python -m pip install kerberos paramiko M2Crypto python -m pip install python-nds2-client conda install python-ldas-tools-framecpp==2.6.14 @@ -111,7 +111,7 @@ conda install python-ldas-tools-framecpp==2.6.14 Then, before running `asimov manage build` make sure to run -``` +```sh export GWDATAFIND_SERVER=datafind.igwn.org ``` @@ -120,7 +120,7 @@ store your credentials in the condor vault. To do this run the following commands and follow the prompts. They will ask for your LIGO credentials. -``` +```sh export HTGETTOKENOPTS="-a vault.ligo.org -i igwn" condor_vault_storer -v igwn kinit diff --git a/docs/source/code_design.qmd b/docs/source/code_design.qmd index 63b3f2620..afdc56512 100644 --- a/docs/source/code_design.qmd +++ b/docs/source/code_design.qmd @@ -7,6 +7,7 @@ Generating reproducible results must be central to any deep learning code. Dingo ### Settings There are a large number of configuration options that must be selected when using Dingo. These include + * Waveform and noise dataset settings, * Training settings, including pre-processing, neural network, and training strategy settings, * Inference settings, including event time or injection data. diff --git a/docs/source/custom.scss b/docs/source/custom.scss index 13cfabf6d..f58b4df27 100644 --- a/docs/source/custom.scss +++ b/docs/source/custom.scss @@ -1,6 +1,6 @@ /*-- scss:defaults --*/ + $link-color: #2478cc; -$code-bg: #f7f8fa; /*-- scss:rules --*/ @@ -12,22 +12,6 @@ a:hover { text-decoration: underline; } -// Code blocks (incl. the example config listings): soft border + rounded corners -// so ini/yaml examples read as tidy panels rather than bare text. -div.sourceCode { - border: 1px solid #e3e6ea; - border-radius: 6px; - margin-block: 1rem; - overflow: hidden; -} -div.sourceCode pre { - padding: 0.8em 1em; - margin-bottom: 0; -} -pre > code.sourceCode { - font-size: 0.85em; -} - // Mermaid: center the flowchart and cap its width so it isn't oversized. .cell-output-display svg, figure.quarto-figure svg, diff --git a/docs/source/dingo_pipe.qmd b/docs/source/dingo_pipe.qmd index 1839a6040..679f3c5b6 100644 --- a/docs/source/dingo_pipe.qmd +++ b/docs/source/dingo_pipe.qmd @@ -87,6 +87,7 @@ Dingo models are typically trained using Welch PSDs. For this reason we do not r ### Injections Injections mirror the [approach of `bilby_pipe`](https://lscsoft.docs.ligo.org/bilby_pipe/master/injections.html) with a few modifications: + * By default, injections are generated using `bilby_pipe`. To use signal waveforms generated by the Dingo code (using the [Injections class](inference.qmd#injections)) set `dingo-injection = True`. Note that Dingo and Bilby injections might differ depending on the `spin_conversion_phase` setting, which is only available in Dingo. * Setting `zero-noise = True` will create a zero-noise injection, which is out-of-distribution with respect to Dingo training data. It should therefore be used with caution. * When running many injections `n-simulation > 1`, one may want to produce a PP plot. This can be done automatically by setting `plot-pp = true`. diff --git a/docs/source/example_gnpe_model.qmd b/docs/source/example_gnpe_model.qmd index 7d0316b8a..e15e4ef2d 100644 --- a/docs/source/example_gnpe_model.qmd +++ b/docs/source/example_gnpe_model.qmd @@ -8,7 +8,7 @@ tutorial, but we repeat it here, for completeness. The file structure is similar to the NPE example, except now there are two training sub-directories and two `train_settings.yaml` files. -``` +```default gnpe_model/ # config files @@ -50,7 +50,7 @@ Step 1 Generating a Waveform Dataset First generate the directory structure: -``` +```sh cd gnpe_model mkdir training_data mkdir training @@ -60,13 +60,13 @@ mkdir training/init_train_dir Generate the waveform dataset: -``` +```sh dingo_generate_dataset --settings waveform_dataset_settings.yaml --out_file training_data/waveform_dataset.hdf5 ``` or using condor: -``` +```sh dingo_generate_dataset_dag --settings_file waveform_dataset_settings.yaml --out_file training_data/waveform_dataset.hdf5 --env_path $DINGO_VENV_PATH --num_jobs 4 @@ -79,14 +79,14 @@ Step 2 Generating an ASD dataset As before we generate a fiducial ASD dataset containing a single ASD: -``` +```sh dingo_generate_asd_dataset --settings_file asd_dataset_settings_fiducial.yaml --data_dir training_data/asd_dataset_fiducial --out_name training_data/asd_dataset_fiducial/asds_O1_fiducial.hdf5 ``` and a large ASD dataset: -``` +```sh dingo_generate_asd_dataset --settings_file asd_dataset_settings.yaml --data_dir training_data/asd_dataset --out_name training_data/asd_dataset/asds_O1.hdf5 ``` @@ -102,14 +102,14 @@ standardize) the times of arrival in the detectors, the inference task will become significantly easier. To do this we first need to train an initialization network which estimates the time of arrival in the detectors: -``` +```sh dingo_train --settings_file train_settings_init.yaml --train_dir training/init_train_dir ``` Notice that the inference parameters are only the `H1_time` and `L1_time`. Also notice that the embedding_net is significantly smaller and the number of flow steps, `num_flow_steps` is reduced. -``` +```sh dingo_train --settings_file train_settings.yaml --train_dir training/main_train_dir ``` @@ -126,7 +126,7 @@ difference is the new attribute under sampler arguments `num-gnpe-iterations` wh number of GNPE steps to take. If the initialization network is not fully converged or if the length of the segment being analyzed is very long, it is recommended to increase this number. -``` +```sh dingo_pipe GW150914.ini ``` diff --git a/docs/source/example_injection.qmd b/docs/source/example_injection.qmd index 0d78242c0..e5525bb4d 100644 --- a/docs/source/example_injection.qmd +++ b/docs/source/example_injection.qmd @@ -6,7 +6,7 @@ the `dingo.gw.injection.Injection` using the metadata from the `dingo.core.models.posterior_model.PosteriorModel` (the trained network). An ASD dataset also needs to be specified, one can take the fiducial asd dataset the network was trained on. -``` +```python from dingo.core.models import PosteriorModel import dingo.gw.injection as injection from dingo.gw.noise.asd_dataset import ASDDataset @@ -55,7 +55,7 @@ strain_data = injection_generator.injection(theta) Then one can create a injections and do inference on them. -``` +```python from dingo.gw.inference.gw_samplers import GWSamplerGNPE, GWSampler init_sampler = GWSampler(model=init_pm) diff --git a/docs/source/example_npe_model.qmd b/docs/source/example_npe_model.qmd index 28395c3c1..2d379f381 100644 --- a/docs/source/example_npe_model.qmd +++ b/docs/source/example_npe_model.qmd @@ -10,7 +10,7 @@ We can repeat the same first few steps from the previous tutorial with a couple differences. The file structure is mostly the same but now there is an additional `asd_dataset_fiducial` which will be explained below. -``` +```default npe_model/ # config files @@ -42,7 +42,7 @@ Step 1 Generating a Waveform Dataset Again the first step is to generate the necessary folders -``` +```sh cd npe_model mkdir training_data mkdir training @@ -50,7 +50,7 @@ mkdir training As before we run `dingo_generate_dataset`: -``` +```sh dingo_generate_dataset --settings waveform_dataset_settings.yaml --out_file training_data/waveform_dataset.hdf5 ``` @@ -70,13 +70,13 @@ We could also generate the waveform dataset using a [condor DAG](https://htcondor.readthedocs.io/en/latest/users-manual/dagman-workflows.html) on a cluster. To do this run -``` +```sh dingo_generate_dataset_dag --settings_file waveform_dataset_settings.yaml --out_file training_data/waveform_dataset.hdf5 --env_path $DINGO_VENV_PATH --num_jobs 4 --request_cpus 64 --request_memory 128000 --request_memory_high 256000 ``` and then submit the generated DAG -``` +```sh condor_submit_dag condor/submit/dingo_generate_dataset_dagman_DATE.submit ``` @@ -88,7 +88,7 @@ Step 2 Generating an ASD dataset To generate an ASD dataset we can run the same command as in the previous tutorial. -``` +```sh dingo_generate_asd_dataset --settings_file asd_dataset_settings_fiducial.yaml --data_dir training_data/asd_dataset_fiducial --out_name training_data/asd_dataset_fiducial/asds_O1_fiducial.hdf5 ``` @@ -102,7 +102,7 @@ the ASD will be variable. Thus, in the second stage many ASDs are used so that d of ASDs from the observing run. We find this split leads to an improvement in overall performance. To generate this second dataset run -``` +```sh dingo_generate_asd_dataset --settings_file asd_dataset_settings.yaml --data_dir training_data/asd_dataset --out_name training_data/asd_dataset/asds_O1.hdf5 ``` @@ -119,7 +119,7 @@ Step 3 Training the network Now we are ready for training. The command is analogous to the previous tutorial but the settings are increased to production values. To run the training do -``` +```sh dingo_train --settings_file train_settings.yaml --train_dir training ``` @@ -153,7 +153,7 @@ Step 4 Doing Inference We can run inference with the same command as before -``` +```sh dingo_pipe GW150914.ini ``` diff --git a/docs/source/example_toy_npe_model.qmd b/docs/source/example_toy_npe_model.qmd index 064aa6a40..44f35b2d5 100644 --- a/docs/source/example_toy_npe_model.qmd +++ b/docs/source/example_toy_npe_model.qmd @@ -17,7 +17,7 @@ There are 4 main steps: In this tutorial as well as the [npe model](example_npe_model.qmd) and [gnpe model](example_gnpe_model.qmd) the following file structure will be employed -``` +```default toy_npe_model/ # config files @@ -51,7 +51,7 @@ Step 1 Generating a waveform dataset After downloading the files for the tutorial first run -``` +```sh cd toy_npe_model/ mkdir training_data mkdir training @@ -59,7 +59,7 @@ mkdir training to set up the file structure. Then run -``` +```sh dingo_generate_dataset --settings waveform_dataset_settings.yaml --out_file training_data/waveform_dataset.hdf5 ``` @@ -137,7 +137,7 @@ Step 2 Generating the Amplitude Spectral Density (ASD) dataset To generate an ASD dataset run -``` +```sh dingo_generate_asd_dataset --settings_file asd_dataset_settings.yaml --data_dir training_data/asd_dataset ``` @@ -166,7 +166,7 @@ Step 3 Training the network To train the network, first the paths to the correct datasets must be specified before executing: -``` +```sh dingo_train --settings_file train_settings.yaml --train_dir training ``` @@ -267,7 +267,7 @@ Step 4 Doing Inference The final step is to do inference, for example on GW150914. To do this we will use [dingo_pipe](dingo_pipe.qmd). For a local run execute: -``` +```sh dingo_pipe GW150914.ini ``` @@ -317,7 +317,7 @@ This will generate files which are described in [dingo_pipe](dingo_pipe.qmd). To We can load and manipulate the data with the following code. For example, here we create a cornerplot -``` +```python from dingo.gw.result import Result result = Result(file_name="outdir_GW150914/result/GW150914_data0_1126259462-4_sampling.hdf5") result.plot_corner() diff --git a/docs/source/installation.qmd b/docs/source/installation.qmd index 13ebd8be6..48d27f44c 100644 --- a/docs/source/installation.qmd +++ b/docs/source/installation.qmd @@ -85,12 +85,15 @@ install `quartodoc`: pip install quartodoc ``` -The documentation sources live in `docs/source`. From that directory, first generate the -API reference pages from the docstrings: +The documentation sources live in `docs/source`. From that directory, first refresh the +API reference: `gen_api.py` scans the `dingo` package and lists every public module in +`_quarto.yml` (the analogue of `sphinx-apidoc`), then `quartodoc build` writes a reference +page for each module's members from their docstrings: ```sh -quartodoc build +python gen_api.py # refresh the API module list (run when modules are added or removed) +quartodoc build # generate the reference/ pages from docstrings ``` -This writes the `reference/` pages. Then render the site: +Then render the site: ```sh quarto render ``` diff --git a/docs/source/quickstart.qmd b/docs/source/quickstart.qmd index 04c6a70aa..2229bd985 100644 --- a/docs/source/quickstart.qmd +++ b/docs/source/quickstart.qmd @@ -52,7 +52,7 @@ Dingo `venv` is active. To generate a waveform dataset for training, execute -``` +```sh dingo_generate_dataset --settings_file waveform_dataset_settings.yaml --num_processes N --out_file waveform_dataset.hdf5 ``` @@ -68,7 +68,7 @@ on a cluster. This is typically useful for slower waveform models. Training also requires a dataset of noise ASDs, which are sampled randomly for each training sample. To generate this dataset based on noise observed during a run, execute -``` +```sh dingo_generate_ASD_dataset --data_dir data_dir --settings_file asd_dataset_settings.yaml ``` @@ -83,7 +83,7 @@ one `data_dir/asd_dataset_settings.yaml`. With a waveform dataset and ASD dataset(s), one can train a neural network. Configure the `train_settings.yaml` file to point to these datasets, and run -``` +```sh dingo_train --settings_file train_settings.yaml --train_dir train_dir ``` @@ -91,7 +91,7 @@ This will configure the network, train it, and store checkpoints, a record of th and the final network in the directory `train_dir`. Alternatively, to resume training from a checkpoint file, run -``` +```sh dingo_train --checkpoint model.pt --train_dir train_dir ``` @@ -127,7 +127,7 @@ found under `examples/pipe/GW150914.ini`. To do inference, cd into the directory with the .ini file and run -``` +```sh dingo_pipe GW150914.ini ``` @@ -135,7 +135,7 @@ dingo_pipe GW150914.ini train_init generate_dataset--->train_init generate_asd--->train_init - generate_dataset--->train_main(["dingo_train + generate_dataset--->train_main(["dingo_train #nbsp; #nbsp; --settings_file train_settings_main.yaml #nbsp; #nbsp; --train_dir model_main"]) style train_main text-align:left @@ -99,33 +99,33 @@ If using CUDA on a machine with several GPUs, be sure to first select the desire number using the `CUDA_VISIBLE_DEVICES` environment variable. If using a cluster, Dingo can be trained using `dingo_train_condor`. -Example training files can be found under `examples/training`. +Example training files can be found under `examples/training`. `train_settings_toy.yaml` and `train_settings_production.yaml` train a flow to estimate the full posterior of the event conditioned on the time of coalescence -in the detectors. The "toy" label is to indicate this should NOT be used for production but -rather to get a feel for the Dingo pipeline. The production settings contain tested +in the detectors. The "toy" label is to indicate this should NOT be used for production but +rather to get a feel for the Dingo pipeline. The production settings contain tested settings. Note that depending on the waveform model and event, these may need to occasionally be tuned. `train_settings_init_toy.yaml` and `train_settings_init_production.yaml` train flows to estimate the time of coalescence in the individual detectors. These two networks are needed to use [GNPE](gnpe.qmd). This is the preferred and -most tested way of using Dingo. +most tested way of using Dingo. Alternatively, the `train_settings_no_gnpe_toy.yaml` and `train_settings_no_gnpe_production.yaml` contain settings to train a network without the GNPE step. Note the lack of a `data/gnpe_time_shifts` option. While this is not -recommended for production, it is still pedagogically useful and is good for prototyping -new ideas or doing a less expensive training. +recommended for production, it is still pedagogically useful and is good for prototyping +new ideas or doing a less expensive training. ## Inference Once a Dingo model is trained, inference for real events can be performed using -[dingo_pipe](dingo_pipe.qmd). There are 3 main inference steps, downloading the data, +[dingo_pipe](dingo_pipe.qmd). There are 3 main inference steps, downloading the data, running Dingo on this data and finally running importance sampling. The basic idea is to create a .ini file which contains the filepaths of the Dingo networks trained above and the segment of data to analyze. An example .ini file can be -found under `examples/pipe/GW150914.ini`. +found under `examples/pipe/GW150914.ini`. -To do inference, cd into the directory with the .ini file and run +To do inference, cd into the directory with the .ini file and run ```sh dingo_pipe GW150914.ini @@ -133,7 +133,7 @@ dingo_pipe GW150914.ini