diff --git a/.editorconfig b/.editorconfig index 3994f81184..d6072f52ec 100644 --- a/.editorconfig +++ b/.editorconfig @@ -9,13 +9,14 @@ [*] end_of_line = lf insert_final_newline = true + max_line_length = 79 # Matches multiple files with brace expansion notation # Set default charset and tab width [*.{js,py}] charset = utf-8 indent_style = space - indent_size = 2 + indent_size = 4 # cpp and hpp file indentation [*.{cpp,hpp}] diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a21c82a05b..95d6f5d26d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -45,34 +45,26 @@ jobs: binder: name: Notebooks runs-on: ubuntu-20.04 - needs: - - docker-build container: - image: mmore500/conduit:GITHUB_ACTION_${{ github.run_number }} + image: mmore500/conduit:GITHUB_ACTION_706 # github actions requires root to access mounted volumes options: --user root steps: + - uses: actions/checkout@v2 + # with: + # submodules: 'recursive' # Adapted from https://askubuntu.com/a/524368 - - run: cd /opt/conduit/binder && ./execute_notebooks.sh + - run: ./binder/execute_notebooks.sh + # adapted from https://unix.stackexchange.com/a/545630 + - run: find -type f -name "*.pdf" -exec bash -c 'gs -sDEVICE=pdfwrite -dCompatibilityLevel=1.4 -dPDFSETTINGS=/screen -dNOPAUSE -dBATCH -dQUIET -sOutputFile="new.pdf" "{}"; rm "{}"; mv "new.pdf" "{}";' {} \; + - run: git config --global --add safe.directory "*" - run: git init - uses: fregante/setup-git-user@v1 - run: git commit --allow-empty -m "Initial commit" - - run: cp -r /opt/conduit/ deploy/ - uses: JamesIves/github-pages-deploy-action@4.0.0 with: + folder: . branch: binder # The branch the action should deploy to. - folder: deploy # The folder the action should deploy. - - run: mkdir -p deploy-artifacts/ - - run: cp -r /opt/conduit/binder/. deploy-artifacts/binder/ - # adapted from https://unix.stackexchange.com/a/107647 - - run: find deploy-artifacts/binder/ -type f ! \( -name '*.csv' -o -name '*.png' -o -name '*.jpg' -o -name '*.pdf' -o -name '*.json' -o -name '*.meta' -o -name '*.txt' -o -name '*.out' -o -name '*.ipynb' \) -delete - # delete symlinks (Overleaf is not compatible with symlinks) - # adapted from https://superuser.com/a/586948 - - run: find deploy-artifacts/binder -type l -delete - - uses: JamesIves/github-pages-deploy-action@4.0.0 - with: - branch: binder-artifacts # The branch the action should deploy to. - folder: deploy-artifacts # The folder the action should deploy. unit: name: Unit Tests diff --git a/Dockerfile b/Dockerfile index 6e1f2606ff..ca578b89ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Pull base image. -FROM ubuntu:bionic-20180125@sha256:d6f6cc62b6bed64387d84ca227b76b9cc45049b0d0aefee0deec21ed19a300bf +FROM ubuntu:focal-20231003@sha256:218bb51abbd1864df8be26166f847547b3851a89999ca7bfceb85ca9b5d2e95d COPY . /opt/conduit/ @@ -36,20 +36,16 @@ RUN \ && \ echo "buffed apt-get resiliency" -RUN \ - find /etc/apt -type f -name '*.list' -exec sed -i 's/\(^deb.*-backports.*\)/#\1/; s/\(^deb.*-updates.*\)/#\1/; s/\(^deb.*-proposed.*\)/#\1/; s/\(^deb.*-security.*\)/#\1/' {} + \ - && \ - rm -rf /var/lib/apt/lists/* \ - && \ - echo "removed -backports, -updates, -proposed, -security repositories" - # adapted in part form https://www.cloudsavvyit.com/13461/how-to-run-puppeteer-and-headless-chrome-in-a-docker-container/ RUN \ apt-get update -qq \ && \ - apt-get install -y --allow-downgrades --no-install-recommends \ + apt-get install -y aptitude git --allow-downgrades --no-install-recommends aptitude \ + && \ + aptitude install -y --without-recommends \ build-essential \ ca-certificates \ + clang-7 \ cmake \ curl \ doxygen \ @@ -57,8 +53,10 @@ RUN \ fonts-liberation \ gconf-service \ gdb \ + ghostscript \ gpg-agent \ gzip \ + g++-9 \ hdf5-helpers \ hdf5-tools \ htop \ @@ -67,6 +65,7 @@ RUN \ libatk1.0-0 \ libc6 \ libcairo2 \ + libclang-7-dev \ libcups2 \ libcurl4 \ libcurl4-openssl-dev \ @@ -80,14 +79,14 @@ RUN \ libgtk-3-0 \ libicu-dev \ libjpeg-dev \ - libhdf5-100 \ - libhdf5-cpp-100 \ + libhdf5-103 \ + libhdf5-cpp-103 \ libhdf5-dev \ libhdf5-doc \ libhdf5-mpi-dev \ - libhdf5-mpich-100 \ + libhdf5-mpich-103 \ libhdf5-mpich-dev \ - libhdf5-openmpi-100 \ + libhdf5-openmpi-103 \ libhdf5-openmpi-dev \ libhdf5-serial-dev \ libmpich-dev \ @@ -95,13 +94,13 @@ RUN \ libnspr4 \ libnss3 \ libopenmpi-dev \ - libopenmpi2 \ + # libopenmpi2 \ libpango-1.0-0 \ libpangocairo-1.0-0 \ libpng-dev \ libpthread-stubs0-dev \ libstdc++-7-dev \ - libstdc++6=8-20180414-1ubuntu2 \ + libstdc++6 \ libx11-6 \ libx11-xcb1 \ libxcb1 \ @@ -115,6 +114,8 @@ RUN \ libxrender1 \ libxss1 \ libxtst6 \ + llvm-7 \ + llvm-7-dev \ locales \ lsb-release \ man \ @@ -128,90 +129,43 @@ RUN \ openmpi-common \ openmpi-doc \ openssh-server \ - python-dev \ - python-h5py \ - python-pip \ - python-setuptools \ - python-sphinx \ - python-virtualenv \ - python-wheel \ + # python-dev \ + # python-h5py \ + # python-pip \ + # python-setuptools \ + # python-virtualenv \ + # python-wheel \ python3-dev \ python3-h5py \ python3-pip \ python3-setuptools \ - python3-sphinx \ python3-virtualenv \ python3-wheel \ rename \ rsync \ slurm-client \ software-properties-common \ + sphinx-doc \ + sphinx-common \ tar \ unzip \ vim \ wget \ xdg-utils \ && \ - add-apt-repository ppa:git-core/ppa -y \ - && \ - apt-get update -qq \ - && \ - apt-get install -y --no-install-recommends git \ - && \ apt-get clean \ && \ rm -rf /var/lib/apt/lists/* \ && \ echo "installed fundamentals" -# adapted in part from https://askubuntu.com/a/916451 -RUN \ - apt-get update -qq \ - && \ - rm /etc/apt/apt.conf.d/docker-gzip-indexes \ - && \ - apt-get purge apt-show-versions \ - && \ - rm /var/lib/apt/lists/*lz4 \ - && \ - apt-get -o Acquire::GzipIndexes=false update \ - && \ - add-apt-repository -y ppa:ubuntu-toolchain-r/test \ - && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \ - && \ - apt-add-repository "deb https://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main" \ - && \ - add-apt-repository -y ppa:ubuntu-toolchain-r/test \ - && \ - apt-get update -qq \ - && \ - apt-get clean \ - && \ - rm -rf /var/lib/apt/lists/* \ - && \ - echo "configured packaging system" - -RUN \ - apt-get update -qq \ - && \ - apt-get install -qq \ - libclang-7-dev=1:7.1.0~svn353565-1~exp1~20190408084827.60 \ - llvm-7=1:7.1.0~svn353565-1~exp1~20190408084827.60 \ - llvm-7-dev=1:7.1.0~svn353565-1~exp1~20190408084827.60 \ - clang-7=1:7.1.0~svn353565-1~exp1~20190408084827.60 \ - g++-9=9.4.0-1ubuntu1~18.04 \ - && \ - apt-get clean \ - && \ - rm -rf /var/lib/apt/lists/* \ - && \ - echo "installed llvm-7 dependencies" - # magic from https://github.com/puppeteer/puppeteer/issues/3451#issuecomment-523961368 RUN echo 'kernel.unprivileged_userns_clone=1' > /etc/sysctl.d/userns.conf +# git confifg, see https://stackoverflow.com/a/76769867 RUN \ + git config --global url."https://".insteadOf git:// \ + && \ python3 -m pip install --timeout 60 --retries 100 --upgrade pip==21.3.1 \ && \ python3 -m pip install --timeout 60 --retries 100 --ignore-installed -r /opt/conduit/third-party/requirements.txt \ @@ -231,9 +185,11 @@ RUN \ && \ update-alternatives --install /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-7 90 \ && \ - npm install -g n@6.7.0 \ + npm install -g n@6.7.0 semver@7.5.4 \ + && \ + npm cache clean --force \ && \ - n 12.18.2 \ + PATH="$PATH" n 12.18.2 \ && \ export python="/usr/bin/python" \ && \ @@ -291,6 +247,7 @@ RUN \ && \ echo "user added and granted permissions to /opt and /home/user" +# git safe directory: https://stackoverflow.com/a/73100228/17332200 RUN \ mkdir /context/ \ && \ @@ -300,6 +257,10 @@ RUN \ && \ chown user:user /__w/ \ && \ + git config --global --add safe.directory '*' \ + && \ + git config --global --add safe.directory /__w/conduit/conduit \ + && \ echo "/context/ /__w/ directories set up, user granted permissions" USER user diff --git a/binder/clear_notebooks.sh b/binder/clear_notebooks.sh index f83722ff1e..763d18030c 100755 --- a/binder/clear_notebooks.sh +++ b/binder/clear_notebooks.sh @@ -90,7 +90,7 @@ echo "-------------------------" shopt -s nullglob for script in "${script_dir}/"*/clear_notebooks.sh; do - "${script}" + "${script}" & done shopt -u nullglob diff --git a/binder/date=2021+project=72k5n/a=computation-vs-communication/a=beleaguerment+what=computation-vs-communication+compute-work=0_64_4096_262144_16777216+simels_per_cpu=1+ext=.ipynb b/binder/date=2021+project=72k5n/a=computation-vs-communication/a=beleaguerment+what=computation-vs-communication+compute-work=0_64_4096_262144_16777216+simels_per_cpu=1+ext=.ipynb new file mode 100644 index 0000000000..c0bdf5df11 --- /dev/null +++ b/binder/date=2021+project=72k5n/a=computation-vs-communication/a=beleaguerment+what=computation-vs-communication+compute-work=0_64_4096_262144_16777216+simels_per_cpu=1+ext=.ipynb @@ -0,0 +1,152 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "\n", + "from keyname import keyname as kn\n", + "from matplotlib import pyplot as plt\n", + "from nbmetalog import nbmetalog as nbm\n", + "import numpy as np\n", + "import pandas as pd\n", + "from scipy import stats\n", + "import seaborn as sns\n", + "from slugify import slugify\n", + "import statsmodels.api as sm\n", + "import statsmodels.formula.api as smf\n", + "from teeplot import teeplot as tp\n", + "import itertools as it\n", + "import matplotlib.patheffects as pe\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(1)\n", + "random.seed(1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from conduitpylib.viz import _get_defaults as cfg\n", + "\n", + "from conduitpylib.viz import beleaguerment_facetplot, beleaguerment_regplot\n", + "\n", + "from conduitpylib.wrangle import (\n", + " retrieve_and_prepare_delta_dataframes,\n", + " wrangle_instrumentation_longform,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nbm.print_metadata()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Get Data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "longitudinal_df, snapshot_df = retrieve_and_prepare_delta_dataframes(\n", + " df_inlet_url=\"https://osf.io/k873q/download\",\n", + " df_outlet_url=\"https://osf.io/5bn96/download\",\n", + " treatment_column=\"Log Compute Work\",\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Compare Finalized and Rolling Measures\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data = wrangle_instrumentation_longform(\n", + " longitudinal_df=longitudinal_df,\n", + " snapshot_df=snapshot_df,\n", + ")\n", + "\n", + "for treatment, group in data.groupby(\"Log Compute Work\"):\n", + " group_df = group.reset_index()\n", + " tp.tee(\n", + " beleaguerment_facetplot,\n", + " data=group_df,\n", + " x=\"Messages Received Per Second\",\n", + " y=\"Messages Sent Per Second\",\n", + " hue=\"Instrumentation\",\n", + " annotation_kwargs=dict(\n", + " xy=(0.05, 0.95),\n", + " ),\n", + " scatter_kwargs=dict(\n", + " alpha=0.6,\n", + " s=30,\n", + " zorder=999,\n", + " ),\n", + " kde_kwargs=dict(\n", + " cut=10,\n", + " kde_thresh=1e-3,\n", + " ),\n", + " regline_kwargs=dict(\n", + " linewidth=6,\n", + " zorder=-9999,\n", + " ),\n", + " teeplot_outattrs={\n", + " \"log-compute-work\": slugify(str(treatment)),\n", + " },\n", + " )\n", + " plt.show()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/binder/date=2021+project=72k5n/a=computation-vs-communication/a=performance_semantics+what=comp-vs-commcompute-work=0_64_4096_262144_16777216+simels_per_cpu=1+ext=.ipynb b/binder/date=2021+project=72k5n/a=computation-vs-communication/a=performance_semantics+what=comp-vs-commcompute-work=0_64_4096_262144_16777216+simels_per_cpu=1+ext=.ipynb new file mode 100644 index 0000000000..6a72ef2a54 --- /dev/null +++ b/binder/date=2021+project=72k5n/a=computation-vs-communication/a=performance_semantics+what=comp-vs-commcompute-work=0_64_4096_262144_16777216+simels_per_cpu=1+ext=.ipynb @@ -0,0 +1,182 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import itertools as it\n", + "from iterdub import iterdub as ib\n", + "from iterpop import iterpop as ip\n", + "from keyname import keyname as kn\n", + "from matplotlib import pyplot as plt\n", + "import matplotlib\n", + "from nbmetalog import nbmetalog as nbm\n", + "import numpy as np\n", + "import pandas as pd\n", + "import patchworklib as pw\n", + "import pathlib\n", + "import random\n", + "from scipy import stats\n", + "import seaborn as sns\n", + "from slugify import slugify\n", + "import statsmodels.api as sm\n", + "import statsmodels.formula.api as smf\n", + "from teeplot import teeplot as tp\n", + "from tqdm import tqdm\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(1)\n", + "random.seed(1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from conduitpylib.viz import _get_defaults as cfg\n", + "from conduitpylib.viz import (\n", + " get_notomoji_font_path,\n", + " performance_semantics_scatterplot,\n", + " performance_semantics_facetplot,\n", + ")\n", + "from conduitpylib.wrangle import retrieve_and_prepare_delta_dataframes\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nbm.print_metadata()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Get Data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def apply(df: pd.DataFrame) -> pd.DataFrame:\n", + " df = df[\n", + " df[\"Log Compute Work\"] <= 3\n", + " ].copy()\n", + "\n", + " return df\n", + "\n", + "longitudinal, snapshot = retrieve_and_prepare_delta_dataframes(\n", + " df_inlet_url=\"https://osf.io/k873q/download\",\n", + " df_outlet_url=\"https://osf.io/5bn96/download\",\n", + " treatment_column=\"Log Compute Work\",\n", + " apply=apply,\n", + ")\n", + "\n", + "longitudinal[\"Log Simstep Period Inlet (ns)\"] = np.log10(\n", + " longitudinal[\"Simstep Period Inlet (ns)\"]\n", + ")\n", + "snapshot[\"Log Simstep Period Inlet (ns)\"] = np.log10(\n", + " snapshot[\"Simstep Period Inlet (ns)\"]\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for data in \"longitudinal\", \"snapshot\":\n", + " for work, group in eval(data).groupby(\"Log Compute Work\"):\n", + " group_df = group.reset_index()\n", + " tp.tee(\n", + " performance_semantics_scatterplot,\n", + " data=group_df,\n", + " x=\"Log Simstep Period Inlet (ns)\",\n", + " y=\"Latency Simsteps Inlet\",\n", + " heat=\"% Msgs Lost\",\n", + " legend_font_name=get_notomoji_font_path(),\n", + " title=\"{}\u2716\ufe0f\ud83e\uddee\".format(work),\n", + " batch_kwargs={\n", + " \"batch_size\": 10,\n", + " \"progress_apply\": tqdm,\n", + " },\n", + " bunching_smear_alpha=0.2,\n", + " scatter_kwargs={\n", + " \"alpha\": 0.4,\n", + " },\n", + " teeplot_outattrs={\n", + " \"work\": slugify(str(work)),\n", + " \"data\": slugify(data),\n", + " },\n", + " )\n", + " plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for data in \"longitudinal\", \"snapshot\":\n", + " plt.clf()\n", + " with pw.patched_axisgrid():\n", + " facetplot = tp.tee(\n", + " performance_semantics_facetplot,\n", + " data=eval(data),\n", + " x=\"Log Simstep Period Inlet (ns)\",\n", + " y=\"Latency Simsteps Inlet\",\n", + " hue=\"Log Compute Work\",\n", + " legend_font_name=get_notomoji_font_path(),\n", + " scatter_kwargs={\n", + " \"x\": \"Simstep Period Inlet (ns)\",\n", + " },\n", + " sharexy=False,\n", + " title=\"\ud83e\uddee/\ud83d\udcec\",\n", + " teeplot_outattrs={\n", + " \"data\": slugify(data),\n", + " },\n", + " )\n", + " display(facetplot)\n", + " plt.show()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/binder/date=2021+project=72k5n/a=intranode-vs-internode/a=beleaguerment+what=intranode-vs-internode+compute-work=0+simels_per_cpu=1+ext=.ipynb b/binder/date=2021+project=72k5n/a=intranode-vs-internode/a=beleaguerment+what=intranode-vs-internode+compute-work=0+simels_per_cpu=1+ext=.ipynb new file mode 100644 index 0000000000..2678c0e792 --- /dev/null +++ b/binder/date=2021+project=72k5n/a=intranode-vs-internode/a=beleaguerment+what=intranode-vs-internode+compute-work=0+simels_per_cpu=1+ext=.ipynb @@ -0,0 +1,169 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "\n", + "from keyname import keyname as kn\n", + "from matplotlib import pyplot as plt\n", + "from nbmetalog import nbmetalog as nbm\n", + "import numpy as np\n", + "import pandas as pd\n", + "from scipy import stats\n", + "import seaborn as sns\n", + "from slugify import slugify\n", + "import statsmodels.api as sm\n", + "import statsmodels.formula.api as smf\n", + "from teeplot import teeplot as tp\n", + "import itertools as it\n", + "import matplotlib.patheffects as pe\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(1)\n", + "random.seed(1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from conduitpylib.viz import _get_defaults as cfg\n", + "\n", + "from conduitpylib.viz import beleaguerment_facetplot, beleaguerment_regplot\n", + "\n", + "from conduitpylib.wrangle import (\n", + " retrieve_and_prepare_delta_dataframes,\n", + " wrangle_instrumentation_longform,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nbm.print_metadata()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Get Data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def apply(df: pd.DataFrame) -> pd.DataFrame:\n", + " row_distiller = lambda row: {\n", + " k: v for k, v in row.items() if k in (\"Num Nodes\", \"Num Processes\")\n", + " }\n", + " df[\"Multiprocessing\"] = df.apply(\n", + " lambda row: {\n", + " frozenset(\n", + " {\"Num Nodes\": 1, \"Num Processes\": 2}.items()\n", + " ): \"Intranode\",\n", + " frozenset(\n", + " {\"Num Nodes\": 2, \"Num Processes\": 2}.items()\n", + " ): \"Internode\",\n", + " }[frozenset(row_distiller(row).items())],\n", + " axis=1,\n", + " )\n", + " return df\n", + "\n", + "\n", + "longitudinal_df, snapshot_df = retrieve_and_prepare_delta_dataframes(\n", + " df_inlet_url=\"https://osf.io/4ys9v/download\",\n", + " df_outlet_url=\"https://osf.io/aum7w/download\",\n", + " apply=apply,\n", + " treatment_column=\"Multiprocessing\",\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Compare Finalized and Rolling Measures\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data = wrangle_instrumentation_longform(\n", + " longitudinal_df=longitudinal_df,\n", + " snapshot_df=snapshot_df,\n", + ")\n", + "\n", + "for treatment, group in data.groupby(\"Multiprocessing\"):\n", + " group_df = group.reset_index()\n", + " tp.tee(\n", + " beleaguerment_facetplot,\n", + " data=group_df,\n", + " x=\"Messages Received Per Second\",\n", + " y=\"Messages Sent Per Second\",\n", + " hue=\"Instrumentation\",\n", + " annotation_kwargs=dict(\n", + " xy=(0.05, 0.95),\n", + " ),\n", + " scatter_kwargs=dict(\n", + " alpha=0.6,\n", + " s=30,\n", + " zorder=999,\n", + " ),\n", + " kde_kwargs=dict(\n", + " cut=10,\n", + " kde_thresh=1e-3,\n", + " ),\n", + " regline_kwargs=dict(\n", + " linewidth=6,\n", + " zorder=-9999,\n", + " ),\n", + " teeplot_outattrs={\n", + " \"treatment\": slugify(treatment),\n", + " },\n", + " )\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/binder/date=2021+project=72k5n/a=intranode-vs-internode/a=performance_semantics+what=interintranode+compute-work=0+simels_per_cpu=1+ext=.ipynb b/binder/date=2021+project=72k5n/a=intranode-vs-internode/a=performance_semantics+what=interintranode+compute-work=0+simels_per_cpu=1+ext=.ipynb new file mode 100644 index 0000000000..8ccd0bb180 --- /dev/null +++ b/binder/date=2021+project=72k5n/a=intranode-vs-internode/a=performance_semantics+what=interintranode+compute-work=0+simels_per_cpu=1+ext=.ipynb @@ -0,0 +1,189 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import itertools as it\n", + "from iterdub import iterdub as ib\n", + "from iterpop import iterpop as ip\n", + "from keyname import keyname as kn\n", + "from matplotlib import pyplot as plt\n", + "import matplotlib\n", + "from nbmetalog import nbmetalog as nbm\n", + "import numpy as np\n", + "import pandas as pd\n", + "import patchworklib as pw\n", + "import pathlib\n", + "import random\n", + "from scipy import stats\n", + "import seaborn as sns\n", + "from slugify import slugify\n", + "import statsmodels.api as sm\n", + "import statsmodels.formula.api as smf\n", + "from teeplot import teeplot as tp\n", + "from tqdm import tqdm\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(1)\n", + "random.seed(1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from conduitpylib.viz import _get_defaults as cfg\n", + "from conduitpylib.viz import (\n", + " get_notomoji_font_path,\n", + " performance_semantics_scatterplot,\n", + " performance_semantics_facetplot,\n", + ")\n", + "from conduitpylib.wrangle import retrieve_and_prepare_delta_dataframes\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nbm.print_metadata()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Get Data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def apply(df: pd.DataFrame) -> pd.DataFrame:\n", + " row_distiller = lambda row: {\n", + " k: v for k, v in row.items() if k in (\"Num Nodes\", \"Num Processes\")\n", + " }\n", + " df[\"Multiprocessing\"] = df.apply(\n", + " lambda row: {\n", + " frozenset(\n", + " {\"Num Nodes\": 1, \"Num Processes\": 2}.items()\n", + " ): \"Intranode\",\n", + " frozenset(\n", + " {\"Num Nodes\": 2, \"Num Processes\": 2}.items()\n", + " ): \"Internode\",\n", + " }[frozenset(row_distiller(row).items())],\n", + " axis=1,\n", + " )\n", + " return df\n", + "\n", + "\n", + "longitudinal, snapshot = retrieve_and_prepare_delta_dataframes(\n", + " df_inlet_url=\"https://osf.io/4ys9v/download\",\n", + " df_outlet_url=\"https://osf.io/aum7w/download\",\n", + " apply=apply,\n", + " treatment_column=\"Multiprocessing\",\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for data in \"longitudinal\", \"snapshot\":\n", + " for allocation, group in eval(data).groupby(\"Multiprocessing\"):\n", + " group_df = group.reset_index()\n", + " tp.tee(\n", + " performance_semantics_scatterplot,\n", + " data=group_df,\n", + " x=\"Simstep Period Inlet (ns)\",\n", + " y=\"Latency Simsteps Inlet\",\n", + " heat=\"% Msgs Lost\",\n", + " legend_font_name=get_notomoji_font_path(),\n", + " title=\"__{}__\".format(\n", + " {\n", + " \"Intranode\": \"\u2681\",\n", + " \"Internode\": \"\u2680\u2680\",\n", + " }[allocation]\n", + " ),\n", + " batch_kwargs={\n", + " \"batch_size\": 10,\n", + " \"progress_apply\": tqdm,\n", + " },\n", + " bunching_smear_alpha=0.2,\n", + " scatter_kwargs={\n", + " \"alpha\": 0.4,\n", + " },\n", + " teeplot_outattrs={\n", + " \"allocation\": slugify(allocation),\n", + " \"data\": slugify(data),\n", + " },\n", + " )\n", + " plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for data in \"longitudinal\", \"snapshot\":\n", + " plt.clf()\n", + " with pw.patched_axisgrid():\n", + " facetplot = tp.tee(\n", + " performance_semantics_facetplot,\n", + " data=eval(data),\n", + " x=\"Simstep Period Inlet (ns)\",\n", + " y=\"Latency Simsteps Inlet\",\n", + " hue=\"Multiprocessing\",\n", + " hue_order=[\"Intranode\", \"Internode\"],\n", + " legend_font_name=get_notomoji_font_path(),\n", + " title=\"\u2680\u2680/\u2681\",\n", + " teeplot_outattrs={\n", + " \"data\": slugify(data),\n", + " },\n", + " )\n", + " display(facetplot)\n", + " plt.show()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/binder/date=2021+project=72k5n/a=multithreading-vs-multiprocessing/a=beleaguerment+what=multithreading-vs-multiprocessing+compute-work=0+simels_per_cpu=1+ext=.ipynb b/binder/date=2021+project=72k5n/a=multithreading-vs-multiprocessing/a=beleaguerment+what=multithreading-vs-multiprocessing+compute-work=0+simels_per_cpu=1+ext=.ipynb new file mode 100644 index 0000000000..dc438e0c70 --- /dev/null +++ b/binder/date=2021+project=72k5n/a=multithreading-vs-multiprocessing/a=beleaguerment+what=multithreading-vs-multiprocessing+compute-work=0+simels_per_cpu=1+ext=.ipynb @@ -0,0 +1,170 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "\n", + "from keyname import keyname as kn\n", + "from matplotlib import pyplot as plt\n", + "from nbmetalog import nbmetalog as nbm\n", + "import numpy as np\n", + "import pandas as pd\n", + "from scipy import stats\n", + "import seaborn as sns\n", + "from slugify import slugify\n", + "import statsmodels.api as sm\n", + "import statsmodels.formula.api as smf\n", + "from teeplot import teeplot as tp\n", + "import itertools as it\n", + "import matplotlib.patheffects as pe\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(1)\n", + "random.seed(1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from conduitpylib.viz import _get_defaults as cfg\n", + "\n", + "from conduitpylib.viz import beleaguerment_facetplot, beleaguerment_regplot\n", + "\n", + "from conduitpylib.wrangle import (\n", + " retrieve_and_prepare_delta_dataframes,\n", + " wrangle_instrumentation_longform,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nbm.print_metadata()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Get Data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def apply(df: pd.DataFrame) -> pd.DataFrame:\n", + " row_distiller = lambda row: {\n", + " k: v for k, v in row.items() if k in (\"Num Threads\", \"Num Processes\")\n", + " }\n", + " df[\"Concurrency\"] = df.apply(\n", + " lambda row: {\n", + " frozenset(\n", + " {\"Num Threads\": 1, \"Num Processes\": 2}.items()\n", + " ): \"Multiprocessing\",\n", + " frozenset(\n", + " {\"Num Threads\": 2, \"Num Processes\": 1}.items()\n", + " ): \"Multithreading\",\n", + " }[frozenset(row_distiller(row).items())],\n", + " axis=1,\n", + " )\n", + " return df\n", + "\n", + "\n", + "longitudinal_df, snapshot_df = retrieve_and_prepare_delta_dataframes(\n", + " df_inlet_url=\"https://osf.io/pe987/download\",\n", + " df_outlet_url=\"https://osf.io/vrxmq/download\",\n", + " apply=apply,\n", + " treatment_column=\"Concurrency\",\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Compare Finalized and Rolling Measures\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data = wrangle_instrumentation_longform(\n", + " longitudinal_df=longitudinal_df,\n", + " snapshot_df=snapshot_df,\n", + ")\n", + "\n", + "for treatment, group in data.groupby(\"Concurrency\"):\n", + " group_df = group.reset_index()\n", + " tp.tee(\n", + " beleaguerment_facetplot,\n", + " data=group_df,\n", + " x=\"Messages Received Per Second\",\n", + " y=\"Messages Sent Per Second\",\n", + " hue=\"Instrumentation\",\n", + " annotation_kwargs=dict(\n", + " xy=(0.05, 0.95),\n", + " ),\n", + " scatter_kwargs=dict(\n", + " alpha=0.6,\n", + " s=30,\n", + " zorder=999,\n", + " ),\n", + " kde_kwargs=dict(\n", + " cut=10,\n", + " kde_thresh=1e-3,\n", + " ),\n", + " regline_kwargs=dict(\n", + " linewidth=6,\n", + " zorder=-9999,\n", + " ),\n", + " teeplot_outattrs={\n", + " \"treatment\": slugify(treatment),\n", + " },\n", + " )\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/binder/date=2021+project=72k5n/a=multithreading-vs-multiprocessing/a=performance_semantics+what=threadproc+compute-work=0+simels_per_cpu=1+ext=.ipynb b/binder/date=2021+project=72k5n/a=multithreading-vs-multiprocessing/a=performance_semantics+what=threadproc+compute-work=0+simels_per_cpu=1+ext=.ipynb new file mode 100644 index 0000000000..b3e13c7290 --- /dev/null +++ b/binder/date=2021+project=72k5n/a=multithreading-vs-multiprocessing/a=performance_semantics+what=threadproc+compute-work=0+simels_per_cpu=1+ext=.ipynb @@ -0,0 +1,189 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import itertools as it\n", + "from iterdub import iterdub as ib\n", + "from iterpop import iterpop as ip\n", + "from keyname import keyname as kn\n", + "from matplotlib import pyplot as plt\n", + "import matplotlib\n", + "from nbmetalog import nbmetalog as nbm\n", + "import numpy as np\n", + "import pandas as pd\n", + "import patchworklib as pw\n", + "import pathlib\n", + "import random\n", + "from scipy import stats\n", + "import seaborn as sns\n", + "from slugify import slugify\n", + "import statsmodels.api as sm\n", + "import statsmodels.formula.api as smf\n", + "from teeplot import teeplot as tp\n", + "from tqdm import tqdm\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(1)\n", + "random.seed(1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from conduitpylib.viz import _get_defaults as cfg\n", + "from conduitpylib.viz import (\n", + " get_notomoji_font_path,\n", + " performance_semantics_scatterplot,\n", + " performance_semantics_facetplot,\n", + ")\n", + "from conduitpylib.wrangle import retrieve_and_prepare_delta_dataframes\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nbm.print_metadata()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Get Data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def apply(df: pd.DataFrame) -> pd.DataFrame:\n", + " row_distiller = lambda row: {\n", + " k: v for k, v in row.items() if k in (\"Num Threads\", \"Num Processes\")\n", + " }\n", + " df[\"Concurrency\"] = df.apply(\n", + " lambda row: {\n", + " frozenset(\n", + " {\"Num Threads\": 1, \"Num Processes\": 2}.items()\n", + " ): \"Multiprocessing\",\n", + " frozenset(\n", + " {\"Num Threads\": 2, \"Num Processes\": 1}.items()\n", + " ): \"Multithreading\",\n", + " }[frozenset(row_distiller(row).items())],\n", + " axis=1,\n", + " )\n", + " return df\n", + "\n", + "\n", + "longitudinal, snapshot = retrieve_and_prepare_delta_dataframes(\n", + " df_inlet_url=\"https://osf.io/pe987/download\",\n", + " df_outlet_url=\"https://osf.io/vrxmq/download\",\n", + " apply=apply,\n", + " treatment_column=\"Concurrency\",\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for data in \"longitudinal\", \"snapshot\":\n", + " for allocation, group in eval(data).groupby(\"Concurrency\"):\n", + " group_df = group.reset_index()\n", + " tp.tee(\n", + " performance_semantics_scatterplot,\n", + " data=group_df,\n", + " x=\"Simstep Period Inlet (ns)\",\n", + " y=\"Latency Simsteps Inlet\",\n", + " heat=\"% Msgs Lost\",\n", + " legend_font_name=get_notomoji_font_path(),\n", + " title=\"__{}__\".format(\n", + " {\n", + " \"Multithreading\": \"\ud83e\udea1\",\n", + " \"Multiprocessing\": \"\u2699\ufe0f\",\n", + " }[allocation]\n", + " ),\n", + " batch_kwargs={\n", + " \"batch_size\": 10,\n", + " \"progress_apply\": tqdm,\n", + " },\n", + " bunching_smear_alpha=0.2,\n", + " scatter_kwargs={\n", + " \"alpha\": 0.4,\n", + " },\n", + " teeplot_outattrs={\n", + " \"allocation\": slugify(allocation),\n", + " \"data\": slugify(data),\n", + " },\n", + " )\n", + " plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for data in \"longitudinal\", \"snapshot\":\n", + " plt.clf()\n", + " with pw.patched_axisgrid():\n", + " facetplot = tp.tee(\n", + " performance_semantics_facetplot,\n", + " data=eval(data),\n", + " x=\"Simstep Period Inlet (ns)\",\n", + " y=\"Latency Simsteps Inlet\",\n", + " hue=\"Concurrency\",\n", + " hue_order=[\"Multithreading\", \"Multiprocessing\"],\n", + " legend_font_name=get_notomoji_font_path(),\n", + " title=\"\ud83e\udea1/\u2699\ufe0f\",\n", + " teeplot_outattrs={\n", + " \"data\": slugify(data),\n", + " },\n", + " )\n", + " display(facetplot)\n", + " plt.show()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/binder/date=2021+project=72k5n/a=weak-scaling/a=beleaguerment+what=weak-scaling+compute-work=0+simels_per_cpu=2048+ext=.ipynb_ b/binder/date=2021+project=72k5n/a=weak-scaling/a=beleaguerment+what=weak-scaling+compute-work=0+simels_per_cpu=2048+ext=.ipynb_ new file mode 100644 index 0000000000..fd7e4d3683 --- /dev/null +++ b/binder/date=2021+project=72k5n/a=weak-scaling/a=beleaguerment+what=weak-scaling+compute-work=0+simels_per_cpu=2048+ext=.ipynb_ @@ -0,0 +1,165 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "\n", + "from keyname import keyname as kn\n", + "from matplotlib import pyplot as plt\n", + "from nbmetalog import nbmetalog as nbm\n", + "import numpy as np\n", + "import pandas as pd\n", + "from scipy import stats\n", + "import seaborn as sns\n", + "from slugify import slugify\n", + "import statsmodels.api as sm\n", + "import statsmodels.formula.api as smf\n", + "from teeplot import teeplot as tp\n", + "import itertools as it\n", + "import matplotlib.patheffects as pe\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(1)\n", + "random.seed(1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from conduitpylib.viz import _get_defaults as cfg\n", + "\n", + "from conduitpylib.viz import beleaguerment_facetplot, beleaguerment_regplot\n", + "\n", + "from conduitpylib.wrangle import (\n", + " retrieve_and_prepare_delta_dataframes,\n", + " wrangle_instrumentation_longform,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nbm.print_metadata()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Get Data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "longitudinal_df, snapshot_df = retrieve_and_prepare_delta_dataframes(\n", + " df_inlet_url=\"https://osf.io/2rdj6/download\",\n", + " df_outlet_url=\"https://osf.io/9utpr/download\",\n", + " treatment_column=None,\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Compare Finalized and Rolling Measures\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data = wrangle_instrumentation_longform(\n", + " longitudinal_df=longitudinal_df,\n", + " snapshot_df=snapshot_df,\n", + ")\n", + "treatment_variables = [\n", + " \"Cpus Per Node\",\n", + " \"Num Simels Per Cpu\",\n", + " \"Num Processes\",\n", + "]\n", + "\n", + "\n", + "for treatment, group in data.groupby(treatment_variables):\n", + " group_df = group.reset_index()\n", + " tp.tee(\n", + " beleaguerment_facetplot,\n", + " data=group_df,\n", + " x=\"Messages Received Per Second\",\n", + " y=\"Messages Sent Per Second\",\n", + " hue=\"Instrumentation\",\n", + " annotation_kwargs=dict(\n", + " xy=(0.05, 0.95),\n", + " ),\n", + " scatter_kwargs=dict(\n", + " alpha=0.6,\n", + " s=30,\n", + " zorder=999,\n", + " ),\n", + " kde_kwargs=dict(\n", + " cut=10,\n", + " kde_thresh=1e-3,\n", + " ),\n", + " regline_kwargs=dict(\n", + " linewidth=6,\n", + " zorder=-9999,\n", + " ),\n", + " teeplot_outattrs={\n", + " \"treatment\": slugify(\n", + " \"-\".join(\n", + " map(\n", + " \"-\".join,\n", + " zip(treatment_variables, map(str, treatment)),\n", + " ),\n", + " ),\n", + " ),\n", + " },\n", + " )\n", + " plt.show()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/binder/date=2021+project=72k5n/a=weak-scaling/a=performance_semantics+what=weak-scaling+compute-work=0+simels_per_cpu=2048+ext=.ipynb_ b/binder/date=2021+project=72k5n/a=weak-scaling/a=performance_semantics+what=weak-scaling+compute-work=0+simels_per_cpu=2048+ext=.ipynb_ new file mode 100644 index 0000000000..1ddfcd0fb2 --- /dev/null +++ b/binder/date=2021+project=72k5n/a=weak-scaling/a=performance_semantics+what=weak-scaling+compute-work=0+simels_per_cpu=2048+ext=.ipynb_ @@ -0,0 +1,211 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "\n", + "from keyname import keyname as kn\n", + "from matplotlib import pyplot as plt\n", + "from nbmetalog import nbmetalog as nbm\n", + "import numpy as np\n", + "import pandas as pd\n", + "import patchworklib as pw\n", + "from scipy import stats\n", + "import seaborn as sns\n", + "from slugify import slugify\n", + "import statsmodels.api as sm\n", + "import statsmodels.formula.api as smf\n", + "from teeplot import teeplot as tp\n", + "import itertools as it\n", + "import matplotlib.patheffects as pe\n", + "from tqdm import tqdm\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(1)\n", + "random.seed(1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from conduitpylib.viz import _get_defaults as cfg\n", + "\n", + "from conduitpylib.viz import (\n", + " get_notomoji_font_path,\n", + " performance_semantics_scatterplot,\n", + " performance_semantics_facetplot,\n", + ")\n", + "\n", + "from conduitpylib.wrangle import (\n", + " retrieve_and_prepare_delta_dataframes,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nbm.print_metadata()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Get Data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "longitudinal, snapshot = retrieve_and_prepare_delta_dataframes(\n", + " df_inlet_url=\"https://osf.io/2rdj6/download\",\n", + " df_outlet_url=\"https://osf.io/9utpr/download\",\n", + " treatment_column=None,\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Compare Finalized and Rolling Measures\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "treatment_variables = [\n", + " \"Cpus Per Node\",\n", + " \"Num Simels Per Cpu\",\n", + " \"Num Processes\",\n", + "]\n", + "\n", + "for data in \"longitudinal\", \"snapshot\":\n", + " for treatment, group in eval(data).groupby(treatment_variables):\n", + " group_df = group.reset_index()\n", + " tp.tee(\n", + " performance_semantics_scatterplot,\n", + " data=group_df,\n", + " x=\"Simstep Period Inlet (ns)\",\n", + " y=\"Latency Simsteps Inlet\",\n", + " heat=\"% Msgs Lost\",\n", + " legend_font_name=get_notomoji_font_path(),\n", + " title=(\n", + " \"\ud83d\udcf6\"\n", + " + \"{}\".format(\"\u2610\u2680\u2681\u2682\u2683\u2684\u2685\"[treatment[0]])\n", + " + (\"\ud83c\udfe0\" if treatment[1] == 1 else \"\ud83c\udfd8\ufe0f\")\n", + " ),\n", + " batch_kwargs={\n", + " \"batch_size\": 10,\n", + " \"progress_apply\": tqdm,\n", + " },\n", + " bunching_smear_alpha=0.2,\n", + " scatter_kwargs={\n", + " \"alpha\": 0.4,\n", + " },\n", + " teeplot_outattrs={\n", + " \"treatment\": slugify(\n", + " \"-\".join(\n", + " map(\n", + " \"-\".join,\n", + " zip(treatment_variables, map(str, treatment)),\n", + " ),\n", + " ),\n", + " ),\n", + " },\n", + " )\n", + " plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "treatment_variables = [\n", + " \"Cpus Per Node\",\n", + " \"Num Simels Per Cpu\",\n", + "]\n", + "\n", + "for data in \"longitudinal\", \"snapshot\":\n", + " for treatment, group in eval(data).groupby(treatment_variables):\n", + " group_df = group.reset_index()\n", + " plt.clf()\n", + " with pw.patched_axisgrid():\n", + " facetplot = tp.tee(\n", + " performance_semantics_facetplot,\n", + " data=eval(data),\n", + " x=\"Simstep Period Inlet (ns)\",\n", + " y=\"Latency Simsteps Inlet\",\n", + " hue=\"Log Num Processes\",\n", + " legend_font_name=get_notomoji_font_path(),\n", + " scatter_kwargs={\n", + " \"x\": \"Simstep Period Inlet (ns)\",\n", + " },\n", + " sharexy=True,\n", + " title=(\n", + " \"\ud83d\udcf6\"\n", + " + \"{}\".format(\"\u2610\u2680\u2681\u2682\u2683\u2684\u2685\"[treatment[0]])\n", + " + (\"\ud83c\udfe0\" if treatment[1] == 1 else \"\ud83c\udfd8\ufe0f\")\n", + " ),\n", + " teeplot_outattrs={\n", + " \"data\": slugify(data),\n", + " \"treatment\": slugify(\n", + " \"-\".join(\n", + " map(\n", + " \"-\".join,\n", + " zip(treatment_variables, map(str, treatment)),\n", + " ),\n", + " ),\n", + " ),\n", + " },\n", + " )\n", + " display(facetplot)\n", + " plt.show()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/binder/date=2021+project=72k5n/a=weak-scaling/a=weak-scaling+compute-work=0+simels_per_cpu=1_2048+ext=.ipynb b/binder/date=2021+project=72k5n/a=weak-scaling/a=weak-scaling+compute-work=0+simels_per_cpu=1_2048+ext=.ipynb index b15be51622..b741cbe68d 100644 --- a/binder/date=2021+project=72k5n/a=weak-scaling/a=weak-scaling+compute-work=0+simels_per_cpu=1_2048+ext=.ipynb +++ b/binder/date=2021+project=72k5n/a=weak-scaling/a=weak-scaling+compute-work=0+simels_per_cpu=1_2048+ext=.ipynb @@ -57,8 +57,6 @@ "df_inlet = pd.read_csv(\n", " 'https://osf.io/2rdj6/download',\n", " compression='gzip',\n", - ").dropna(\n", - " subset=['Process Instance UUID'],\n", ")\n", "nbm.print_dataframe_summary(*eval(nbm.nvp_expr(\n", " 'df_inlet'\n", diff --git a/binder/date=2021+project=72k5n/a=with-lac-417-vs-sans-lac-417/_rename_allocation.py b/binder/date=2021+project=72k5n/a=with-lac-417-vs-sans-lac-417/_rename_allocation.py new file mode 100644 index 0000000000..c107d5f157 --- /dev/null +++ b/binder/date=2021+project=72k5n/a=with-lac-417-vs-sans-lac-417/_rename_allocation.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def rename_allocation(df: pd.DataFrame) -> pd.DataFrame: + df = df.copy() + df["Allocation"] = df["Allocation"].map( + { + "With lac-417": "Incl. lac-417", + "Sans lac-417": "Excl. lac-417", + } + ) + return df diff --git a/binder/date=2021+project=72k5n/a=with-lac-417-vs-sans-lac-417/a=beleaguerment+what=lac-417+compute-work=0+simels_per_cpu=2048+ext=.ipynb b/binder/date=2021+project=72k5n/a=with-lac-417-vs-sans-lac-417/a=beleaguerment+what=lac-417+compute-work=0+simels_per_cpu=2048+ext=.ipynb new file mode 100644 index 0000000000..8723612574 --- /dev/null +++ b/binder/date=2021+project=72k5n/a=with-lac-417-vs-sans-lac-417/a=beleaguerment+what=lac-417+compute-work=0+simels_per_cpu=2048+ext=.ipynb @@ -0,0 +1,141 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "\n", + "from keyname import keyname as kn\n", + "from matplotlib import pyplot as plt\n", + "from nbmetalog import nbmetalog as nbm\n", + "import numpy as np\n", + "from slugify import slugify\n", + "from teeplot import teeplot as tp\n", + "from tqdm import tqdm\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(1)\n", + "random.seed(1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from conduitpylib.viz import beleaguerment_facetplot, beleaguerment_regplot\n", + "from conduitpylib.wrangle import (\n", + " retrieve_and_prepare_delta_dataframes,\n", + " wrangle_instrumentation_longform,\n", + ")\n", + "from _rename_allocation import rename_allocation\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nbm.print_metadata()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Get Data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "longitudinal_df, snapshot_df = retrieve_and_prepare_delta_dataframes(\n", + " df_inlet_url=\"https://osf.io/jgpnv/download\",\n", + " df_outlet_url=\"https://osf.io/ncdfq/download\",\n", + " treatment_column=\"Allocation\",\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Compare Finalized and Rolling Measures\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data = wrangle_instrumentation_longform(\n", + " longitudinal_df=longitudinal_df,\n", + " snapshot_df=snapshot_df,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for allocation, group in data.groupby(\"Allocation\"):\n", + " group_df = group.reset_index()\n", + " tp.tee(\n", + " beleaguerment_facetplot,\n", + " data=rename_allocation(group_df),\n", + " x=\"Messages Received Per Second\",\n", + " y=\"Messages Sent Per Second\",\n", + " hue=\"Instrumentation\",\n", + " rugplot_kwargs={\n", + " \"alpha\": 1.0,\n", + " \"linewidth\": 1,\n", + " \"batch_limit\": 5,\n", + " \"progress_apply\": tqdm,\n", + " },\n", + " teeplot_outattrs={\n", + " \"allocation\": slugify(allocation),\n", + " },\n", + " )\n", + " plt.show()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/binder/date=2021+project=72k5n/a=with-lac-417-vs-sans-lac-417/a=instrumentation+what=lac-417+compute-work=0+simels_per_cpu=2048+ext=.ipynb b/binder/date=2021+project=72k5n/a=with-lac-417-vs-sans-lac-417/a=instrumentation+what=lac-417+compute-work=0+simels_per_cpu=2048+ext=.ipynb new file mode 100644 index 0000000000..4fba542f42 --- /dev/null +++ b/binder/date=2021+project=72k5n/a=with-lac-417-vs-sans-lac-417/a=instrumentation+what=lac-417+compute-work=0+simels_per_cpu=2048+ext=.ipynb @@ -0,0 +1,164 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import contextlib\n", + "import random\n", + "\n", + "from keyname import keyname as kn\n", + "from matplotlib import pyplot as plt\n", + "from nbmetalog import nbmetalog as nbm\n", + "import numpy as np\n", + "import pandas as pd\n", + "import patchworklib as pw\n", + "from scipy import stats\n", + "import seaborn as sns\n", + "from slugify import slugify\n", + "from teeplot import teeplot as tp\n", + "import itertools as it\n", + "import matplotlib.patheffects as pe\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(1)\n", + "random.seed(1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from conduitpylib.viz import (\n", + " runtime_indicators_concat_regplot_pairplot,\n", + " runtime_indicators_pairplot,\n", + " runtime_indicators_regplot,\n", + ")\n", + "from conduitpylib.wrangle import (\n", + " retrieve_and_prepare_delta_dataframes,\n", + " wrangle_instrumentation_longform,\n", + ")\n", + "from _rename_allocation import rename_allocation\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nbm.print_metadata()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Get Data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "longitudinal_df, snapshot_df = retrieve_and_prepare_delta_dataframes(\n", + " df_inlet_url=\"https://osf.io/jgpnv/download\",\n", + " df_outlet_url=\"https://osf.io/ncdfq/download\",\n", + " treatment_column=\"Allocation\",\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Compare Finalized and Rolling Measures\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data = wrangle_instrumentation_longform(\n", + " longitudinal_df=longitudinal_df,\n", + " snapshot_df=snapshot_df,\n", + ")\n", + "\n", + "patchwork_context = pw.patched_axisgrid\n", + "for context in contextlib.nullcontext, patchwork_context:\n", + " for allocation, group in data.groupby(\"Allocation\"):\n", + " plt.clf()\n", + " with context():\n", + " group_df = group.reset_index()\n", + " regplot_facetgrid = tp.tee(\n", + " runtime_indicators_regplot,\n", + " data=rename_allocation(group_df),\n", + " compare=\"Instrumentation\",\n", + " comparison_x=\"Longitudinal\",\n", + " comparison_y=\"Snapshot\",\n", + " teeplot_outattrs={\n", + " \"allocation\": slugify(allocation),\n", + " },\n", + " teeplot_save=(context is not patchwork_context),\n", + " )\n", + " pairplot_grid = tp.tee(\n", + " runtime_indicators_pairplot,\n", + " data=rename_allocation(group_df),\n", + " hue=\"Instrumentation\",\n", + " teeplot_outattrs={\n", + " \"allocation\": slugify(allocation),\n", + " },\n", + " teeplot_save=(context is not patchwork_context),\n", + " )\n", + "\n", + " if context is patchwork_context:\n", + " tp.tee(\n", + " runtime_indicators_concat_regplot_pairplot,\n", + " regplot_facetgrid,\n", + " pairplot_grid,\n", + " teeplot_outattrs={\n", + " \"allocation\": slugify(allocation),\n", + " },\n", + " )\n", + "\n", + " plt.show()\n", + " plt.clf()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/binder/date=2021+project=72k5n/a=with-lac-417-vs-sans-lac-417/a=performance_semantics+what=lac-417+compute-work=0+simels_per_cpu=2048+ext=.ipynb b/binder/date=2021+project=72k5n/a=with-lac-417-vs-sans-lac-417/a=performance_semantics+what=lac-417+compute-work=0+simels_per_cpu=2048+ext=.ipynb new file mode 100644 index 0000000000..72eb811b9f --- /dev/null +++ b/binder/date=2021+project=72k5n/a=with-lac-417-vs-sans-lac-417/a=performance_semantics+what=lac-417+compute-work=0+simels_per_cpu=2048+ext=.ipynb @@ -0,0 +1,169 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import itertools as it\n", + "from iterdub import iterdub as ib\n", + "from iterpop import iterpop as ip\n", + "from keyname import keyname as kn\n", + "from matplotlib import pyplot as plt\n", + "import matplotlib\n", + "from nbmetalog import nbmetalog as nbm\n", + "import numpy as np\n", + "import pandas as pd\n", + "import patchworklib as pw\n", + "import pathlib\n", + "import random\n", + "from scipy import stats\n", + "import seaborn as sns\n", + "from slugify import slugify\n", + "import statsmodels.api as sm\n", + "import statsmodels.formula.api as smf\n", + "from teeplot import teeplot as tp\n", + "from tqdm import tqdm\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(1)\n", + "random.seed(1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from conduitpylib.viz import _get_defaults as cfg\n", + "from conduitpylib.viz import (\n", + " get_notomoji_font_path,\n", + " performance_semantics_scatterplot,\n", + " performance_semantics_facetplot,\n", + ")\n", + "from conduitpylib.wrangle import retrieve_and_prepare_delta_dataframes\n", + "from _rename_allocation import rename_allocation\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nbm.print_metadata()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Get Data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "longitudinal, snapshot = retrieve_and_prepare_delta_dataframes(\n", + " df_inlet_url=\"https://osf.io/jgpnv/download\",\n", + " df_outlet_url=\"https://osf.io/ncdfq/download\",\n", + " treatment_column=\"Allocation\",\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for data in \"longitudinal\", \"snapshot\":\n", + " for allocation, group in eval(data).groupby(\"Allocation\"):\n", + " group_df = group.reset_index()\n", + " tp.tee(\n", + " performance_semantics_scatterplot,\n", + " data=rename_allocation(group_df),\n", + " x=\"Simstep Period Inlet (ns)\",\n", + " y=\"Latency Simsteps Inlet\",\n", + " heat=\"% Msgs Lost\",\n", + " legend_font_name=get_notomoji_font_path(),\n", + " title=\"__{}__\".format(\n", + " {\n", + " \"With lac-417\": \"\ud83e\udd15\",\n", + " \"Sans lac-417\": \"\ud83d\ude42\",\n", + " }[allocation]\n", + " ),\n", + " batch_kwargs={\n", + " \"batch_size\": 10,\n", + " \"progress_apply\": tqdm,\n", + " },\n", + " bunching_smear_alpha=0.2,\n", + " scatter_kwargs={\n", + " \"alpha\": 0.4,\n", + " },\n", + " teeplot_outattrs={\n", + " \"allocation\": slugify(allocation),\n", + " \"data\": slugify(data),\n", + " },\n", + " )\n", + " plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for data in \"longitudinal\", \"snapshot\":\n", + " plt.clf()\n", + " with pw.patched_axisgrid():\n", + " facetplot = tp.tee(\n", + " performance_semantics_facetplot,\n", + " data=rename_allocation(eval(data)),\n", + " x=\"Simstep Period Inlet (ns)\",\n", + " y=\"Latency Simsteps Inlet\",\n", + " hue=\"Allocation\",\n", + " legend_font_name=get_notomoji_font_path(),\n", + " title=\"\ud83d\ude42/\ud83e\udd15\",\n", + " teeplot_outattrs={\n", + " \"data\": slugify(data),\n", + " },\n", + " )\n", + " display(facetplot)\n", + " plt.show()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/binder/date=2021+project=72k5n/a=with-lac-417-vs-sans-lac-417/a=with-lac-417-vs-sans-lac-417+compute-work=0+simels_per_cpu=2048+ext=.ipynb b/binder/date=2021+project=72k5n/a=with-lac-417-vs-sans-lac-417/a=with-lac-417-vs-sans-lac-417+compute-work=0+simels_per_cpu=2048+ext=.ipynb index 81dabe7669..f39324c8b6 100644 --- a/binder/date=2021+project=72k5n/a=with-lac-417-vs-sans-lac-417/a=with-lac-417-vs-sans-lac-417+compute-work=0+simels_per_cpu=2048+ext=.ipynb +++ b/binder/date=2021+project=72k5n/a=with-lac-417-vs-sans-lac-417/a=with-lac-417-vs-sans-lac-417+compute-work=0+simels_per_cpu=2048+ext=.ipynb @@ -6,6 +6,8 @@ "metadata": {}, "outputs": [], "source": [ + "import warnings\n", + "\n", "from iterdub import iterdub as ib\n", "from iterpop import iterpop as ip\n", "from keyname import keyname as kn\n", @@ -29,316 +31,34 @@ "metadata": {}, "outputs": [], "source": [ - "from conduitpylib.utils import consolidate_merge\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "nbm.print_metadata()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Get Data\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_inlet = pd.read_csv(\n", - " 'https://osf.io/jgpnv/download',\n", - " compression='gzip',\n", - ").dropna(\n", - " subset=['Process Instance UUID'],\n", - ")\n", - "nbm.print_dataframe_summary(*eval(nbm.nvp_expr(\n", - " 'df_inlet'\n", - ")))\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_outlet = pd.read_csv(\n", - " 'https://osf.io/ncdfq/download',\n", - " compression='gzip',\n", - ").dropna(\n", - " subset=['Process Instance UUID'],\n", - ")\n", - "nbm.print_dataframe_summary(*eval(nbm.nvp_expr(\n", - " 'df_outlet'\n", - ")))\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df = consolidate_merge(\n", - " df_inlet,\n", - " df_outlet,\n", - " on=['Process Instance UUID', 'Update'],\n", - " suffixes=(' Inlet', ' Outlet'),\n", - " how='outer',\n", + "from conduitpylib.utils import (\n", + " consolidate_merge,\n", + " count_outliers,\n", + " count_nonoutliers,\n", + " count_proportion_outliers,\n", ")\n", - "if all(df_inlet['Runtime Seconds Elapsed'] == df_outlet['Runtime Seconds Elapsed']):\n", - " df['Runtime Seconds Elapsed Inlet'] = df['Runtime Seconds Elapsed']\n", - " df['Runtime Seconds Elapsed Outlet'] = df['Runtime Seconds Elapsed']\n", - "nbm.print_dataframe_synopsis(*eval(nbm.nvp_expr(\n", - " 'df'\n", - ")))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Prep Data\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df = df.astype({\n", - " 'Num Inlets' : 'int64',\n", - " 'Num Outlets' : 'int64',\n", - " 'Num Puts Attempted' : 'int64',\n", - " 'Num Try Puts Attempted' : 'int64',\n", - " 'Num Blocking Puts' : 'int64',\n", - " 'Num Try Puts That Succeeded' : 'int64',\n", - " 'Num Puts That Succeeded Eventually' : 'int64',\n", - " 'Num Blocking Puts That Succeeded Immediately' : 'int64',\n", - " 'Num Puts That Succeeded Immediately' : 'int64',\n", - " 'Num Puts That Blocked' : 'int64',\n", - " 'Num Dropped Puts' : 'int64',\n", - " 'Num Round Trip Touches Inlet' : 'int64',\n", - " 'Net Flux Through Duct' : 'int64',\n", - " 'proc' : 'int64',\n", - " 'Snapshot' : 'int64',\n", - " 'Has Execution Blur' : 'bool',\n", - " 'Replicate' : 'int64',\n", - " 'Async Mode' : 'int64',\n", - " 'Num Threads' : 'int64',\n", - " 'Num Processes' : 'int64',\n", - " 'SLURM_NNODES' : 'int64',\n", - " 'SLURM_NTASKS' : 'int64',\n", - " 'SLURM_CPUS_ON_NODE' : 'int64',\n", - "})\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df['Hostname'] = df.apply(\n", - " lambda row: kn.unpack(row['Source File Inlet'])['_hostname'],\n", - " axis=1,\n", - ")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df['Num Nodes'] = df['SLURM_NNODES']\n", - "df['Num Tasks'] = df['SLURM_NTASKS']\n", - "df['Num Simels Per Cpu'] = df['Num Simulation Elements Per Cpu']\n", - "df['Num Cpus'] = df['Num Threads'] * df['Num Processes']\n", - "df['Allocated Tasks Per Node'] = df['Num Tasks'] // df['Num Nodes']\n", - "df['Cpus Per Node'] = df['Allocated Tasks Per Node']\n", - "df['Delivery Time Inlet'] = (df['Num Puts Attempted'] - 1) / df['Num Round Trip Touches Inlet']\n", - "df['Delivery Time Outlet'] = (df['Num Pulls Attempted'] - 1) / df['Num Round Trip Touches Outlet']\n", - "df['Intermittancy'] = df['Num Pulls That Were Laden Immediately'] / df[['Net Flux Through Duct', 'Num Pulls Attempted']].min(axis=1)\n", - "df['Inlet-Seconds Elapsed'] = df['Num Inlets'] * df['Runtime Seconds Elapsed Inlet']\n", - "df['Outlet-Seconds Elapsed'] = df['Num Outlets'] * df['Runtime Seconds Elapsed Outlet']\n", - "df['Latency Simsteps Inlet'] = df['Delivery Time Inlet']\n", - "df['Latency Simsteps Outlet'] = df['Delivery Time Inlet']\n", - "df['Simstep Period Inlet (s)'] = df['Inlet-Seconds Elapsed'] / df['Num Puts Attempted']\n", - "df['Simstep Period Outlet (s)'] = df['Outlet-Seconds Elapsed'] / df['Num Pulls Attempted']\n", - "df['Latency Walltime Inlet (s)'] = df['Latency Simsteps Inlet'] * df['Simstep Period Inlet (s)']\n", - "df['Latency Walltime Outlet (s)'] = df['Latency Simsteps Outlet'] * df['Simstep Period Outlet (s)']\n", - "df['Log Num Processes'] = np.log(df['Num Processes']) / np.log(4)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "row_distiller = lambda row: {k : v for k, v in row.items() if k in ('Num Nodes', 'Num Processes')}\n", "\n", - "allocation_idx_mapper = {\n", - " val : idx\n", - " for idx, val\n", - " in enumerate(df['Allocation'].unique())\n", - "}\n", - "allocation_idx_mapped_title = ' | '.join(f'{idx} = {val}' for val, idx in allocation_idx_mapper.items())\n", - "df[allocation_idx_mapped_title] = df.apply(\n", - " lambda row: allocation_idx_mapper[row['Allocation']],\n", - " axis=1,\n", + "from conduitpylib.wrangle import (\n", + " find_treat_idx_mapped_col,\n", + " retrieve_and_prepare_delta_dataframes,\n", + " wrangle_world_sums,\n", ")\n" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Prep DataFrame Variants\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# https://stackoverflow.com/a/40629420\n", - "df_finalized_observations = df.sort_values('Update', ascending=False).drop_duplicates(['Process Instance UUID'])\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_blurry_snapshots = df[\n", - " df['Has Execution Blur'].astype(bool)\n", - " & (df['Snapshot'] <= 5 )\n", - " # exclude excess, unintended snapshots from runs that took a while to shut down\n", - " # (i.e., from at the 6 minute mark and beyond)\n", - "]\n" - ] - }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "df_world_sum = df_finalized_observations.groupby([\n", - " 'Replicate',\n", - " 'Async Mode',\n", - " 'Num Processes',\n", - " 'Num Nodes',\n", - " 'Num Simels Per Cpu',\n", - " 'Allocated Tasks Per Node',\n", - " 'Cpus Per Node',\n", - " 'Allocation',\n", - " allocation_idx_mapped_title,\n", - "], as_index=False).sum()\n", - "\n", - "df_world_sum['Fraction Messages Utilized'] = df_world_sum['Num Reads That Were Fresh'] / df_world_sum['Num Try Puts Attempted']\n", - "df_world_sum['Fraction Messages Delivered'] = df_world_sum['Num Try Puts That Succeeded'] / df_world_sum['Num Try Puts Attempted']\n", - "df_world_sum['Delivery Failure Rate'] = 1.0 - df_world_sum['Fraction Messages Delivered']\n", - "df_world_sum['Fraction Messages Dropped'] = df_world_sum['Delivery Failure Rate']\n", - "df_world_sum['Fraction Try Pulls That Were Laden'] = df_world_sum['Num Try Pulls That Were Laden'] / df_world_sum['Num Try Pulls Attempted']\n", - "df_world_sum['Round Trip Touches Per Attempted Pull'] = df_world_sum['Num Round Trip Touches Outlet'] / df_world_sum['Num Try Pulls Attempted']\n", - "df_world_sum['Round Trip Touches Per Attempted Put'] = df_world_sum['Num Round Trip Touches Inlet'] / df_world_sum['Num Try Puts Attempted']\n", - "df_world_sum['Num Inflight Messages'] = 2.0 / df_world_sum['Round Trip Touches Per Attempted Put'] - 1\n", - "df_world_sum['Fraction Duct Flux Stepped Through'] = df_world_sum['Num Revisions Pulled'] / df_world_sum['Net Flux Through Duct']\n", - "df_world_sum['Fraction Duct Flux Jumped Over'] = 1.0 - df_world_sum['Fraction Duct Flux Stepped Through']\n", - "df_world_sum['Round Trip Touches Per Runtime Second'] = df_world_sum['Num Round Trip Touches Inlet'] / df_world_sum['Runtime Seconds Elapsed Inlet']\n", - "df_world_sum['Latency Simsteps Inlet'] = (df_world_sum['Num Puts Attempted'] - 1) / df_world_sum['Num Round Trip Touches Inlet']\n", - "df_world_sum['Latency Simsteps Outlet'] = (df_world_sum['Num Pulls Attempted'] - 1) / df_world_sum['Num Round Trip Touches Outlet']\n", - "df_world_sum['Delivery Clumpiness'] = 1.0 - df_world_sum['Num Pulls That Were Laden Immediately'] / df_world_sum[['Net Flux Through Duct', 'Num Pulls Attempted']].min(axis=1)\n", - "df_world_sum['Intermittancy'] = df_world_sum['Delivery Clumpiness']\n", - "df_world_sum['Simstep Period Inlet (s)'] = df_world_sum['Inlet-Seconds Elapsed'] / df_world_sum['Num Puts Attempted']\n", - "df_world_sum['Simstep Period Outlet (s)'] = df_world_sum['Outlet-Seconds Elapsed'] / df_world_sum['Num Pulls Attempted']\n", - "df_world_sum['Latency Walltime Inlet (s)'] = df_world_sum['Latency Simsteps Inlet'] * df_world_sum['Simstep Period Inlet (s)']\n", - "df_world_sum['Latency Walltime Outlet (s)'] = df_world_sum['Latency Simsteps Outlet'] * df_world_sum['Simstep Period Outlet (s)']\n" + "nbm.print_metadata()\n" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "df_snapshot_diffs = df_blurry_snapshots.groupby(\n", - " [\n", - " 'Process Instance UUID',\n", - " 'Snapshot',\n", - " # subsequent items aren't meaningful to groupby\n", - " # but are just included so they pass through untouched\n", - " 'Async Mode',\n", - " 'Num Nodes',\n", - " 'Allocated Tasks Per Node',\n", - " 'Cpus Per Node',\n", - " 'Num Processes',\n", - " 'Log Num Processes',\n", - " 'Num Simels Per Cpu',\n", - " 'Replicate',\n", - " 'proc',\n", - " 'Hostname',\n", - " 'Num Inlets',\n", - " 'Num Outlets',\n", - " 'Execution Instance UUID',\n", - " 'Num Threads',\n", - " 'Allocation',\n", - " allocation_idx_mapped_title,\n", - "\n", - " ],\n", - " as_index=False,\n", - ").aggregate({\n", - " 'Num Puts Attempted' : np.ptp,\n", - " 'Num Try Puts Attempted' : np.ptp,\n", - " 'Num Blocking Puts' : np.ptp,\n", - " 'Num Try Puts That Succeeded' : np.ptp,\n", - " 'Num Puts That Succeeded Eventually' : np.ptp,\n", - " 'Num Blocking Puts That Succeeded Immediately' : np.ptp,\n", - " 'Num Puts That Succeeded Immediately' : np.ptp,\n", - " 'Num Puts That Blocked' : np.ptp,\n", - " 'Num Dropped Puts' : np.ptp,\n", - " 'Num Reads Performed' : np.ptp,\n", - " 'Num Reads That Were Fresh' : np.ptp,\n", - " 'Num Reads That Were Stale' : np.ptp,\n", - " 'Num Revisions Pulled' : np.ptp,\n", - " 'Num Try Pulls Attempted' : np.ptp,\n", - " 'Num Blocking Pulls' : np.ptp,\n", - " 'Num Blocking Pulls That Blocked' : np.ptp,\n", - " 'Num Revisions From Try Pulls' : np.ptp,\n", - " 'Num Revisions From Blocking Pulls' : np.ptp,\n", - " 'Num Pulls Attempted' : np.ptp,\n", - " 'Num Pulls That Were Laden Eventually' : np.ptp,\n", - " 'Num Blocking Pulls That Were Laden Immediately' : np.ptp,\n", - " 'Num Blocking Pulls That Were Laden Eventually' : np.ptp,\n", - " 'Num Pulls That Were Laden Immediately' : np.ptp,\n", - " 'Num Try Pulls That Were Laden' : np.ptp,\n", - " 'Num Try Pulls That Were Unladen' : np.ptp,\n", - " 'Net Flux Through Duct' : np.ptp,\n", - " 'Num Round Trip Touches Inlet' : np.ptp,\n", - " 'Num Round Trip Touches Outlet' : np.ptp,\n", - "# why are these missing?\n", - "# 'Row Initial Timepoint (ns) Inlet' : np.ptp,\n", - "# 'Row Initial Timepoint (ns) Outlet' : np.ptp,\n", - " 'Row Final Timepoint (ns) Inlet' : np.ptp,\n", - " 'Row Final Timepoint (ns) Outlet' : np.ptp,\n", - " 'Runtime Seconds Elapsed Inlet' : np.mean,\n", - " 'Runtime Seconds Elapsed Outlet' : np.mean,\n", - "})\n" + "# Get Data\n" ] }, { @@ -347,45 +67,18 @@ "metadata": {}, "outputs": [], "source": [ - "df_snapshot_diffs['Fraction Messages Delivered'] = (\n", - " df_snapshot_diffs['Num Try Puts That Succeeded']\n", - " / df_snapshot_diffs['Num Try Puts Attempted']\n", - ")\n", - "df_snapshot_diffs['Delivery Success Rate'] = (\n", - " df_snapshot_diffs['Num Try Puts That Succeeded']\n", - " / df_snapshot_diffs['Num Try Puts Attempted']\n", + "merge_df, (\n", + " df_finalized_observations,\n", + " df_snapshot_diffs,\n", + ") = retrieve_and_prepare_delta_dataframes(\n", + " df_inlet_url=\"https://osf.io/jgpnv/download\",\n", + " df_outlet_url=\"https://osf.io/ncdfq/download\",\n", + " treatment_column=\"Allocation\",\n", + " return_merge_df=True,\n", ")\n", - "df_snapshot_diffs['Delivery Failure Rate'] = 1 - df_snapshot_diffs['Delivery Success Rate']\n", - "df_snapshot_diffs['Fraction Messages Dropped'] = df_snapshot_diffs['Delivery Failure Rate']\n", - "df_snapshot_diffs['Fraction Try Pulls That Were Laden'] = (\n", - " df_snapshot_diffs['Num Try Pulls That Were Laden']\n", - " / df_snapshot_diffs['Num Try Pulls Attempted']\n", - ")\n", - "\n", - "df_snapshot_diffs['Round Trip Touches Per Attempted Put'] = (\n", - " df_snapshot_diffs['Num Round Trip Touches Inlet']\n", - ") / df_snapshot_diffs['Num Try Puts Attempted']\n", "\n", - "df_snapshot_diffs['Round Trip Touches Per Attempted Pull'] = (\n", - " df_snapshot_diffs['Num Round Trip Touches Outlet']\n", - ") / df_snapshot_diffs['Num Try Pulls Attempted']\n", - "\n", - "df_snapshot_diffs['Round Trip Touches Per Runtime Nanosecond'] = (\n", - " df_snapshot_diffs['Num Round Trip Touches Outlet']\n", - ") / df_snapshot_diffs['Row Final Timepoint (ns) Outlet']\n", - "\n", - "df_snapshot_diffs['Latency Simsteps Inlet'] = df_snapshot_diffs['Num Puts Attempted'] / df_snapshot_diffs['Num Round Trip Touches Inlet']\n", - "df_snapshot_diffs['Latency Simsteps Outlet'] = df_snapshot_diffs['Num Pulls Attempted'] / df_snapshot_diffs['Num Round Trip Touches Outlet']\n", - "df_snapshot_diffs['Delivery Clumpiness'] = 1.0 - df_snapshot_diffs['Num Pulls That Were Laden Immediately'] / df_snapshot_diffs[['Net Flux Through Duct', 'Num Pulls Attempted']].min(axis=1)\n", - "df_snapshot_diffs['Intermittancy'] = df_snapshot_diffs['Delivery Clumpiness']\n", - "df_snapshot_diffs['Inlet-Nanoseconds Elapsed'] = df_snapshot_diffs['Num Inlets'] * df_snapshot_diffs['Row Final Timepoint (ns) Inlet']\n", - "df_snapshot_diffs['Outlet-Nanoseconds Elapsed'] = df_snapshot_diffs['Num Outlets'] * df_snapshot_diffs['Row Final Timepoint (ns) Outlet']\n", - "df_snapshot_diffs['Simsteps Elapsed Inlet'] = df_snapshot_diffs['Num Puts Attempted'] / df_snapshot_diffs['Num Inlets']\n", - "df_snapshot_diffs['Simsteps Elapsed Outlet'] = df_snapshot_diffs['Num Pulls Attempted'] / df_snapshot_diffs['Num Outlets']\n", - "df_snapshot_diffs['Simstep Period Inlet (ns)'] = df_snapshot_diffs['Inlet-Nanoseconds Elapsed'] / df_snapshot_diffs['Num Puts Attempted']\n", - "df_snapshot_diffs['Simstep Period Outlet (ns)'] = df_snapshot_diffs['Outlet-Nanoseconds Elapsed'] / df_snapshot_diffs['Num Pulls Attempted']\n", - "df_snapshot_diffs['Latency Walltime Inlet (ns)'] = df_snapshot_diffs['Latency Simsteps Inlet'] * df_snapshot_diffs['Simstep Period Inlet (ns)']\n", - "df_snapshot_diffs['Latency Walltime Outlet (ns)'] = df_snapshot_diffs['Latency Simsteps Outlet'] * df_snapshot_diffs['Simstep Period Outlet (ns)']\n" + "allocation_idx_mapped_title = find_treat_idx_mapped_col(df_snapshot_diffs)\n", + "allocation_idx_mapped_title\n" ] }, { @@ -394,22 +87,7 @@ "metadata": {}, "outputs": [], "source": [ - "df_snapshot_diffs = df_snapshot_diffs.astype({\n", - " 'Num Inlets' : 'int64',\n", - " 'Num Outlets' : 'int64',\n", - " 'proc' : 'int64',\n", - " 'Snapshot' : 'int64',\n", - " 'Replicate' : 'int64',\n", - " 'Async Mode' : 'int64',\n", - " 'Num Threads' : 'int64',\n", - " 'Num Processes' : 'int64',\n", - " 'Allocated Tasks Per Node' : 'int64',\n", - " 'Cpus Per Node' : 'int64',\n", - " 'Num Nodes' : 'int64',\n", - " 'Num Simels Per Cpu' : 'int64',\n", - " allocation_idx_mapped_title : 'int64',\n", - "\n", - "})\n" + "df_world_sum = wrangle_world_sums(df_finalized_observations)\n" ] }, { @@ -427,20 +105,7 @@ "metadata": {}, "outputs": [], "source": [ - "def facet_boxplot(*, data, col=None, row=None, x, y, showfliers=False):\n", - " g = sns.FacetGrid(\n", - " data,\n", - " col=col if col is not None and data[col].nunique() > 1 else None,\n", - " row=row if row is not None and data[row].nunique() > 1 else None,\n", - " margin_titles=True,\n", - " sharey='row',\n", - " )\n", - " g.map_dataframe(\n", - " sns.boxplot,\n", - " x,\n", - " y,\n", - " showfliers=showfliers,\n", - " )\n" + "from conduitpylib.viz import facet_boxplot\n" ] }, { @@ -671,174 +336,995 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from conduitpylib.viz import (\n", + " facet_barplot,\n", + " facet_boxplot_withfliers,\n", + " facet_boxplot_nofliers,\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Latency Walltime\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for viz in facet_barplot, facet_boxplot_withfliers, facet_boxplot_nofliers:\n", + " tp.tee(\n", + " viz,\n", + " data=df_snapshot_diffs,\n", + " row='Num Simels Per Cpu',\n", + " x=allocation_idx_mapped_title,\n", + " y='Latency Walltime Inlet (ns)',\n", + " teeplot_outattrs={\n", + " **{\n", + " 'transform' : 'snapshot_diffs',\n", + " },\n", + " **nbm.collate_outattr_metadata(),\n", + " },\n", + " teeplot_subdir='latency-walltime-inlet-ns',\n", + " )\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Simple Mean and Median\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# adapted from https://stackoverflow.com/a/13592901\n", + "df_snapshot_diffs.groupby([\n", + " allocation_idx_mapped_title,\n", + "]).agg({\n", + " 'Latency Walltime Inlet (ns)' : [\n", + " np.mean,\n", + " np.median,\n", + " ],\n", + "})\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Median of Replicate Means\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "group_medians = df_snapshot_diffs.groupby([allocation_idx_mapped_title, 'Replicate']).agg({\n", + " 'Latency Walltime Inlet (ns)': np.mean\n", + "}).reset_index()\n", + "\n", + "median_of_medians = group_medians.groupby(allocation_idx_mapped_title).agg({\n", + " 'Latency Walltime Inlet (ns)': np.mean\n", + "})\n", + "\n", + "baseline = median_of_medians.loc[1, \"Latency Walltime Inlet (ns)\"].squeeze()\n", + "median_of_medians[\"normed delta\"] = (median_of_medians[\"Latency Walltime Inlet (ns)\"] - baseline) / baseline * 100\n", + "median_of_medians\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Median of Replicate Medians\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "group_medians = df_snapshot_diffs.groupby([allocation_idx_mapped_title, 'Replicate']).agg({\n", + " 'Latency Walltime Inlet (ns)': np.median\n", + "}).reset_index()\n", + "\n", + "median_of_medians = group_medians.groupby(allocation_idx_mapped_title).agg({\n", + " 'Latency Walltime Inlet (ns)': np.median\n", + "})\n", + "\n", + "baseline = median_of_medians.loc[1, \"Latency Walltime Inlet (ns)\"].squeeze()\n", + "median_of_medians[\"normed delta\"] = (median_of_medians[\"Latency Walltime Inlet (ns)\"] - baseline) / baseline * 100\n", + "median_of_medians\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Median Absolute Deviance\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "xdf = df_snapshot_diffs.copy()\n", + "xdf['median'] = xdf['Latency Walltime Inlet (ns)']\n", + "xdf['median_abs_deviation'] = xdf['Latency Walltime Inlet (ns)']\n", + "xdf = xdf.groupby([allocation_idx_mapped_title, 'Replicate']).agg({\n", + " 'median': np.median,\n", + " 'median_abs_deviation': stats.median_abs_deviation,\n", + "}).reset_index()\n", + "xdf[\"normed median_abs_deviation\"] = xdf[\"median_abs_deviation\"] / xdf[\"median\"] * 100\n", + "\n", + "name1, name2 = xdf[allocation_idx_mapped_title].unique()\n", + "\n", + "# Extract medians for each group\n", + "group1 = xdf[xdf[allocation_idx_mapped_title] == name1]['normed median_abs_deviation']\n", + "group2 = xdf[xdf[allocation_idx_mapped_title] == name2]['normed median_abs_deviation']\n", + "\n", + "print(len(group1), len(group2))\n", + "stats.mannwhitneyu(group1, group2)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ax = sns.histplot(data=xdf, x=\"normed median_abs_deviation\", hue=allocation_idx_mapped_title)\n", + "\n", + "xdf.groupby(allocation_idx_mapped_title)['normed median_abs_deviation'].median().reset_index()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Percent Outliers\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nonoutlier_counts = df_snapshot_diffs.groupby(allocation_idx_mapped_title).agg({\n", + " 'Latency Walltime Inlet (ns)': count_nonoutliers,\n", + "}).reset_index()\n", + "nonoutlier_counts\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "outlier_counts = df_snapshot_diffs.groupby(allocation_idx_mapped_title).agg({\n", + " 'Latency Walltime Inlet (ns)': count_outliers,\n", + "}).reset_index()\n", + "outlier_counts\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "stats.chi2_contingency(\n", + " np.vstack(\n", + " [\n", + " outlier_counts[\"Latency Walltime Inlet (ns)\"],\n", + " nonoutlier_counts[\"Latency Walltime Inlet (ns)\"],\n", + " ],\n", + " ),\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_snapshot_diffs.groupby(allocation_idx_mapped_title).agg({\n", + " 'Latency Walltime Inlet (ns)': lambda x: count_proportion_outliers(x) * 100,\n", + "}).reset_index()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Distributions\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for viz in facet_barplot, facet_boxplot_withfliers, facet_boxplot_nofliers:\n", + " tp.tee(\n", + " viz,\n", + " data=df_snapshot_diffs,\n", + " row='Num Simels Per Cpu',\n", + " x=allocation_idx_mapped_title,\n", + " y='Latency Walltime Outlet (ns)',\n", + " teeplot_outattrs={\n", + " **{\n", + " 'transform' : 'snapshot_diffs',\n", + " },\n", + " **nbm.collate_outattr_metadata(),\n", + " },\n", + " teeplot_subdir='latency-walltime-outlet-ns',\n", + " )\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# adapted from https://stackoverflow.com/a/13592901\n", + "df_snapshot_diffs.groupby([\n", + " allocation_idx_mapped_title,\n", + "]).agg({\n", + " 'Latency Walltime Outlet (ns)' : [\n", + " np.mean,\n", + " np.median,\n", + " ],\n", + "})\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Latency Simsteps\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for viz in facet_barplot, facet_boxplot_withfliers, facet_boxplot_nofliers:\n", + " tp.tee(\n", + " viz,\n", + " data=df_snapshot_diffs,\n", + " row='Num Simels Per Cpu',\n", + " x=allocation_idx_mapped_title,\n", + " y='Latency Simsteps Inlet',\n", + " teeplot_outattrs={\n", + " **{\n", + " 'transform' : 'snapshot_diffs',\n", + " },\n", + " **nbm.collate_outattr_metadata(),\n", + " },\n", + " teeplot_subdir='latency-simsteps-inlet',\n", + " )\n" + ] + }, + { + "cell_type": "markdown", + "id": "3fdb3f6f-4c83-4804-a1e7-b773554af98d", + "metadata": {}, + "source": [ + "### Simple Mean and Median\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "05d5be0f-86c7-4104-b318-55caaba2f1ff", + "metadata": {}, + "outputs": [], + "source": [ + "# adapted from https://stackoverflow.com/a/13592901\n", + "df_snapshot_diffs.groupby([\n", + " allocation_idx_mapped_title,\n", + "]).agg({\n", + " 'Latency Simsteps Inlet' : [\n", + " np.mean,\n", + " np.median,\n", + " ],\n", + "})\n" + ] + }, + { + "cell_type": "markdown", + "id": "f7f5d8c3-5acb-456a-b0b3-d944248f6809", + "metadata": {}, + "source": [ + "### Median of Replicate Means\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8cfa1a75-b008-4d25-ae26-cd018aee0a44", + "metadata": {}, + "outputs": [], + "source": [ + "group_medians = df_snapshot_diffs.groupby([allocation_idx_mapped_title, 'Replicate']).agg({\n", + " 'Latency Simsteps Inlet': np.mean\n", + "}).reset_index()\n", + "\n", + "median_of_medians = group_medians.groupby(allocation_idx_mapped_title).agg({\n", + " 'Latency Simsteps Inlet': np.mean\n", + "})\n", + "\n", + "baseline = median_of_medians.loc[1, \"Latency Simsteps Inlet\"].squeeze()\n", + "median_of_medians[\"normed delta\"] = (median_of_medians[\"Latency Simsteps Inlet\"] - baseline) / baseline * 100\n", + "median_of_medians\n" + ] + }, + { + "cell_type": "markdown", + "id": "28d666eb-4ea7-4112-ab4c-fd05c992ad03", + "metadata": {}, + "source": [ + "### Median of Replicate Medians\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02351780-005a-496c-bde5-49a0400f6303", + "metadata": {}, + "outputs": [], + "source": [ + "group_medians = df_snapshot_diffs.groupby([allocation_idx_mapped_title, 'Replicate']).agg({\n", + " 'Latency Simsteps Inlet': np.median\n", + "}).reset_index()\n", + "\n", + "median_of_medians = group_medians.groupby(allocation_idx_mapped_title).agg({\n", + " 'Latency Simsteps Inlet': np.median\n", + "})\n", + "\n", + "baseline = median_of_medians.loc[1, \"Latency Simsteps Inlet\"].squeeze()\n", + "median_of_medians[\"normed delta\"] = (median_of_medians[\"Latency Simsteps Inlet\"] - baseline) / baseline * 100\n", + "median_of_medians\n" + ] + }, + { + "cell_type": "markdown", + "id": "6467fb06-d857-40b3-bdc4-2858ba94a2df", + "metadata": {}, + "source": [ + "### Median Absolute Deviance\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d782ed5-420a-48f4-bb57-d4e8ee8e530c", + "metadata": {}, + "outputs": [], + "source": [ + "xdf = df_snapshot_diffs.copy()\n", + "xdf['median'] = xdf['Latency Simsteps Inlet']\n", + "xdf['median_abs_deviation'] = xdf['Latency Simsteps Inlet']\n", + "xdf = xdf.groupby([allocation_idx_mapped_title, 'Replicate']).agg({\n", + " 'median': np.median,\n", + " 'median_abs_deviation': stats.median_abs_deviation,\n", + "}).reset_index()\n", + "xdf[\"normed median_abs_deviation\"] = xdf[\"median_abs_deviation\"] / xdf[\"median\"] * 100\n", + "\n", + "name1, name2 = xdf[allocation_idx_mapped_title].unique()\n", + "\n", + "# Extract medians for each group\n", + "group1 = xdf[xdf[allocation_idx_mapped_title] == name1]['normed median_abs_deviation']\n", + "group2 = xdf[xdf[allocation_idx_mapped_title] == name2]['normed median_abs_deviation']\n", + "\n", + "print(len(group1), len(group2))\n", + "stats.mannwhitneyu(group1, group2)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7fd12be-ab9f-4c97-8d96-8c12836460c0", + "metadata": {}, + "outputs": [], + "source": [ + "ax = sns.histplot(data=xdf, x=\"normed median_abs_deviation\", hue=allocation_idx_mapped_title)\n", + "\n", + "xdf.groupby(allocation_idx_mapped_title)['normed median_abs_deviation'].median().reset_index()\n" + ] + }, + { + "cell_type": "markdown", + "id": "bc6e4cf7-6589-4243-8140-29165c4be39d", + "metadata": {}, + "source": [ + "### Percent Outliers\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4413cca-12b7-4552-9b91-cbf5aada8446", + "metadata": {}, + "outputs": [], + "source": [ + "nonoutlier_counts = df_snapshot_diffs.groupby(allocation_idx_mapped_title).agg({\n", + " 'Latency Simsteps Inlet': count_nonoutliers,\n", + "}).reset_index()\n", + "nonoutlier_counts\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a49edb75-c9eb-4a92-981f-ce75b86516c5", + "metadata": {}, + "outputs": [], + "source": [ + "outlier_counts = df_snapshot_diffs.groupby(allocation_idx_mapped_title).agg({\n", + " 'Latency Simsteps Inlet': count_outliers,\n", + "}).reset_index()\n", + "outlier_counts\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6f606740-6046-44ab-a421-9278c719bdbb", + "metadata": {}, + "outputs": [], + "source": [ + "stats.chi2_contingency(\n", + " np.vstack(\n", + " [\n", + " outlier_counts[\"Latency Simsteps Inlet\"],\n", + " nonoutlier_counts[\"Latency Simsteps Inlet\"],\n", + " ],\n", + " ),\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14469ba8-9d0e-482e-b309-18dbca4a2201", + "metadata": {}, + "outputs": [], + "source": [ + "df_snapshot_diffs.groupby(allocation_idx_mapped_title).agg({\n", + " 'Latency Simsteps Inlet': lambda x: count_proportion_outliers(x) * 100,\n", + "}).reset_index()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for viz in facet_barplot, facet_boxplot_withfliers, facet_boxplot_nofliers:\n", + " tp.tee(\n", + " viz,\n", + " data=df_snapshot_diffs,\n", + " row='Num Simels Per Cpu',\n", + " x=allocation_idx_mapped_title,\n", + " y='Latency Simsteps Outlet',\n", + " teeplot_outattrs={\n", + " **{\n", + " 'transform' : 'snapshot_diffs',\n", + " },\n", + " **nbm.collate_outattr_metadata(),\n", + " },\n", + " teeplot_subdir='latency-simsteps-outlet',\n", + " )\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# adapted fmke_regr_make_regression_rowrom https://stackoverflow.com/a/13592901\n", + "df_snapshot_diffs.groupby(\n", + " [\n", + " allocation_idx_mapped_title,\n", + " ]\n", + ").agg(\n", + " {\n", + " \"Latency Simsteps Outlet\": [\n", + " np.mean,\n", + " np.median,\n", + " ],\n", + " }\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Delivery Failure Rate\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for viz in facet_barplot, facet_boxplot_withfliers, facet_boxplot_nofliers:\n", + " tp.tee(\n", + " viz,\n", + " data=df_snapshot_diffs,\n", + " row='Num Simels Per Cpu',\n", + " x=allocation_idx_mapped_title,\n", + " y='Delivery Failure Rate',\n", + " teeplot_outattrs={\n", + " **{\n", + " 'transform' : 'snapshot_diffs',\n", + " },\n", + " **nbm.collate_outattr_metadata(),\n", + " },\n", + " teeplot_subdir='delivery-failure-rate',\n", + " )\n" + ] + }, + { + "cell_type": "markdown", + "id": "583d05ae-9ff3-457b-a9dd-0d6c787f7142", + "metadata": {}, + "source": [ + "### Simple Mean and Median\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "904d7985-4a3d-4038-a108-8707eed6a1fb", + "metadata": {}, + "outputs": [], + "source": [ + "# adapted from https://stackoverflow.com/a/13592901\n", + "df_snapshot_diffs.groupby([\n", + " allocation_idx_mapped_title,\n", + "]).agg({\n", + " 'Delivery Failure Rate' : [\n", + " np.mean,\n", + " np.median,\n", + " ],\n", + "})\n" + ] + }, + { + "cell_type": "markdown", + "id": "afae0790-82f6-401e-af06-d59938cd7b44", + "metadata": {}, + "source": [ + "### Median of Replicate Means\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "adbccb85-e96a-4dfc-8cd9-14205c572e96", + "metadata": {}, + "outputs": [], + "source": [ + "group_medians = df_snapshot_diffs.groupby([allocation_idx_mapped_title, 'Replicate']).agg({\n", + " 'Delivery Failure Rate': np.mean\n", + "}).reset_index()\n", + "\n", + "median_of_medians = group_medians.groupby(allocation_idx_mapped_title).agg({\n", + " 'Delivery Failure Rate': np.mean\n", + "})\n", + "\n", + "baseline = median_of_medians.loc[1, \"Delivery Failure Rate\"].squeeze()\n", + "median_of_medians[\"normed delta\"] = (median_of_medians[\"Delivery Failure Rate\"] - baseline) / baseline * 100\n", + "median_of_medians\n" + ] + }, + { + "cell_type": "markdown", + "id": "ba4c19ee-ec1c-4b99-9342-8ab53dfc54a0", + "metadata": {}, + "source": [ + "### Median of Replicate Medians\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "68664a4a-5cfa-492e-a736-913d4aade9c6", + "metadata": {}, + "outputs": [], + "source": [ + "group_medians = df_snapshot_diffs.groupby([allocation_idx_mapped_title, 'Replicate']).agg({\n", + " 'Delivery Failure Rate': np.median\n", + "}).reset_index()\n", + "\n", + "median_of_medians = group_medians.groupby(allocation_idx_mapped_title).agg({\n", + " 'Delivery Failure Rate': np.median\n", + "})\n", + "\n", + "baseline = median_of_medians.loc[1, \"Delivery Failure Rate\"].squeeze()\n", + "median_of_medians[\"normed delta\"] = (median_of_medians[\"Delivery Failure Rate\"] - baseline) / baseline * 100\n", + "median_of_medians\n" + ] + }, + { + "cell_type": "markdown", + "id": "3fbc9ca0-2b86-4d78-93f7-6b82a1501aae", + "metadata": {}, + "source": [ + "### Median Absolute Deviance\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "541f4ac8-c220-4cea-863b-bd787361236a", + "metadata": {}, + "outputs": [], + "source": [ + "xdf = df_snapshot_diffs.copy()\n", + "xdf['median'] = xdf['Delivery Failure Rate']\n", + "xdf['median_abs_deviation'] = xdf['Delivery Failure Rate']\n", + "xdf = xdf.groupby([allocation_idx_mapped_title, 'Replicate']).agg({\n", + " 'median': np.median,\n", + " 'median_abs_deviation': stats.median_abs_deviation,\n", + "}).reset_index()\n", + "xdf[\"normed median_abs_deviation\"] = xdf[\"median_abs_deviation\"] / xdf[\"median\"] * 100\n", + "\n", + "name1, name2 = xdf[allocation_idx_mapped_title].unique()\n", + "\n", + "# Extract medians for each group\n", + "group1 = xdf[xdf[allocation_idx_mapped_title] == name1]['normed median_abs_deviation']\n", + "group2 = xdf[xdf[allocation_idx_mapped_title] == name2]['normed median_abs_deviation']\n", + "\n", + "try:\n", + " print(len(group1), len(group2))\n", + " print(group1.isna().all(), group2.isna().all())\n", + " res = stats.mannwhitneyu(group1, group2)\n", + " display(res)\n", + "except Exception as e:\n", + " warnings.warn(f\"{type(e).__name__}: {e}\", RuntimeWarning)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5fbf214-a609-46fa-a98c-541714ed88bb", + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " ax = sns.histplot(data=xdf, x=\"normed median_abs_deviation\", hue=allocation_idx_mapped_title)\n", + "except Exception as e:\n", + " warnings.warn(f\"{type(e).__name__}: {e}\", RuntimeWarning)\n", + "\n", + "try:\n", + " xdf.groupby(allocation_idx_mapped_title)['normed median_abs_deviation'].median().reset_index()\n", + "except Exception as e:\n", + " warnings.warn(f\"{type(e).__name__}: {e}\", RuntimeWarning)\n" + ] + }, + { + "cell_type": "markdown", + "id": "57e88ca0-24c6-4e4e-a336-7820e326c681", + "metadata": {}, + "source": [ + "### Percent Outliers\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e193a7ab-b264-43b4-bec0-85dc1647f247", + "metadata": {}, + "outputs": [], + "source": [ + "nonoutlier_counts = df_snapshot_diffs.groupby(allocation_idx_mapped_title).agg({\n", + " 'Delivery Failure Rate': count_nonoutliers,\n", + "}).reset_index()\n", + "nonoutlier_counts\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3af8cf0f-ba65-4291-b9d6-90abd0ec3e60", + "metadata": {}, + "outputs": [], + "source": [ + "outlier_counts = df_snapshot_diffs.groupby(allocation_idx_mapped_title).agg({\n", + " 'Delivery Failure Rate': count_outliers,\n", + "}).reset_index()\n", + "outlier_counts\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9af1ced2-08e5-4a4c-94f2-3cd9ccc15241", + "metadata": {}, + "outputs": [], + "source": [ + "stats.chi2_contingency(\n", + " np.vstack(\n", + " [\n", + " outlier_counts[\"Delivery Failure Rate\"],\n", + " nonoutlier_counts[\"Delivery Failure Rate\"],\n", + " ],\n", + " ),\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "92e27cdd-acef-4753-b395-48026fde9803", + "metadata": {}, + "outputs": [], + "source": [ + "df_snapshot_diffs.groupby(allocation_idx_mapped_title).agg({\n", + " 'Delivery Failure Rate': lambda x: count_proportion_outliers(x) * 100,\n", + "}).reset_index()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Delivery Clumpiness\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for viz in facet_barplot, facet_boxplot_withfliers, facet_boxplot_nofliers:\n", + " tp.tee(\n", + " viz,\n", + " data=df_snapshot_diffs,\n", + " row='Num Simels Per Cpu',\n", + " x=allocation_idx_mapped_title,\n", + " y='Delivery Clumpiness',\n", + " teeplot_outattrs={\n", + " **{\n", + " 'transform' : 'snapshot_diffs',\n", + " },\n", + " **nbm.collate_outattr_metadata(),\n", + " },\n", + " teeplot_subdir='delivery-clumpiness',\n", + " )\n" + ] + }, + { + "cell_type": "markdown", + "id": "fa2a7b00-69ed-4ad0-a0a9-75c81905f7b4", + "metadata": {}, + "source": [ + "### Simple Mean and Median\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dda86c6b-e390-4c28-b5c4-491377986542", + "metadata": {}, + "outputs": [], + "source": [ + "# adapted from https://stackoverflow.com/a/13592901\n", + "df_snapshot_diffs.groupby([\n", + " allocation_idx_mapped_title,\n", + "]).agg({\n", + " 'Delivery Clumpiness' : [\n", + " np.mean,\n", + " np.median,\n", + " ],\n", + "})\n" + ] + }, + { + "cell_type": "markdown", + "id": "f3c96c69-348b-457c-86a6-e58bd196f29b", + "metadata": {}, + "source": [ + "### Median of Replicate Means\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "98644ac1-cd06-4948-96aa-99578d2d9b2e", + "metadata": {}, + "outputs": [], + "source": [ + "group_medians = df_snapshot_diffs.groupby([allocation_idx_mapped_title, 'Replicate']).agg({\n", + " 'Delivery Clumpiness': np.mean\n", + "}).reset_index()\n", + "\n", + "median_of_medians = group_medians.groupby(allocation_idx_mapped_title).agg({\n", + " 'Delivery Clumpiness': np.mean\n", + "})\n", + "\n", + "baseline = median_of_medians.loc[1, \"Delivery Clumpiness\"].squeeze()\n", + "median_of_medians[\"normed delta\"] = (median_of_medians[\"Delivery Clumpiness\"] - baseline) / baseline * 100\n", + "median_of_medians\n" + ] + }, + { + "cell_type": "markdown", + "id": "17984f4e-ecf7-48b5-94cd-7e8f89d8ec95", + "metadata": {}, + "source": [ + "### Median of Replicate Medians\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e23c7941-5073-48c1-b39b-9294fd2a5b06", + "metadata": {}, + "outputs": [], + "source": [ + "group_medians = df_snapshot_diffs.groupby([allocation_idx_mapped_title, 'Replicate']).agg({\n", + " 'Delivery Clumpiness': np.median\n", + "}).reset_index()\n", + "\n", + "median_of_medians = group_medians.groupby(allocation_idx_mapped_title).agg({\n", + " 'Delivery Clumpiness': np.median\n", + "})\n", + "\n", + "baseline = median_of_medians.loc[1, \"Delivery Clumpiness\"].squeeze()\n", + "median_of_medians[\"normed delta\"] = (median_of_medians[\"Delivery Clumpiness\"] - baseline) / baseline * 100\n", + "median_of_medians\n" + ] + }, + { + "cell_type": "markdown", + "id": "dfcd7358-58c9-4029-942c-53688bf9261e", "metadata": {}, - "outputs": [], "source": [ - "def facet_barplot(*, data, col=None, row=None, x, y, hue=None):\n", - " g = sns.FacetGrid(\n", - " data,\n", - " col=col if col is not None and data[col].nunique() > 1 else None,\n", - " row=row if row is not None and data[row].nunique() > 1 else None,\n", - " margin_titles=True,\n", - " sharey='row',\n", - " )\n", - " g.map_dataframe(\n", - " sns.barplot,\n", - " x=x,\n", - " y=y,\n", - " hue=hue,\n", - " )\n", - "\n", - " # adapted from https://stackoverflow.com/a/48208266\n", - " g.set_axis_labels(x_var=x, y_var=y)\n" + "### Median Absolute Deviance\n" ] }, { "cell_type": "code", "execution_count": null, + "id": "cef9e496-f089-4fe7-ac34-5fdd383aa74d", "metadata": {}, "outputs": [], "source": [ - "def facet_boxplot_withfliers(*, data, col=None, row=None, x, y, hue=None):\n", - " g = sns.FacetGrid(\n", - " data,\n", - " col=col if col is not None and data[col].nunique() > 1 else None,\n", - " row=row if row is not None and data[row].nunique() > 1 else None,\n", - " margin_titles=True,\n", - " sharey='row',\n", - " )\n", - " g.map_dataframe(\n", - " sns.boxplot,\n", - " x=x,\n", - " y=y,\n", - " hue=hue,\n", - " showfliers=True,\n", - " )\n", + "xdf = df_snapshot_diffs.copy()\n", + "xdf['median'] = xdf['Delivery Clumpiness']\n", + "xdf['median_abs_deviation'] = xdf['Delivery Clumpiness']\n", + "xdf = xdf.groupby([allocation_idx_mapped_title, 'Replicate']).agg({\n", + " 'median': np.median,\n", + " 'median_abs_deviation': stats.median_abs_deviation,\n", + "}).reset_index()\n", + "xdf[\"normed median_abs_deviation\"] = xdf[\"median_abs_deviation\"] / xdf[\"median\"] * 100\n", + "\n", + "name1, name2 = xdf[allocation_idx_mapped_title].unique()\n", "\n", - " # adapted from https://stackoverflow.com/a/48208266\n", - " g.set_axis_labels(x_var=x, y_var=y)\n" + "# Extract medians for each group\n", + "group1 = xdf[xdf[allocation_idx_mapped_title] == name1]['normed median_abs_deviation']\n", + "group2 = xdf[xdf[allocation_idx_mapped_title] == name2]['normed median_abs_deviation']\n", + "\n", + "print(len(group1), len(group2))\n", + "stats.mannwhitneyu(group1, group2)\n" ] }, { "cell_type": "code", "execution_count": null, + "id": "8a3bbba1-60bd-4422-a2d9-19c22e9c6cd1", "metadata": {}, "outputs": [], "source": [ - "def facet_boxplot_nofliers(*, data, col=None, row=None, x, y, hue=None):\n", - " g = sns.FacetGrid(\n", - " data,\n", - " col=col if col is not None and data[col].nunique() > 1 else None,\n", - " row=row if row is not None and data[row].nunique() > 1 else None,\n", - " margin_titles=True,\n", - " sharey='row',\n", - " )\n", - " g.map_dataframe(\n", - " sns.boxplot,\n", - " x=x,\n", - " y=y,\n", - " hue=hue,\n", - " showfliers=False,\n", - " )\n", + "ax = sns.histplot(data=xdf, x=\"normed median_abs_deviation\", hue=allocation_idx_mapped_title)\n", "\n", - " # adapted from https://stackoverflow.com/a/48208266\n", - " g.set_axis_labels(x_var=x, y_var=y)\n" + "xdf.groupby(allocation_idx_mapped_title)['normed median_abs_deviation'].median().reset_index()\n" ] }, { "cell_type": "markdown", + "id": "493085c4-5764-4221-9a79-176965927ac9", "metadata": {}, "source": [ - "## Latency Walltime\n" + "### Percent Outliers\n" ] }, { "cell_type": "code", "execution_count": null, + "id": "ff5ab4d2-ee9c-48a3-b7f3-c36fd6859412", "metadata": {}, "outputs": [], "source": [ - "for viz in facet_barplot, facet_boxplot_withfliers, facet_boxplot_nofliers:\n", - " tp.tee(\n", - " viz,\n", - " data=df_snapshot_diffs,\n", - " row='Num Simels Per Cpu',\n", - " x=allocation_idx_mapped_title,\n", - " y='Latency Walltime Inlet (ns)',\n", - " teeplot_outattrs={\n", - " **{\n", - " 'transform' : 'snapshot_diffs',\n", - " },\n", - " **nbm.collate_outattr_metadata(),\n", - " },\n", - " teeplot_subdir='latency-walltime-inlet-ns',\n", - " )\n" + "nonoutlier_counts = df_snapshot_diffs.groupby(allocation_idx_mapped_title).agg({\n", + " 'Delivery Clumpiness': count_nonoutliers,\n", + "}).reset_index()\n", + "nonoutlier_counts\n" ] }, { "cell_type": "code", "execution_count": null, + "id": "0621e3c5-da83-4ee2-9196-07bbb513b58b", "metadata": {}, "outputs": [], "source": [ - "# adapted from https://stackoverflow.com/a/13592901\n", - "df_snapshot_diffs.groupby([\n", - " allocation_idx_mapped_title,\n", - "]).agg({\n", - " 'Latency Walltime Inlet (ns)' : [\n", - " np.mean,\n", - " np.median,\n", - " ],\n", - "})\n" + "outlier_counts = df_snapshot_diffs.groupby(allocation_idx_mapped_title).agg({\n", + " 'Delivery Clumpiness': count_outliers,\n", + "}).reset_index()\n", + "outlier_counts\n" ] }, { "cell_type": "code", "execution_count": null, + "id": "9571015e-40ae-45ca-ad3a-3e2af18d2793", "metadata": {}, "outputs": [], "source": [ - "for viz in facet_barplot, facet_boxplot_withfliers, facet_boxplot_nofliers:\n", - " tp.tee(\n", - " viz,\n", - " data=df_snapshot_diffs,\n", - " row='Num Simels Per Cpu',\n", - " x=allocation_idx_mapped_title,\n", - " y='Latency Walltime Outlet (ns)',\n", - " teeplot_outattrs={\n", - " **{\n", - " 'transform' : 'snapshot_diffs',\n", - " },\n", - " **nbm.collate_outattr_metadata(),\n", - " },\n", - " teeplot_subdir='latency-walltime-outlet-ns',\n", - " )\n" + "stats.chi2_contingency(\n", + " np.vstack(\n", + " [\n", + " outlier_counts[\"Delivery Clumpiness\"],\n", + " nonoutlier_counts[\"Delivery Clumpiness\"],\n", + " ],\n", + " ),\n", + ")\n" ] }, { "cell_type": "code", "execution_count": null, + "id": "e35fa28f-cc60-4c0c-969e-09a89203502a", "metadata": {}, "outputs": [], "source": [ - "# adapted from https://stackoverflow.com/a/13592901\n", - "df_snapshot_diffs.groupby([\n", - " allocation_idx_mapped_title,\n", - "]).agg({\n", - " 'Latency Walltime Outlet (ns)' : [\n", - " np.mean,\n", - " np.median,\n", - " ],\n", - "})\n" + "df_snapshot_diffs.groupby(allocation_idx_mapped_title).agg({\n", + " 'Delivery Clumpiness': lambda x: count_proportion_outliers(x) * 100,\n", + "}).reset_index()\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Latency Simsteps\n" + "## Simstep Period\n" ] }, { @@ -853,20 +1339,29 @@ " data=df_snapshot_diffs,\n", " row='Num Simels Per Cpu',\n", " x=allocation_idx_mapped_title,\n", - " y='Latency Simsteps Inlet',\n", + " y='Simstep Period Inlet (ns)',\n", " teeplot_outattrs={\n", " **{\n", " 'transform' : 'snapshot_diffs',\n", " },\n", " **nbm.collate_outattr_metadata(),\n", " },\n", - " teeplot_subdir='latency-simsteps-inlet',\n", + " teeplot_subdir='simstep-period-inlet-ns',\n", " )\n" ] }, + { + "cell_type": "markdown", + "id": "80e56a70-c0c0-44c3-a1ba-ae587bbcd9c2", + "metadata": {}, + "source": [ + "### Simple Mean and Median\n" + ] + }, { "cell_type": "code", "execution_count": null, + "id": "c8416f0f-1577-451a-a2e8-038a6a9ae35a", "metadata": {}, "outputs": [], "source": [ @@ -874,192 +1369,176 @@ "df_snapshot_diffs.groupby([\n", " allocation_idx_mapped_title,\n", "]).agg({\n", - " 'Latency Simsteps Inlet' : [\n", + " 'Simstep Period Inlet (ns)' : [\n", " np.mean,\n", " np.median,\n", " ],\n", "})\n" ] }, + { + "cell_type": "markdown", + "id": "2bc12349-f485-4414-a722-2fb770dddc62", + "metadata": {}, + "source": [ + "### Median of Replicate Means\n" + ] + }, { "cell_type": "code", "execution_count": null, + "id": "2a3f4265-ab45-49fc-8257-c952e6f81019", "metadata": {}, "outputs": [], "source": [ - "for viz in facet_barplot, facet_boxplot_withfliers, facet_boxplot_nofliers:\n", - " tp.tee(\n", - " viz,\n", - " data=df_snapshot_diffs,\n", - " row='Num Simels Per Cpu',\n", - " x=allocation_idx_mapped_title,\n", - " y='Latency Simsteps Outlet',\n", - " teeplot_outattrs={\n", - " **{\n", - " 'transform' : 'snapshot_diffs',\n", - " },\n", - " **nbm.collate_outattr_metadata(),\n", - " },\n", - " teeplot_subdir='latency-simsteps-outlet',\n", - " )\n" + "group_medians = df_snapshot_diffs.groupby([allocation_idx_mapped_title, 'Replicate']).agg({\n", + " 'Simstep Period Inlet (ns)': np.mean\n", + "}).reset_index()\n", + "\n", + "median_of_medians = group_medians.groupby(allocation_idx_mapped_title).agg({\n", + " 'Simstep Period Inlet (ns)': np.mean\n", + "})\n", + "\n", + "baseline = median_of_medians.loc[1, \"Simstep Period Inlet (ns)\"].squeeze()\n", + "median_of_medians[\"normed delta\"] = (median_of_medians[\"Simstep Period Inlet (ns)\"] - baseline) / baseline * 100\n", + "median_of_medians\n" + ] + }, + { + "cell_type": "markdown", + "id": "29e918d7-dcd3-4ec5-9651-ea489683b003", + "metadata": {}, + "source": [ + "### Median of Replicate Medians\n" ] }, { "cell_type": "code", "execution_count": null, + "id": "b4527daa-195a-4f50-91f9-2cd224b86ed9", "metadata": {}, "outputs": [], "source": [ - "# adapted from https://stackoverflow.com/a/13592901\n", - "df_snapshot_diffs.groupby([\n", - " allocation_idx_mapped_title,\n", - "]).agg({\n", - " 'Latency Simsteps Outlet' : [\n", - " np.mean,\n", - " np.median,\n", - " ],\n", - "})\n" + "group_medians = df_snapshot_diffs.groupby([allocation_idx_mapped_title, 'Replicate']).agg({\n", + " 'Simstep Period Inlet (ns)': np.median\n", + "}).reset_index()\n", + "\n", + "median_of_medians = group_medians.groupby(allocation_idx_mapped_title).agg({\n", + " 'Simstep Period Inlet (ns)': np.median\n", + "})\n", + "\n", + "baseline = median_of_medians.loc[1, \"Simstep Period Inlet (ns)\"].squeeze()\n", + "median_of_medians[\"normed delta\"] = (median_of_medians[\"Simstep Period Inlet (ns)\"] - baseline) / baseline * 100\n", + "median_of_medians\n" ] }, { "cell_type": "markdown", + "id": "6ba8ccfb-a27e-40c2-94d0-273fad16b241", "metadata": {}, "source": [ - "## Delivery Failure Rate\n" + "### Median Absolute Deviance\n" ] }, { "cell_type": "code", "execution_count": null, + "id": "099ea495-33f4-44b1-b197-8e5440e4a3cb", "metadata": {}, "outputs": [], "source": [ - "for viz in facet_barplot, facet_boxplot_withfliers, facet_boxplot_nofliers:\n", - " tp.tee(\n", - " viz,\n", - " data=df_snapshot_diffs,\n", - " row='Num Simels Per Cpu',\n", - " x=allocation_idx_mapped_title,\n", - " y='Delivery Failure Rate',\n", - " teeplot_outattrs={\n", - " **{\n", - " 'transform' : 'snapshot_diffs',\n", - " },\n", - " **nbm.collate_outattr_metadata(),\n", - " },\n", - " teeplot_subdir='delivery-failure-rate',\n", - " )\n" + "xdf = df_snapshot_diffs.copy()\n", + "xdf['median'] = xdf['Simstep Period Inlet (ns)']\n", + "xdf['median_abs_deviation'] = xdf['Simstep Period Inlet (ns)']\n", + "xdf = xdf.groupby([allocation_idx_mapped_title, 'Replicate']).agg({\n", + " 'median': np.median,\n", + " 'median_abs_deviation': stats.median_abs_deviation,\n", + "}).reset_index()\n", + "xdf[\"normed median_abs_deviation\"] = xdf[\"median_abs_deviation\"] / xdf[\"median\"] * 100\n", + "\n", + "name1, name2 = xdf[allocation_idx_mapped_title].unique()\n", + "\n", + "# Extract medians for each group\n", + "group1 = xdf[xdf[allocation_idx_mapped_title] == name1]['normed median_abs_deviation']\n", + "group2 = xdf[xdf[allocation_idx_mapped_title] == name2]['normed median_abs_deviation']\n", + "\n", + "print(len(group1), len(group2))\n", + "stats.mannwhitneyu(group1, group2)\n" ] }, { "cell_type": "code", "execution_count": null, + "id": "6c8a7abf-a5d3-4099-994a-bf90f63c6a4b", "metadata": {}, "outputs": [], "source": [ - "# adapted from https://stackoverflow.com/a/13592901\n", - "df_snapshot_diffs.groupby([\n", - " allocation_idx_mapped_title,\n", - "]).agg({\n", - " 'Delivery Failure Rate' : [\n", - " np.mean,\n", - " np.median,\n", - " ],\n", - "})\n" + "ax = sns.histplot(data=xdf, x=\"normed median_abs_deviation\", hue=allocation_idx_mapped_title)\n", + "\n", + "xdf.groupby(allocation_idx_mapped_title)['normed median_abs_deviation'].median().reset_index()\n" ] }, { "cell_type": "markdown", + "id": "0068a609-4cee-4034-a6fa-a56b79072c0f", "metadata": {}, "source": [ - "## Delivery Clumpiness\n" + "### Percent Outliers\n" ] }, { "cell_type": "code", "execution_count": null, + "id": "0a6e6ec3-f417-4a85-872b-3786215fb465", "metadata": {}, "outputs": [], "source": [ - "for viz in facet_barplot, facet_boxplot_withfliers, facet_boxplot_nofliers:\n", - " tp.tee(\n", - " viz,\n", - " data=df_snapshot_diffs,\n", - " row='Num Simels Per Cpu',\n", - " x=allocation_idx_mapped_title,\n", - " y='Delivery Clumpiness',\n", - " teeplot_outattrs={\n", - " **{\n", - " 'transform' : 'snapshot_diffs',\n", - " },\n", - " **nbm.collate_outattr_metadata(),\n", - " },\n", - " teeplot_subdir='delivery-clumpiness',\n", - " )\n" + "nonoutlier_counts = df_snapshot_diffs.groupby(allocation_idx_mapped_title).agg({\n", + " 'Simstep Period Inlet (ns)': count_nonoutliers,\n", + "}).reset_index()\n", + "nonoutlier_counts\n" ] }, { "cell_type": "code", "execution_count": null, + "id": "d9557f1c-6b18-4e32-a0d8-89413a01d813", "metadata": {}, "outputs": [], "source": [ - "# adapted from https://stackoverflow.com/a/13592901\n", - "df_snapshot_diffs.groupby([\n", - " allocation_idx_mapped_title,\n", - "]).agg({\n", - " 'Delivery Clumpiness' : [\n", - " np.mean,\n", - " np.median,\n", - " ],\n", - "})\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Simstep Period\n" + "outlier_counts = df_snapshot_diffs.groupby(allocation_idx_mapped_title).agg({\n", + " 'Simstep Period Inlet (ns)': count_outliers,\n", + "}).reset_index()\n", + "outlier_counts\n" ] }, { "cell_type": "code", "execution_count": null, + "id": "378faa89-5487-4b05-a789-7f70ff86c5cb", "metadata": {}, "outputs": [], "source": [ - "for viz in facet_barplot, facet_boxplot_withfliers, facet_boxplot_nofliers:\n", - " tp.tee(\n", - " viz,\n", - " data=df_snapshot_diffs,\n", - " row='Num Simels Per Cpu',\n", - " x=allocation_idx_mapped_title,\n", - " y='Simstep Period Inlet (ns)',\n", - " teeplot_outattrs={\n", - " **{\n", - " 'transform' : 'snapshot_diffs',\n", - " },\n", - " **nbm.collate_outattr_metadata(),\n", - " },\n", - " teeplot_subdir='simstep-period-inlet-ns',\n", - " )\n" + "stats.chi2_contingency(\n", + " np.vstack(\n", + " [\n", + " outlier_counts[\"Simstep Period Inlet (ns)\"],\n", + " nonoutlier_counts[\"Simstep Period Inlet (ns)\"],\n", + " ],\n", + " ),\n", + ")\n" ] }, { "cell_type": "code", "execution_count": null, + "id": "78b50650-8d7d-4a4b-9d42-fff088abc596", "metadata": {}, "outputs": [], "source": [ - "# adapted from https://stackoverflow.com/a/13592901\n", - "df_snapshot_diffs.groupby([\n", - " allocation_idx_mapped_title,\n", - "]).agg({\n", - " 'Simstep Period Inlet (ns)' : [\n", - " np.mean,\n", - " np.median,\n", - " ],\n", - "})\n" + "df_snapshot_diffs.groupby(allocation_idx_mapped_title).agg({\n", + " 'Simstep Period Inlet (ns)': lambda x: count_proportion_outliers(x) * 100,\n", + "}).reset_index()\n" ] }, { @@ -1115,110 +1594,7 @@ "metadata": {}, "outputs": [], "source": [ - "def make_regression_row(*, data, independent_variable, dependent_variable, regression, row_filter):\n", - "\n", - " filtered_data = data[ data.apply(eval(row_filter), axis=1) ]\n", - "\n", - " regfun = {\n", - " 'Ordinary Least Squares Regression' : smf.ols,\n", - " 'Quantile Regression' : smf.quantreg,\n", - " }[regression]\n", - " model = regfun(f\"Q('{dependent_variable}') ~ Q('{independent_variable}')\", filtered_data)\n", - " fit_model = model.fit()\n", - "\n", - " slope = fit_model.params[f\"Q('{independent_variable}')\"]\n", - " intercept = fit_model.params['Intercept']\n", - "\n", - " slope_ci_lb, slope_ci_ub = fit_model.conf_int().loc[f\"Q('{independent_variable}')\"].tolist()\n", - " intercept_ci_lb, intercept_ci_ub = fit_model.conf_int().loc['Intercept'].tolist()\n", - "\n", - " p = fit_model.pvalues.loc[f\"Q('{independent_variable}')\"]\n", - "\n", - " # normalize to \"control\", i.e., lowest num processes observed\n", - " effect_size_normalization_data = data[\n", - " data[independent_variable] == data[independent_variable].min()\n", - " ][dependent_variable]\n", - " effect_size_normalization_factor = {\n", - " 'Ordinary Least Squares Regression' : lambda x: x.mean(),\n", - " 'Quantile Regression' : lambda x: x.median(),\n", - " }[regression](effect_size_normalization_data)\n", - " relative_effect_size = slope / effect_size_normalization_factor\n", - " relative_effect_size_ci_lb = slope_ci_lb / effect_size_normalization_factor\n", - " relative_effect_size_ci_ub = slope_ci_ub / effect_size_normalization_factor\n", - " relative_effect_size_ci_width = (\n", - " relative_effect_size_ci_ub\n", - " - relative_effect_size_ci_lb\n", - " )\n", - "\n", - " is_significant = p < 0.05 if np.isfinite(p) else None\n", - "\n", - " res = {\n", - " 'Independent Variable' : independent_variable,\n", - " 'Dependent Variable' : dependent_variable,\n", - " 'Dependent Variable Slug' : slugify(dependent_variable),\n", - " 'Cpus Per Node' : ib.dub( data['Cpus Per Node'] ),\n", - " 'Num Simels Per Cpu' : ip.pophomogeneous( data['Num Simels Per Cpu'] ),\n", - " 'Slope Estimate' : slope,\n", - " 'Slope Estimate 95% CI Lower Bound' : slope_ci_lb,\n", - " 'Slope Estimate 95% CI Upper Bound' : slope_ci_ub,\n", - " 'Absolute Effect Size' : slope,\n", - " 'Absolute Effect Size 95% CI Lower Bound' : slope_ci_lb,\n", - " 'Absolute Effect Size 95% CI Upper Bound' : slope_ci_ub,\n", - " 'Absolute Effect Size 95% CI Width' : slope_ci_ub - slope_ci_lb,\n", - " 'Relative Effect Size' : relative_effect_size,\n", - " 'Relative Effect Size 95% CI Lower Bound' : relative_effect_size_ci_lb,\n", - " 'Relative Effect Size 95% CI Upper Bound' : relative_effect_size_ci_ub,\n", - " 'Relative Effect Size 95% CI Width' : relative_effect_size_ci_width,\n", - " 'Intercept Estimate' : intercept,\n", - " 'Intercept Estimate 95% CI Lower Bound' : intercept_ci_lb,\n", - " 'Intercept Estimate 95% CI Upper Bound' : intercept_ci_ub,\n", - " 'R^2' : fit_model.rsquared,\n", - " 'p' : fit_model.pvalues.loc[f\"Q('{independent_variable}')\"],\n", - " 'Significant?' : is_significant,\n", - " 'Significant Effect Sign' : (\n", - " '-' if is_significant and slope < 0\n", - " else '+' if is_significant and slope > 0\n", - " else '0' if is_significant is not None\n", - " else None\n", - " ),\n", - " 'n' : len(filtered_data),\n", - " 'Filter' : row_filter,\n", - " 'Num Processes' : ib.dub(filtered_data['Num Processes']),\n", - " 'Num Processes Prettyprint' : (\n", - " '/'.join(filtered_data['Num Processes'].sort_values().astype(str).unique())\n", - " ),\n", - " 'Regression Model' : regression,\n", - " 'Regression Model Slug' : slugify(regression),\n", - " 'Statistic' : {\n", - " 'Ordinary Least Squares Regression' : 'mean',\n", - " 'Quantile Regression' : 'median',\n", - " }[regression],\n", - " }\n", - "\n", - " # dump regression summary to file\n", - " summary_filename = kn.pack({\n", - " **{\n", - " 'a' : 'regression_summary',\n", - " 'ext' : '.txt',\n", - " },\n", - " **{\n", - " slugify(k) : slugify(str(v))\n", - " for k, v in res.items()\n", - " if k in [\n", - " 'Independent Variable',\n", - " 'Dependent Variable',\n", - " 'Cpus Per Node',\n", - " 'Num Simels Per Cpu',\n", - " 'Regression Model',\n", - " ]\n", - " },\n", - " })\n", - "\n", - " pathlib.Path('outplots').mkdir(parents=True, exist_ok=True)\n", - " with open(f'outplots/{summary_filename}', 'w') as file:\n", - " print(fit_model.summary(), file=file)\n", - "\n", - " return res\n" + "from conduitpylib.wrangle import make_regression_row\n" ] }, { @@ -1268,7 +1644,7 @@ "\n", "regression_data_tuples = [\n", " (\n", - " 'Ordinary Least Squares Regression',\n", + " 'Quantile Regression over Means',\n", " df_snapshot_diffs.groupby([\n", " 'Execution Instance UUID',\n", " ]).mean().reset_index().astype({\n", @@ -1279,7 +1655,7 @@ " })\n", " ),\n", " (\n", - " 'Quantile Regression',\n", + " 'Quantile Regression over Medians',\n", " df_snapshot_diffs.groupby([\n", " 'Execution Instance UUID',\n", " ]).median().reset_index().astype({\n", @@ -1338,8 +1714,8 @@ " if k and k[0] != '_' and k != 'ext'\n", " }\n", " for source_filename in [\n", - " *df['Source File Inlet'].unique(),\n", - " *df['Source File Outlet'].unique(),\n", + " *merge_df['Source File Inlet'].unique(),\n", + " *merge_df['Source File Outlet'].unique(),\n", " ]\n", "]).dropna(\n", " axis='columns',\n", @@ -1403,70 +1779,7 @@ "metadata": {}, "outputs": [], "source": [ - "# adapted from https://stackoverflow.com/questions/30385975/seaborn-factor-plot-custom-error-bars\n", - "# and https://pandas.pydata.org/pandas-docs/stable/user_guide/visualization.html#visualization-errorbars\n", - "def errplot(x, y, err_lb, err_ub, **kwargs):\n", - " ax = plt.gca()\n", - " data = kwargs.pop('data')\n", - " yerr=np.abs(\n", - " data[[err_lb, err_ub]].to_numpy()\n", - " - data[[y, y]].to_numpy()\n", - " ).transpose()\n", - " plt.axhline(\n", - " y=0,\n", - " zorder=1,\n", - " color='black',\n", - " linewidth=2,\n", - " )\n", - " data.plot(\n", - " x=x,\n", - " y=y,\n", - " yerr=yerr,\n", - " kind='bar',\n", - " ax=ax,\n", - " zorder=3,\n", - " **kwargs,\n", - " ).grid(\n", - " axis='y',\n", - " zorder=0,\n", - " )\n", - "\n", - " if x is None:\n", - " # adapted from https://stackoverflow.com/a/12998531\n", - " plt.tick_params(\n", - " axis='x', # changes apply to the x-axis\n", - " which='both', # both major and minor ticks are affected\n", - " bottom=False, # ticks along the bottom edge are off\n", - " top=False, # ticks along the top edge are off\n", - " labelbottom=False,\n", - " )\n", - "\n", - "\n", - "def facet_errplot(*, data, x=None, y, err_lb, err_ub, estimated_statistic, col=None, row=None, size_inches=None, **kwargs):\n", - " g = sns.FacetGrid(\n", - " subset,\n", - " col=col if col is not None and data[col].nunique() > 1 else None,\n", - " row=row if row is not None and data[row].nunique() > 1 else None,\n", - " margin_titles=True,\n", - " sharey=False,\n", - " )\n", - " g.map_dataframe(\n", - " errplot,\n", - " x,\n", - " y,\n", - " err_lb,\n", - " err_ub,\n", - " **kwargs,\n", - " )\n", - "\n", - " if size_inches is not None:\n", - " plt.gcf().set_size_inches(*size_inches)\n", - "\n", - " # adapted from https://stackoverflow.com/a/29814281\n", - " plt.gcf().subplots_adjust(top=0.9)\n", - " plt.gcf().suptitle(\n", - " f\"Estimated Statistic = {estimated_statistic}\",\n", - " )\n" + "from conduitpylib.viz import errplot, facet_errplot\n" ] }, { @@ -1492,8 +1805,8 @@ " x='Dependent Variable',\n", " y='Relative Effect Size',\n", " estimated_statistic={\n", - " 'Quantile Regression' : 'Median',\n", - " 'Ordinary Least Squares Regression' : 'Mean',\n", + " 'Quantile Regression over Medians' : 'Median',\n", + " 'Quantile Regression over Means' : 'Mean',\n", " }[regression],\n", " size_inches=(8, 8),\n", " teeplot_outattrs={\n", @@ -1530,8 +1843,8 @@ " row='Num Simels Per Cpu',\n", " y='Relative Effect Size',\n", " estimated_statistic={\n", - " 'Quantile Regression' : f'{dependent_variable} Median',\n", - " 'Ordinary Least Squares Regression' : f'{dependent_variable} Mean',\n", + " 'Quantile Regression over Medians' : f'{dependent_variable} Median',\n", + " 'Quantile Regression over Means' : f'{dependent_variable} Mean',\n", " }[regression],\n", " teeplot_outattrs={\n", " **{\n", @@ -1568,8 +1881,8 @@ " row='Num Simels Per Cpu',\n", " y='Absolute Effect Size',\n", " estimated_statistic={\n", - " 'Quantile Regression' : f'{dependent_variable} Median',\n", - " 'Ordinary Least Squares Regression' : f'{dependent_variable} Mean',\n", + " 'Quantile Regression over Medians' : f'{dependent_variable} Median',\n", + " 'Quantile Regression over Means' : f'{dependent_variable} Mean',\n", " }[regression],\n", " teeplot_outattrs={\n", " **{\n", @@ -1587,111 +1900,16 @@ "metadata": {}, "outputs": [], "source": [ - "def quantile_regplot(fit_reg=True, color=None, *args, **kwargs):\n", - " x, y, data = kwargs['x'], kwargs['y'], kwargs['data']\n", - " sns.regplot(\n", - " *args,\n", - " **kwargs,\n", - " fit_reg=False,\n", - " color=color,\n", - " )\n", - "\n", - " if fit_reg:\n", - " model = smf.quantreg(\n", - " f\"Q('{y}') ~ Q('{x}')\",\n", - " data\n", - " )\n", - " res = model.fit(q=0.5)\n", - " m = res.params[f\"Q('{x}')\"]\n", - " b = res.params['Intercept']\n", - "\n", - " m_ci = res.conf_int().loc[f\"Q('{x}')\"].tolist()\n", - " b_ci = res.conf_int().loc['Intercept'].tolist()\n", - "\n", - " center_x = np.mean([data[x].min(), data[x].max()])\n", - " center_y = m * center_x + b\n", - "\n", - " xs = sorted(set(data[x]) | {center_x})\n", - " ys = [\n", - " m * x_ + b\n", - " for x_ in xs\n", - " ]\n", - " y1 = [ min(\n", - " m_ * ( x_ - center_x ) + center_y\n", - " for m_ in m_ci\n", - " ) for x_ in xs ]\n", - " y2 = [ max(\n", - " m_ * ( x_ - center_x ) + center_y\n", - " for m_ in m_ci\n", - " ) for x_ in xs ]\n", - "\n", - " plt.gca().plot(\n", - " xs,\n", - " ys,\n", - " color=color,\n", - " )\n", - " plt.gca().fill_between(\n", - " xs,\n", - " y1,\n", - " y2,\n", - " alpha=0.2,\n", - " color=color,\n", - " )\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def unsplit_regression(*args, regplot, **kwargs):\n", - " del kwargs['color']\n", - " regplot(\n", - " *args,\n", - " **kwargs,\n", - " color='black',\n", - " fit_reg=False,\n", - " )\n", - " regplot(\n", - " *args,\n", - " **kwargs,\n", - " color='purple',\n", - " scatter=False,\n", - " )\n", - "\n", - " # adapted from https://www.scivision.dev/matplotlib-force-integer-labeling-of-axis/\n", - " plt.gca().xaxis.set_major_locator(\n", - " matplotlib.ticker.MaxNLocator(\n", - " integer=True,\n", - " ),\n", - " )\n", - "\n", - "def facet_unsplit_regression(*, data, col=None, row=None, x, y, regression, **kwargs):\n", - " g = sns.FacetGrid(\n", - " data,\n", - " col=col if col is not None and data[col].nunique() > 1 else None,\n", - " row=row if row is not None and data[row].nunique() > 1 else None,\n", - " margin_titles=True,\n", - " sharey=False,\n", - " )\n", - " g.map_dataframe(\n", - " unsplit_regression,\n", - " regplot={\n", - " 'Ordinary Least Squares Regression' : sns.regplot,\n", - " 'Quantile Regression' : quantile_regplot,\n", - " }[regression],\n", - " x=x,\n", - " y=y,\n", - " **kwargs,\n", - " )\n", - "\n", - " # adapted from https://stackoverflow.com/a/48208266\n", - " g.set_axis_labels(x_var=x, y_var=y)\n", - "\n", - " # adapted from https://stackoverflow.com/a/29814281\n", - " plt.gcf().subplots_adjust(top=0.8)\n", - " plt.gcf().suptitle(regression)\n" + "from conduitpylib.viz import quantile_regplot\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from conduitpylib.viz import unsplit_regression, facet_unsplit_regression\n" ] }, { diff --git a/binder/execute_notebooks.sh b/binder/execute_notebooks.sh index 07cb95ff9e..4d6f67937b 100755 --- a/binder/execute_notebooks.sh +++ b/binder/execute_notebooks.sh @@ -29,28 +29,19 @@ echo "--------------------------------------" shopt -s nullglob -for notebook in "${script_dir}/"*.ipynb; do - echo "notebook ${notebook}" - export NOTEBOOK_NAME="$(basename "${notebook%.*}")" - export NOTEBOOK_PATH="$(realpath "${notebook}")" +# Find all .ipynb files and use xargs to run nbconvert in parallel, within the directory of each notebook +find "${script_dir}/" -name "*.ipynb" -print0 | xargs -0 -n 1 -P 2 -I {} bash -c '{ + notebook="{}" + notebook_dir=$(dirname "$notebook") + notebook_base=$(basename "$notebook") + echo "Processing notebook ${notebook_base} in directory ${notebook_dir}" + cd "${notebook_dir}" + export KEYNAME_CHOP_CHUNK_SIZE=255 + echo "KEYNAME_CHOP_CHUNK_SIZE ${KEYNAME_CHOP_CHUNK_SIZE}" jupyter nbconvert \ --to notebook --execute --inplace \ --ExecutePreprocessor.timeout=600 \ - "${notebook}" -done - -shopt -u nullglob - -################################################################################ -echo -echo "recurse to subdirectories" -echo "-------------------------" -################################################################################ - -shopt -s nullglob - -for script in "${script_dir}/"*/execute_notebooks.sh; do - "${script}" "$@" -done + "$notebook_base" +}' shopt -u nullglob diff --git a/conduitpylib/test/test_utils/test_UnequalSentinel.py b/conduitpylib/test/test_utils/test_UnequalSentinel.py new file mode 100644 index 0000000000..f68429b2a8 --- /dev/null +++ b/conduitpylib/test/test_utils/test_UnequalSentinel.py @@ -0,0 +1,20 @@ +import typing + +import numpy as np +import pytest + +from conduitpylib.utils import UnequalSentinel + + +def test_self(): + s = UnequalSentinel() + assert s != s and not s == s + + +@pytest.mark.parametrize( + "other", [1, 0, "asdf", "", np.nan, None, True, False, UnequalSentinel()] +) +def test_other(other: typing.Any): + s = UnequalSentinel() + assert s != other and not s == other + assert other != s and not other == s diff --git a/conduitpylib/test/test_utils/test_hash_with_lax_fallback.py b/conduitpylib/test/test_utils/test_hash_with_lax_fallback.py new file mode 100644 index 0000000000..afda2b26be --- /dev/null +++ b/conduitpylib/test/test_utils/test_hash_with_lax_fallback.py @@ -0,0 +1,74 @@ +import pytest +import pandas as pd +import numpy as np + +from conduitpylib.utils import hash_with_lax_fallback + + +# Test with hashable objects +@pytest.mark.parametrize("obj, expected", [ + (42, hash(42)), # integer + ("string", hash("string")), # string + ((1, 2), hash((1, 2))), # tuple +]) +def test_hashable_objects(obj, expected): + assert hash_with_lax_fallback(obj) == expected + +# Test with Pandas objects +def test_pandas_objects(): + df = pd.DataFrame({'col1': [1, 2], 'col2': [3, 4]}) + series = pd.Series([1, 2, 3]) + + df_hash = hash_with_lax_fallback(df) + series_hash = hash_with_lax_fallback(series) + + # The hashes should not be the default id hashes + assert df_hash != id(df) + assert series_hash != id(series) + + # Re-hashing should give the same result + assert df_hash == hash_with_lax_fallback(df) + assert series_hash == hash_with_lax_fallback(series) + +# Test with Numpy arrays +def test_numpy_arrays(): + arr = np.array([1, 2, 3]) + + arr_hash = hash_with_lax_fallback(arr) + + # The hash should not be the default id hash + assert arr_hash != id(arr) + + # Re-hashing should give the same result + assert arr_hash == hash_with_lax_fallback(arr) + +# Test with unhashable and complex objects +def test_unhashable_objects(): + # Use a list as an example of unhashable object + unhashable = [1, 2, 3] + + unhashable_hash = hash_with_lax_fallback(unhashable) + + # The hash should not be the default id hash + assert unhashable_hash != id(unhashable) + + # The hash should be consistent for the same object content + assert unhashable_hash == hash_with_lax_fallback([1, 2, 3]) + +# Test fallback for an object that fails both hash and pickle +class UnhashableAndUnpickleable: + def __eq__(self, other): + return isinstance(other, UnhashableAndUnpickleable) + + def __hash__(self): + raise TypeError + + def __reduce__(self): + raise TypeError + +def test_unhashable_unpickleable_fallback(): + obj = UnhashableAndUnpickleable() + fallback_hash = hash_with_lax_fallback(obj) + + # Should fall back to using id for hashing + assert fallback_hash == id(obj) diff --git a/conduitpylib/test/test_utils/test_lru_cache_universal.py b/conduitpylib/test/test_utils/test_lru_cache_universal.py new file mode 100644 index 0000000000..8e2960c1f1 --- /dev/null +++ b/conduitpylib/test/test_utils/test_lru_cache_universal.py @@ -0,0 +1,142 @@ +import copy +import pickle + +import pytest + +from conduitpylib.utils import lru_cache_universal + + +# testing helper +class MutableObj: + def __init__(self, value): + self.value = value + + def __eq__(self, other): + return self.value == other.value + + def __hash__(self): + return hash(self.value) + + def __add__(self, other): + return MutableObj(self.value + other.value) + + +class UnhashUnpickle: + def __add__(self, other): + return UnhashUnpickle() + + def __eq__(self, other): + return isinstance(other, UnhashUnpickle) + + def __reduce__(self): + raise pickle.PicklingError() + + +@pytest.mark.parametrize( + "args, kwargs, maxsize, expected_result", + [ + ## ARGS + # Test with hashable arguments + ((1, 2), {}, None, 3), + # Test with mutable but hashable arguments + ((MutableObj(1), MutableObj(2)), {}, None, MutableObj(3)), + # Test cache with a maxsize of 1, expecting eviction of the first key + ((1, 2), {}, 1, 3), + # Test cache with None as maxsize, which means unlimited + ((1, 2), {}, None, 3), + # Test with unhashable arguments + (([1], [2]), {}, None, [1, 2]), + # Test with unhashble and unpickleable arguments + ((UnhashUnpickle(), UnhashUnpickle()), {}, None, UnhashUnpickle()), + ## KWARGS + # Test with hashable arguments + (tuple(), {"a": 1, "b": 2}, None, 3), + # Test with mutable but hashable arguments + (tuple(), {"a": MutableObj(1), "b": MutableObj(2)}, None, MutableObj(3)), + # Test cache with a maxsize of 1, expecting eviction of the first key + (tuple(), {"a": 1, "b": 2}, 1, 3), + # Test cache with None as maxsize, which means unlimited + (tuple(), {"a": 1, "b": 2}, None, 3), + # Test with unhashable arguments + (tuple(), {"a": [1], "b": [2]}, None, [1, 2]), + # Test with unhashble and unpickleable arguments + ( + tuple(), + {"a": UnhashUnpickle(), "b": UnhashUnpickle()}, + None, + UnhashUnpickle(), + ), + ## ARGS AND KWARGS + # Test with hashable arguments + ((1,), {"b": 2}, None, 3), + # Test with mutable but hashable arguments + ((MutableObj(1),), {"b": MutableObj(2)}, None, MutableObj(3)), + # Test cache with a maxsize of 1, expecting eviction of the first key + ((1,), {"b": 2}, 1, 3), + # Test cache with None as maxsize, which means unlimited + ((1,), {"b": 2}, None, 3), + # Test with unhashable arguments + (([1],), {"b": [2]}, None, [1, 2]), + # Test with unhashble and unpickleable arguments + ((UnhashUnpickle(),), {"b": UnhashUnpickle()}, None, UnhashUnpickle()), + ], +) +def test_lru_cache_universal( + args, + kwargs, + maxsize, + expected_result, +): + def test_function(a, b): + return a + b + + cached_test_function = lru_cache_universal(maxsize)(test_function) + for __ in range(2): + assert cached_test_function(*args, **kwargs) == expected_result + + try: + args = copy.deepcopy(args) + kwargs = copy.deepcopy(kwargs) + assert cached_test_function(*args, **kwargs) == expected_result + except Exception: + uutup = (UnhashUnpickle(), UnhashUnpickle()) + if args == uutup: + assert cached_test_function(*uutup) == expected_result + + +def test_cache_eviction_policy(): + call_log = [] + + def logged_test_function(*args, **kwargs): + call_log.append((args, kwargs)) + + cached_test_function = lru_cache_universal(2)(logged_test_function) + + assert len(call_log) == 0 + cached_test_function(1, 2) + assert len(call_log) == 1 + cached_test_function(2, 3) + assert len(call_log) == 2 + cached_test_function(1, 2) + assert len(call_log) == 2 + cached_test_function(2, 3) + assert len(call_log) == 2 + cached_test_function(3, 4) + assert len(call_log) == 3 + cached_test_function(2, 3) + assert len(call_log) == 3 + cached_test_function(1, 2) + assert len(call_log) == 4 + + +def test_cache_clear(): + def test_function(*args, **kwargs): + pass + + cached_test_function = lru_cache_universal(2)(test_function) + + cached_test_function(1, 2) + cached_test_function(2, 3) + assert len(cached_test_function.cache) == 2 + cached_test_function.cache_clear() + assert len(cached_test_function.cache) == 0 diff --git a/conduitpylib/test/test_utils/test_pickle_cache.py b/conduitpylib/test/test_utils/test_pickle_cache.py new file mode 100644 index 0000000000..34ab3d3671 --- /dev/null +++ b/conduitpylib/test/test_utils/test_pickle_cache.py @@ -0,0 +1,53 @@ +import pytest +import pandas as pd + +from conduitpylib.utils import pickle_cache + + +@pickle_cache() +def sample_function(df): + # A simple function that might perform some operations on a DataFrame + return df.describe() + + +@pickle_cache() +def another_function(df): + # A simple function that might perform some operations on a DataFrame + return f"{df.describe()}42" + + +@pytest.fixture +def sample_dataframe(): + # Creating a sample DataFrame to be used in tests + return pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]}) + + +@pytest.fixture +def another_dataframe(): + # Creating a sample DataFrame to be used in tests + return pd.DataFrame({"A": [1, 2, 3], "B": [1, 5, 6]}) + + +def test_cache_hit(sample_dataframe: pd.DataFrame): + sample_function(sample_dataframe) + sample_function.info["hits"] = 0 + sample_function(sample_dataframe) + assert sample_function.info["hits"] == 1 + + +def test_cache_integrity( + sample_dataframe: pd.DataFrame, another_dataframe: pd.DataFrame +): + cached_result = sample_function(sample_dataframe) + + new_result = sample_function(sample_dataframe) + assert cached_result.equals(new_result) + + another_result = another_function(another_dataframe) + assert not new_result.equals(another_result) + + yet_another_result = another_function(sample_dataframe) + assert not new_result.equals(yet_another_result) + + +# Additional tests can be added for different scenarios and edge cases diff --git a/conduitpylib/test/test_utils/test_round_to_multiple.py b/conduitpylib/test/test_utils/test_round_to_multiple.py new file mode 100644 index 0000000000..3df747c6eb --- /dev/null +++ b/conduitpylib/test/test_utils/test_round_to_multiple.py @@ -0,0 +1,13 @@ +from conduitpylib.utils import round_to_multiple + + +def test_round_to_multiple(): + # Normal cases + assert round_to_multiple(10.5, 2) == 10 + assert round_to_multiple(15.3, 5) == 15 + assert round_to_multiple(2.6, 1) == 3 + + # Edge cases + assert round_to_multiple(0, 1) == 0 + assert round_to_multiple(-5.3, 2) == -6 + assert round_to_multiple(1000000.7, 1000) == 1000000 diff --git a/conduitpylib/test/test_utils/test_seaborn_monkeypatch_kdecache.py b/conduitpylib/test/test_utils/test_seaborn_monkeypatch_kdecache.py new file mode 100644 index 0000000000..18064b3b0b --- /dev/null +++ b/conduitpylib/test/test_utils/test_seaborn_monkeypatch_kdecache.py @@ -0,0 +1,24 @@ +import seaborn as sns + +from conduitpylib.utils import seaborn_monkeypatched_kdecache + + +def test_seaborn_monkeypatch_kdecache() -> None: + geyser = sns.load_dataset("geyser") + + with seaborn_monkeypatched_kdecache(): + sns.kdeplot(data=geyser, x="waiting", y="duration", hue="kind") + sns.kdeplot(data=geyser, x="waiting", y="duration", hue="kind") + + cache_info = sns._statistics.KDE.__call__.cache_info + assert cache_info["hits"] > 0 and cache_info["misses"] > 0 + + prehits = cache_info["hits"] + sns.kdeplot(data=geyser, x="waiting", y="duration") + assert cache_info["hits"] == prehits + + sns.kdeplot(data=geyser, x="waiting", y="duration") + assert cache_info["hits"] > prehits + + assert not hasattr(sns._statistics.KDE.__call__, "cache_info") + sns.kdeplot(data=geyser, x="waiting", y="duration", hue="kind") diff --git a/conduitpylib/test/test_utils/test_splice.py b/conduitpylib/test/test_utils/test_splice.py new file mode 100644 index 0000000000..e60eac89a3 --- /dev/null +++ b/conduitpylib/test/test_utils/test_splice.py @@ -0,0 +1,29 @@ +from conduitpylib.utils import splice + + +def test_splice_removal_only(): + assert splice("Hello world", (5, 11)) == "Hello" + + +def test_splice_insertion_only(): + assert splice("123456789", (3, 3), "ABC") == "123ABC456789" + + +def test_splice_removal_and_insertion(): + assert splice("Hello world", (0, 5), "Goodbye") == "Goodbye world" + + +def test_splice_empty_string(): + assert splice("", (0, 0), "Hello") == "Hello" + + +def test_splice_full_replacement(): + assert splice("Hi", (0, 2), "Hello") == "Hello" + + +def test_splice_out_of_bounds(): + assert splice("Hello", (0, 15)) == "" + + +def test_splice_negative_indices(): + splice("Hello", (-3, -1)) diff --git a/conduitpylib/test/test_viz/__init__.py b/conduitpylib/test/test_viz/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/conduitpylib/test/test_viz/test_compact_xaxis_units.py b/conduitpylib/test/test_viz/test_compact_xaxis_units.py new file mode 100644 index 0000000000..9aa38614ca --- /dev/null +++ b/conduitpylib/test/test_viz/test_compact_xaxis_units.py @@ -0,0 +1,73 @@ +from matplotlib import pyplot as plt + +from conduitpylib.viz import compact_xaxis_units + + +def test_prefix_removal(): + # setup + ax = plt.gca() + ax.ticklabel_format(style="sci", scilimits=(-2, 2)), + ax.plot([1000, 2000, 3000], [1, 2, 3]) + ax.set_xlabel("Time (ms)") + + # apply + compact_xaxis_units(ax, base_unit="s") + + # verify + expected_label = "Time (s)" + assert ax.get_xlabel() == expected_label + + # clean up + plt.clf() + + +def test_prefix_addition(): + # setup + ax = plt.gca() + ax.ticklabel_format(style="sci", scilimits=(-2, 2)), + ax.plot([1000, 2000, 3000], [1, 2, 3]) + ax.set_xlabel("Weight (g)") + # apply + compact_xaxis_units(ax, base_unit="g") + + # verify + expected_label = "Weight (kg)" + assert ax.get_xlabel() == expected_label + + # clean up + plt.clf() + + +def test_prefix_nop(): + # setup + ax = plt.gca() + ax.ticklabel_format(style="sci", scilimits=(-2, 2)), + ax.plot([1, 2, 3], [1, 2, 3]) + ax.set_xlabel("Weight (g)") + # apply + compact_xaxis_units(ax, base_unit="g") + + # verify + expected_label = "Weight (g)" + assert ax.get_xlabel() == expected_label + + # clean up + plt.clf() + + +def test_nanoseconds_to_milliseconds(): + # setup + ax = plt.gca() + ax.ticklabel_format(style="sci", scilimits=(-2, 2)), + ax.plot([1e6, 2e6, 3e6], [1, 2, 3]) + ax.set_xlabel("Time (ns)") + + # apply + compact_xaxis_units(ax, base_unit="s") + + # verify + expected_label = "Time (ms)" + assert ax.get_xlabel() == expected_label + + # clean up + plt.clf() diff --git a/conduitpylib/utils/UnequalSentinel.py b/conduitpylib/utils/UnequalSentinel.py new file mode 100644 index 0000000000..8aaf1ca234 --- /dev/null +++ b/conduitpylib/utils/UnequalSentinel.py @@ -0,0 +1,11 @@ +import typing + + +class UnequalSentinel: + """Sentinel value that is unequal to everything, including itself.""" + + def __eq__(self: "UnequalSentinel", other: typing.Any) -> bool: + return False + + def __ne__(self: "UnequalSentinel", other: typing.Any) -> bool: + return True diff --git a/conduitpylib/utils/__init__.py b/conduitpylib/utils/__init__.py index 4ab0862614..3af836e9e5 100644 --- a/conduitpylib/utils/__init__.py +++ b/conduitpylib/utils/__init__.py @@ -1,10 +1,43 @@ """Miscellaneous utilities.""" from .consolidate_merge import consolidate_merge +from .count_outliers import ( + count_nonoutliers, + count_outliers, + count_proportion_outliers, +) +from .get_signif_asterisks import get_signif_asterisks +from .hash_with_lax_fallback import hash_with_lax_fallback +from .lru_cache_universal import lru_cache_universal +from .open_dummy_axis import open_dummy_axis +from .round_to_multiple import round_to_multiple +from .pickle_cache import pickle_cache +from .seaborn_monkeypatch_kdecache import ( + seaborn_monkeypatch_kdecache, + seaborn_monkeypatched_kdecache, + seaborn_unmonkeypatch_kdecache, +) +from .splice import splice from .strip_end import strip_end +from .UnequalSentinel import UnequalSentinel + # adapted from https://stackoverflow.com/a/31079085 __all__ = [ 'consolidate_merge', + 'count_nonoutliers', + 'count_outliers', + 'count_proportion_outliers', + 'hash_with_lax_fallback', + 'lru_cache_universal', + 'get_signif_asterisks', + 'open_dummy_axis', + 'round_to_multiple', + 'pickle_cache', + 'seaborn_monkeypatch_kdecache', + 'seaborn_monkeypatched_kdecache', + 'seaborn_unmonkeypatch_kdecache', + 'splice', 'strip_end', + 'UnequalSentinel', ] diff --git a/conduitpylib/utils/count_outliers.py b/conduitpylib/utils/count_outliers.py new file mode 100644 index 0000000000..0b1a400c08 --- /dev/null +++ b/conduitpylib/utils/count_outliers.py @@ -0,0 +1,31 @@ +import pandas as pd + + +def count_outliers(data): + """Calculate the number of outliers in a pandas column using the IQR + method.""" + + # Calculate Q1 and Q3 + Q1 = data.quantile(0.25) + Q3 = data.quantile(0.75) + + # Calculate IQR + IQR = Q3 - Q1 + + # Define bounds for outliers + lower_bound = Q1 - 1.5 * IQR + upper_bound = Q3 + 1.5 * IQR + + # Count outliers + return ( + (data < lower_bound) + | (data > upper_bound) + ).sum() + + +def count_proportion_outliers(data): + return count_outliers(data) / len(data) + + +def count_nonoutliers(data): + return len(data) - count_outliers(data) diff --git a/conduitpylib/utils/equals_by_value.py b/conduitpylib/utils/equals_by_value.py new file mode 100644 index 0000000000..c3514616b7 --- /dev/null +++ b/conduitpylib/utils/equals_by_value.py @@ -0,0 +1,40 @@ +import contextlib +import typing + +import numpy as np +import pandas as pd + +def _equals_by_value(obj1: typing.Any, obj2: typing.Any) -> bool: + + # Handle builtin datatypes recursively + if isinstance(obj1, (tuple, list)) and isinstance(obj2, (tuple, list)): + return all(_equals_by_value(o1, o2) for o1, o2 in zip(obj1, obj2)) + + if isinstance(obj1, dict) and isinstance(obj2, dict): + return len(obj1) == len(obj2) and all( + k in obj2 and _equals_by_value(obj1[k], obj2[k]) for k in obj1 + ) + + # Handle comparison between pandas DataFrames or Series + if all(isinstance(o, pd.Series) for o in (obj1, obj2)): + return np.allclose(obj1, obj2) + + elif all(isinstance(o, pd.DataFrame) for o in (obj1, obj2)): + return obj1.equals(obj2) + + # Handle comparison between numpy ndarrays + elif all(isinstance(o, np.ndarray) for o in (obj1, obj2)): + return np.all_close(obj1, obj2) + + # Fall back to 'is' comparison for NaN types + elif pd.isnull(obj1) and pd.isnull(obj2): + return True + + # Handle everything else with the '==' operator + else: + return obj1 == obj2 + + +def equals_by_value(obj1: typing.Any, obj2: typing.Any) -> bool: + with contextlib.suppress(ValueError): + return _equals_by_value(obj1, obj2) diff --git a/conduitpylib/utils/get_signif_asterisks.py b/conduitpylib/utils/get_signif_asterisks.py new file mode 100644 index 0000000000..a833035650 --- /dev/null +++ b/conduitpylib/utils/get_signif_asterisks.py @@ -0,0 +1,9 @@ +def get_signif_asterisks(pvalue: float) -> str: + if pvalue < 0.001: + return "***" + elif pvalue < 0.01: + return "**" + elif pvalue < 0.05: + return "*" + else: + return "" diff --git a/conduitpylib/utils/hash_with_lax_fallback.py b/conduitpylib/utils/hash_with_lax_fallback.py new file mode 100644 index 0000000000..376c4fa68b --- /dev/null +++ b/conduitpylib/utils/hash_with_lax_fallback.py @@ -0,0 +1,35 @@ +import contextlib +import hashlib +import pickle +import typing + +import numpy as np +import pandas as pd + + +def hash_with_lax_fallback(obj: typing.Any) -> int: + """Hash an object by value, falling back to an arbitrary hash by instance. + + Parameters + ---------- + obj : Any + The object to hash. + + Returns + ------- + int + The computed hash. + """ + with contextlib.suppress(TypeError): + return hash(obj) + + if isinstance(obj, (pd.DataFrame, pd.Series)): + return hash(tuple(pd.util.hash_pandas_object(obj))) + + elif isinstance(obj, np.ndarray): + return hash(hashlib.sha256(obj).hexdigest()) + + with contextlib.suppress(Exception): + return hashlib.sha256(pickle.dumps(obj)).hexdigest() + + return id(obj) diff --git a/conduitpylib/utils/lru_cache_universal.py b/conduitpylib/utils/lru_cache_universal.py new file mode 100644 index 0000000000..58133d7719 --- /dev/null +++ b/conduitpylib/utils/lru_cache_universal.py @@ -0,0 +1,109 @@ +import contextlib +import copy +import functools +import typing + +from .equals_by_value import equals_by_value +from .hash_with_lax_fallback import hash_with_lax_fallback +from .UnequalSentinel import UnequalSentinel + + +def _robust_copy(obj: typing.Any) -> typing.Any: + with contextlib.suppress(Exception): + return copy.deepcopy(obj) + with contextlib.suppress(Exception): + return copy.copy(obj) + with contextlib.suppress(Exception): + return obj + + +def lru_cache_universal(maxsize: typing.Optional[int] = 64) -> typing.Callable: + """Memoizing decorator that supports calls with mutable and unhashable + arguments. + + Mutable arguments will be hashed by value and unhashable arguments will be + hashed by instance id. (As such, calls with unhashable arguments that + compare equal may result in a cache miss.) + + Parameters + ---------- + maxsize : int or None, default 64 + The cache size limit, if any. + + Returns + ------- + function + Memoizing decorator. + """ + + def decorator(func: typing.Callable) -> typing.Callable: + cache = dict() + info = { + "hits": 0, + "misses": 0, + } + + @functools.wraps(func) + def wrapper(*args, **kwargs) -> typing.Any: + # prepare keys + hashable_args = tuple(map(hash_with_lax_fallback, args)) + hashable_kwargs = frozenset( + zip( + kwargs.keys(), + map(hash_with_lax_fallback, kwargs.values()), + ), + ) + query_lookup_key = (hashable_args, hashable_kwargs) + query_full_key = (args, kwargs) + + # try cache lookup + cached_full_key, cached_result = cache.get( + query_lookup_key, + (UnequalSentinel(), None), + ) + + # try indexing by lookupkey + if equals_by_value(cached_full_key, query_full_key): + # cache hit! + info["hits"] += 1 + return _robust_copy(cached_result) + + # try searching through all full keys + for cached_full_key, cached_result in cache.values(): + cached_args, cached_kwargs = cached_full_key + + if ( + all( + equals_by_value(arg, cached_arg) + for arg, cached_arg in zip(args, cached_args) + ) + and len(cached_kwargs) == len(hashable_kwargs) + and all( + equals_by_value(kwargs[key], cached_kwargs[key]) + for key in kwargs.keys() + ) + ): + # cache hit! + info["hits"] += 1 + return _robust_copy(cached_result) + + # ... cache miss! + info["misses"] += 1 + # trim cache if at max size + if maxsize is not None and len(cache) >= maxsize: + # python iterates dicts in insertion order + del cache[next(iter(cache))] + + # compute result and cache it + call_result = func(*args, **kwargs) + cache[query_lookup_key] = (query_full_key, call_result) + + return _robust_copy(call_result) + + wrapper.cache = cache + wrapper.cache_clear = cache.clear + wrapper.cache_info = info + + return wrapper + + return decorator diff --git a/conduitpylib/utils/open_dummy_axis.py b/conduitpylib/utils/open_dummy_axis.py new file mode 100644 index 0000000000..016a7b80c0 --- /dev/null +++ b/conduitpylib/utils/open_dummy_axis.py @@ -0,0 +1,25 @@ +from contextlib import contextmanager + +from matplotlib import pyplot as plt + + +@contextmanager +def open_dummy_axis(): + # Save current figure and axis + original_fig = plt.gcf() + original_ax = plt.gca() + + dummy_fig = None + try: + # Create a new figure for the dummy axis + dummy_fig = plt.figure() + dummy_ax = dummy_fig.add_subplot(111) + + # Yield control back to the block of code using the context manager + yield dummy_ax + finally: + # Close the dummy figure and restore the original figure and axis + if dummy_fig is not None: + plt.close(dummy_fig) + plt.figure(original_fig.number) + plt.sca(original_ax) diff --git a/conduitpylib/utils/pickle_cache.py b/conduitpylib/utils/pickle_cache.py new file mode 100644 index 0000000000..f7c9fbd539 --- /dev/null +++ b/conduitpylib/utils/pickle_cache.py @@ -0,0 +1,70 @@ +from functools import wraps +import hashlib +import inspect +from pathlib import Path +import shelve +import typing + +from .equals_by_value import equals_by_value + + +def _raises(callable: typing.Callable) -> bool: + try: + callable() + except Exception: + return True + else: + return False + + +def pickle_cache(maxsize: int = 256) -> typing.Callable: + def decorator(func: typing.Callable) -> typing.Callable: + info = { + "hits": 0, + "misses": 0, + } + + @wraps(func) + def wrapped(*args, **kwargs) -> typing.Any: + # Serialize the arguments for hashing + own_source = Path(__file__).read_text() + try: + func_source = Path(inspect.getfile(func)).read_text() + except TypeError: + func_source = inspect.getfile(func) + context = (func.__qualname__, func_source, args, kwargs, own_source) + serialized_context = str(context) + hash_value = hashlib.sha256(serialized_context.encode()).hexdigest() + + # Define the shelve file path + cache_dir = ( + Path.home() / ".cache" / "conduitpylib.utils.pickle_cache" + ) + cache_dir.mkdir(exist_ok=True) + shelve_path = cache_dir / f"{func.__qualname__}.shlv" + + with shelve.open(str(shelve_path), "c") as shelf: + # Check if cache size limit is reached + if len(shelf) >= maxsize: + # Remove least recently used item + oldest, *_rest = shelf.keys() + del shelf[oldest] + + if ( + hash_value not in shelf + or _raises(lambda: shelf[hash_value]) + or not equals_by_value(shelf[hash_value][0], context) + ): + info["misses"] += 1 + result = func(*args, **kwargs) + shelf[hash_value] = (context, result) + return result + else: + info["hits"] += 1 + # optimization: could mark item as most recently used here + return shelf[hash_value][1] + + wrapped.info = info + return wrapped + + return decorator diff --git a/conduitpylib/utils/round_to_multiple.py b/conduitpylib/utils/round_to_multiple.py new file mode 100644 index 0000000000..11c0193fc2 --- /dev/null +++ b/conduitpylib/utils/round_to_multiple.py @@ -0,0 +1,18 @@ +def round_to_multiple(number: float, multiple: int) -> int: + """Round to the nearest multiple of an integer. + + Parameters + ---------- + number : float + The number to be rounded. + multiple : float + The multiple to which the number will be rounded. + + Returns + ------- + float + The number rounded to the nearest multiple. + """ + res = int(round(number / multiple) * multiple) + assert res % multiple == 0 + return res diff --git a/conduitpylib/utils/seaborn_monkeypatch_kdecache.py b/conduitpylib/utils/seaborn_monkeypatch_kdecache.py new file mode 100644 index 0000000000..47ad1436da --- /dev/null +++ b/conduitpylib/utils/seaborn_monkeypatch_kdecache.py @@ -0,0 +1,44 @@ +import contextlib + +import seaborn as sns + +from .equals_by_value import equals_by_value +from .lru_cache_universal import lru_cache_universal + + +original_eq_func = sns._statistics.KDE.__eq__ +original_kde_func = sns._statistics.KDE.__call__ + + +def patched_eq_func(self, other): + if type(self) == type(other): + sd, od = self.__dict__, other.__dict__ + if len(sd) != len(od): + return False + for key in sd: + if key in od and equals_by_value(sd[key], od[key]): + continue + else: + return False + + return True + + +patched_kde_func = lru_cache_universal()(original_kde_func) + + +def seaborn_monkeypatch_kdecache() -> None: + sns._statistics.KDE.__call__ = patched_kde_func + sns._statistics.KDE.__eq__ = patched_eq_func + + +def seaborn_unmonkeypatch_kdecache() -> None: + sns._statistics.KDE.__call__ = original_kde_func + sns._statistics.KDE.__eq__ = original_eq_func + + +@contextlib.contextmanager +def seaborn_monkeypatched_kdecache(): + seaborn_monkeypatch_kdecache() + yield + seaborn_unmonkeypatch_kdecache() diff --git a/conduitpylib/utils/splice.py b/conduitpylib/utils/splice.py new file mode 100644 index 0000000000..30be40eff9 --- /dev/null +++ b/conduitpylib/utils/splice.py @@ -0,0 +1,33 @@ +import typing + + +def splice(string: str, span: typing.Tuple[int, int], insert: str = "") -> str: + """Remove index span from string and optionally insert new content. + + Parameters + ---------- + string : str + The original string to be spliced. + span : tuple[int, int] + A tuple indicating the start and end indices for the splice operation. + + The start index is inclusive and the end index is exclusive. + insert : str, default "" + The string to be inserted in place of the removed span. + + Defaults to an empty string, i.e., simple removal. + + Returns + ------- + str + The spliced string after removal and insert operation. + + Examples + -------- + >>> splice("Hello world", (1, 6)) + 'Hworld' + >>> splice("Hello world", (0, 5), "Goodbye") + 'Goodbye world' + """ + start, end = span + return string[:start] + insert + string[end:] diff --git a/conduitpylib/viz/_DrawBatched.py b/conduitpylib/viz/_DrawBatched.py new file mode 100644 index 0000000000..32a342a184 --- /dev/null +++ b/conduitpylib/viz/_DrawBatched.py @@ -0,0 +1,59 @@ +import typing +import sys + +from matplotlib import pyplot as plt +import more_itertools as mit +import numpy as np +import pandas as pd + + +class DrawBatched: + DataPlaceholder = object() + ZorderPlaceholder = object() + + _batches: typing.Any + _draw_func: typing.Callable + + def __init__( + self: "DrawBatched", + draw_func: typing.Callable, + data: pd.DataFrame, + batch_size: int = 1, + batch_limit: int = sys.maxsize, + mutate: bool = False, + progress_apply: typing.Callable = lambda x: x, + random_state: typing.Optional[int] = 1, + sort_by: typing.Optional[str] = None, + ) -> None: + if not mutate: + data = data.copy() + + round_up = batch_size - 1 + num_batches = (len(data) + round_up) // batch_size + num_batches = min(num_batches, batch_limit) + if num_batches == 0: + assert len(data) == 0 + + data = data.sample(frac=1, random_state=random_state) + if sort_by is not None: + data = data.sort_values(sort_by) + data.reset_index(drop=True, inplace=True) + + self._draw_func = draw_func + batches = np.array_split(data, num_batches) if num_batches else [] + self._batches = progress_apply(batches) + + def __call__(self: "DrawBatched", *args, **kwargs) -> typing.Any: + for i, batch in enumerate(self._batches): + + def replace(obj): + return { + id(self.DataPlaceholder): batch, + id(self.ZorderPlaceholder): i, + }.get(id(obj), obj) + + kwargs_ = {key: replace(value) for key, value in kwargs.items()} + self._draw_func( + *map(replace, args), + **{"zorder": i, **kwargs_}, + ) diff --git a/conduitpylib/viz/__init__.py b/conduitpylib/viz/__init__.py new file mode 100644 index 0000000000..f0ecf63b5a --- /dev/null +++ b/conduitpylib/viz/__init__.py @@ -0,0 +1,65 @@ +"""Visualization tools.""" + +from ._align_marker import align_marker +from ._annotate_spearman import annotate_spearman +from ._apply_symmetric_axes import apply_symmetric_axes +from ._beleaguerment_regplot import beleaguerment_regplot +from ._beleaguerment_facetplot import beleaguerment_facetplot +from ._compact_xaxis_units import compact_xaxis_units +from ._draw_edge_markers import draw_edge_markers +from ._DrawBatched import DrawBatched +from ._errplot import errplot, facet_errplot +from ._facet_barplot import facet_barplot +from ._facet_boxplot import ( + facet_boxplot, + facet_boxplot_nofliers, + facet_boxplot_withfliers, +) +from ._frame_scatter_subsets import frame_scatter_subsets +from ._get_notomoji_font_path import get_notomoji_font_path +from ._performance_semantics_facetplot import performance_semantics_facetplot +from ._performance_semantics_kdeplot import performance_semantics_kdeplot +from ._performance_semantics_scatterplot import ( + performance_semantics_scatterplot, +) +from ._quantile_regplot import quantile_regplot +from ._runtime_indicator_regplot import runtime_indicator_regplot +from ._runtime_indicators_concat_regplot_pairplot import ( + runtime_indicators_concat_regplot_pairplot, +) +from ._runtime_indicators_pairplot import runtime_indicators_pairplot +from ._runtime_indicators_regplot import runtime_indicators_regplot +from ._set_kde_lims import set_kde_lims +from ._unsplit_regression import facet_unsplit_regression, unsplit_regression + + +# adapted from https://stackoverflow.com/a/31079085 +__all__ = [ + "align_marker", + "apply_symmetric_axes", + "annotate_spearman", + "beleaguerment_facetplot", + "beleaguerment_regplot", + "compact_xaxis_units", + "draw_edge_markers", + "DrawBatched", + "errplot", + "facet_barplot", + "facet_errplot", + "frame_scatter_subsets", + "facet_barplot", + "facet_boxplot_nofliers", + "facet_boxplot_withfliers", + "facet_unsplit_regression", + "get_notomoji_font_path", + "performance_semantics_facetplot", + "performance_semantics_kdeplot", + "performance_semantics_scatterplot", + "quantile_regplot", + "runtime_indicator_regplot", + "runtime_indicators_concat_regplot_pairplot", + "runtime_indicators_pairplot", + "runtime_indicators_regplot", + "set_kde_lims", + "unsplit_regression", +] diff --git a/conduitpylib/viz/_align_marker.py b/conduitpylib/viz/_align_marker.py new file mode 100644 index 0000000000..ce43ea22cd --- /dev/null +++ b/conduitpylib/viz/_align_marker.py @@ -0,0 +1,70 @@ +import typing + +from matplotlib import markers as mpl_markers +from matplotlib.path import Path as mpl_Path + + +# adapted from https://stackoverflow.com/a/26726237 +def align_marker( + marker: str, + halign: typing.Literal["left", "center", "right"] = "center", + valign: typing.Literal["top", "middle", "bottom"] = "middle", + pad: float = 1.0, +) -> mpl_Path: + """Create markers with specified alignment. + + Parameters + ---------- + marker : a valid marker specification. + See mpl.markers + halign : string, float {'left', 'center', 'right'} + Specifies the horizontal alignment of the marker. *float* values + specify the alignment in units of the markersize/2 (0 is 'center', + -1 is 'right', 1 is 'left'). + valign : string, float {'top', 'middle', 'bottom'} + Specifies the vertical alignment of the marker. *float* values + specify the alignment in units of the markersize/2 (0 is 'middle', + -1 is 'top', 1 is 'bottom'). + + Returns + ------- + marker_array : numpy.ndarray + A Nx2 array that specifies the marker path relative to the + plot target point at (0, 0). + + Notes + ----- + The mark_array can be passed directly to ax.plot and ax.scatter, e.g.:: + + ax.plot(1, 1, marker=align_marker('>', 'left')) + """ + + if isinstance(halign, str): + halign = { + "right": -pad, + "middle": 0.0, + "center": 0.0, + "left": pad, + }[halign] + + if isinstance(valign, str): + valign = { + "top": -pad, + "middle": 0.0, + "center": 0.0, + "bottom": pad, + }[valign] + + # Define the base marker + bm = mpl_markers.MarkerStyle(marker) + + # Get the marker path and apply the marker transform to get the + # actual marker vertices (they should all be in a unit-square + # centered at (0, 0)) + m_arr = bm.get_path().transformed(bm.get_transform()).vertices + + # Shift the marker vertices for the specified alignment. + m_arr[:, 0] += halign / 2 + m_arr[:, 1] += valign / 2 + + return mpl_Path(m_arr, bm.get_path().codes) diff --git a/conduitpylib/viz/_annotate_spearman.py b/conduitpylib/viz/_annotate_spearman.py new file mode 100644 index 0000000000..8ff3caf15e --- /dev/null +++ b/conduitpylib/viz/_annotate_spearman.py @@ -0,0 +1,44 @@ +import numpy as np +from matplotlib import pyplot as plt +from scipy import stats + +from ..utils import get_signif_asterisks + + +def annotate_spearman(x: str, y: str, **kwargs) -> None: + data = kwargs.pop("data") + ax = kwargs.pop("ax", plt.gca()) + + # Calculate Spearman correlation and significance + corr, p_value = stats.spearmanr(data[x], data[y]) + stars = get_signif_asterisks(p_value) + + # Linear regression + slope, _intercept, _r_value, _p_value, _std_err = stats.linregress( + data[x], data[y] + ) + + if np.isnan(corr): + assert np.isnan(slope) + return # Nop + + # Annotate the plot + ax.annotate( + f"Rho: {corr:.2f} {stars} / Slope: {slope:.2f}\nn:{len(data)}", + **{ + **dict( + xy=(0.05, 0.95), + xycoords="axes fraction", + ha="left", + va="top", + bbox=dict( + alpha=1.0, + boxstyle="round,pad=0.5", + facecolor="white", + zorder=900, + ), + zorder=1000, + ), + **kwargs, + }, + ) diff --git a/conduitpylib/viz/_apply_symmetric_axes.py b/conduitpylib/viz/_apply_symmetric_axes.py new file mode 100644 index 0000000000..7b1bc100ca --- /dev/null +++ b/conduitpylib/viz/_apply_symmetric_axes.py @@ -0,0 +1,37 @@ +import itertools as it +import operator + +from matplotlib import pyplot as plt +from matplotlib import ticker as mpl_ticker + + +def apply_symmetric_axes(ax: plt.Axes) -> None: + """ + Configures a Matplotlib axis to have a square aspect ratio, + the same axis limits on X and Y, and uses the default Matplotlib ticker + to set the ticks on X and Y. + + Parameters: + ax (matplotlib.axes.Axes): The axis to configure. + """ + # 1) Setting the aspect of the axis to be equal (square plot) + ax.set_aspect("equal", adjustable="box") + + # 2) Setting the same limits for x and y axes + xlim = ax.get_xlim() + ylim = ax.get_ylim() + + shared_lim = (min(*xlim, *ylim), max(*xlim, *ylim)) + lim_width = operator.sub(*shared_lim) + + padded_lim = ( + shared_lim[0] - lim_width * 0.05, + shared_lim[1] + lim_width * 0.05, + ) + + ax.set_xlim(*padded_lim) + ax.set_ylim(*padded_lim) + + # 3) Using the default Matplotlib ticker for setting the ticks + ax.xaxis.set_major_locator(mpl_ticker.AutoLocator()) + ax.yaxis.set_major_locator(mpl_ticker.AutoLocator()) diff --git a/conduitpylib/viz/_beleaguerment_facetplot.py b/conduitpylib/viz/_beleaguerment_facetplot.py new file mode 100644 index 0000000000..863946a90f --- /dev/null +++ b/conduitpylib/viz/_beleaguerment_facetplot.py @@ -0,0 +1,142 @@ +import typing + +from frozendict import frozendict +import numpy as np +import pandas as pd +import seaborn as sns + +from ._apply_symmetric_axes import apply_symmetric_axes +from ._DrawBatched import DrawBatched +from ._beleaguerment_regplot import beleaguerment_regplot +from ._get_defaults import get_default_palette + + +def beleaguerment_facetplot( + data: pd.DataFrame, + x: str = "Messages Received Per Second", + y: str = "Messages Sent Per Second", + hue: typing.Optional[str] = None, + hue_order: typing.Optional[typing.List[str]] = None, + palette: typing.Optional[typing.List[str]] = None, + annotation_kwargs: typing.Dict = frozendict(), + kde_kwargs: typing.Dict = frozendict(), + regline_kwargs: typing.Dict = frozendict(), + rugplot_kwargs: typing.Dict = frozendict(), + scatter_kwargs: typing.Dict = frozendict(), + scatter_outline_kwargs: typing.Dict = frozendict(), +) -> sns.FacetGrid: + if palette is None: + palette = get_default_palette() + + data_real = data.copy() + data_dummy = data.copy() + + if hue is not None: + if hue_order is None: + hue_order = sorted(data[hue].unique()) + if not set(hue_order) <= set(data[hue].unique()): + raise ValueError("hue_order has hue categories missing from data.") + elif hue_order is not None: + raise ValueError("If hue is None, hue_order must also be None.") + + if hue is not None: + data_dummy[hue] = "dummy" + data_dummy[x] = np.nan + data_dummy[y] = np.nan + + data_with_spoof = pd.concat([data_dummy, data_real]) + + g = sns.FacetGrid( + data_with_spoof, + aspect=1.2, + col=hue, + hue=hue, + hue_order=["dummy", *hue_order], + sharex=False, + sharey=False, + palette=[(0.0, 0.0, 0.0, 0.0), *palette], + height=4, + ) + + bothax = g.axes.flat[0] + bothax.clear() + bothax.cla() + sns.kdeplot( + data=data, + x=x, + y=y, + hue=hue, + hue_order=hue_order, + ax=bothax, + **{ + "alpha": 0.7, + "cut": 10, + "fill": True, + "legend": True, + "palette": palette, + **kde_kwargs, + }, + ) + sns.rugplot( + data=data, + x=x, + y=y, + ax=bothax, + **{ + "color": "darkgray", + "linewidth": 20, + "height": 0.12, + "legend": False, + }, + ) + + batching_kwargs = { + key: rugplot_kwargs.pop(key) + for key in ( + "batch_size", + "batch_limit", + "progress_apply", + "random_state", + "sort_by", + ) + if key in rugplot_kwargs + } + DrawBatched(sns.rugplot, data, **batching_kwargs)( + data=DrawBatched.DataPlaceholder, + x=x, + y=y, + hue=hue, + hue_order=hue_order, + ax=bothax, + **{ + "linewidth": 5, + "height": 0.1, + "legend": False, + "palette": palette, + **rugplot_kwargs, + }, + ) + sns.move_legend(bothax, loc="upper left") + + bothax.axline((0, 0), slope=1, color="black", linestyle=":", lw=2) + + g.axes.flat[0].set_xlim(left=0) + g.axes.flat[0].set_ylim(bottom=0) + + map(apply_symmetric_axes, g.axes.flat) + + g.map_dataframe( + beleaguerment_regplot, + x=x, + y=y, + annotation_kwargs=annotation_kwargs, + regline_kwargs=regline_kwargs, + scatter_kwargs=scatter_kwargs, + scatter_outline_kwargs=scatter_outline_kwargs, + ) + + for ax in g.axes.flat: + ax.ticklabel_format(scilimits=(-2, 2)) + ax.grid(True) + + return g diff --git a/conduitpylib/viz/_beleaguerment_regplot.py b/conduitpylib/viz/_beleaguerment_regplot.py new file mode 100644 index 0000000000..f157341166 --- /dev/null +++ b/conduitpylib/viz/_beleaguerment_regplot.py @@ -0,0 +1,79 @@ +import typing + +from frozendict import frozendict +from matplotlib import axes as mpl_axes +from matplotlib import patheffects as mpl_pe +from matplotlib import pyplot as plt +import pandas as pd +import seaborn as sns + +from ._annotate_spearman import annotate_spearman + + +def beleaguerment_regplot( + data: pd.DataFrame, + x: str = "Messages Received Per Second", + y: str = "Messages Sent Per Second", + ax: mpl_axes.Axes = None, + annotation_kwargs: typing.Dict = frozendict(), + regline_kwargs: typing.Dict = frozendict(), + scatter_kwargs: typing.Dict = frozendict(), + scatter_outline_kwargs: typing.Dict = frozendict(), + **kwargs, +) -> mpl_axes.Axes: + if ax is None: + ax = plt.gca() + + # outline scattered points + sns.kdeplot( + data=data, + x=x, + y=y, + ax=ax, + fill=False, + **{ + "thresh": 1e-6, + "cut": 10, + "lw": 0.5, + "alpha": 0.6, + "levels": 1, + **scatter_outline_kwargs, + }, + ) + + sns.regplot( + data=data, + x=x, + y=y, + ax=ax, + line_kws={ + **dict( + # zorder=100, + lw=3, + ls="--", + path_effects=[ + mpl_pe.Stroke(linewidth=4, foreground="white"), + mpl_pe.Stroke(linewidth=2, foreground="lightsalmon"), + ], + ), + **regline_kwargs, + }, + scatter_kws={ + "linewidths": 1, + "marker": "D", + "edgecolors": "black", + **scatter_kwargs, + **kwargs, + }, + truncate=False, + ) + # style seaborn's regplot error band + # adapted from https://stackoverflow.com/a/48673499 + plt.setp( + ax.collections[-1], + alpha=0.2, + ec="gray", + linewidth=2, + ) + + annotate_spearman(data=data, x=x, y=y, ax=ax, **annotation_kwargs) diff --git a/conduitpylib/viz/_build_quasiroman_arrow.py b/conduitpylib/viz/_build_quasiroman_arrow.py new file mode 100644 index 0000000000..924baf36aa --- /dev/null +++ b/conduitpylib/viz/_build_quasiroman_arrow.py @@ -0,0 +1,22 @@ +def build_quasiroman_arrow(n: int) -> str: + # Simple Roman numeral conversion using only I, V, and X + roman_parts = [] + num = n + roman_parts.extend(["M"] * (num // 1000)) # Add 'X' for each multiple of 10 + num %= 1000 + roman_parts.extend(["D"] * (num // 500)) # Add 'X' for each multiple of 10 + num %= 500 + roman_parts.extend(["C"] * (num // 100)) # Add 'X' for each multiple of 10 + num %= 100 + roman_parts.extend(["L"] * (num // 50)) # Add 'X' for each multiple of 10 + num %= 50 + roman_parts.extend(["X"] * (num // 10)) # Add 'X' for each multiple of 10 + num %= 10 + roman_parts.extend(["V"] * (num // 5)) # Add 'V' for each multiple of 5 + num %= 5 + roman_parts.extend([r"/"] * num) # Add 'I' for the remainder + + # Join with " \! " to form the arrow-like structure + res = "$" + r" \! ".join(roman_parts) + r" \! $-$\!\!\guilsinglright$" + print(n, res) + return res diff --git a/conduitpylib/viz/_calc_performance_semantics_axis_lims.py b/conduitpylib/viz/_calc_performance_semantics_axis_lims.py new file mode 100644 index 0000000000..5d55bcc2fd --- /dev/null +++ b/conduitpylib/viz/_calc_performance_semantics_axis_lims.py @@ -0,0 +1,74 @@ +import itertools as it +import typing + + +from matplotlib.collections import ( + PathCollection as mpl_PathCollection, +) +from matplotlib.font_manager import FontProperties as mpl_FontProperties +from matplotlib.legend_handler import ( + HandlerPathCollection as mpl_HandlerPathCollection, + HandlerLine2D as mpl_HandlerLine2D, +) +import matplotlib.patches as mpl_patches +from matplotlib import pyplot as plt +import numpy as np +import pandas as pd +import seaborn as sns + +from ..utils import ( + lru_cache_universal, + seaborn_monkeypatched_kdecache, +) +from ._frame_scatter_subsets import frame_scatter_subsets +from ._set_kde_lims import set_kde_lims + + +@lru_cache_universal() +@seaborn_monkeypatched_kdecache() +def calc_performance_semantics_axis_lims( + data: pd.DataFrame, + x: str = "Simstep Period Inlet (ns)", + y: str = "Latency Simsteps Inlet", + hue: typing.Optional[str] = None, + outlier_percentile_x: float = 99.0, +) -> typing.Tuple[float, float, float, float]: + dummy_fig = plt.figure() + dummy_ax = dummy_fig.add_subplot(111) + + percentile99x = np.percentile(data[x], outlier_percentile_x) + outlier_thresh = percentile99x * 2 + + sns.kdeplot( + ax=dummy_ax, + data=data[data[x] < outlier_thresh], + x=x, + y=y, + alpha=0.0, + cut=10, + fill=True, # needed for set_kde_lims + legend=False, + levels=2, + log_scale=(False, True), + thresh=1e-12, + zorder=100, + **({"hue": hue} if hue is not None else {}), + ) + + # ensure that entire KDE is visible + set_kde_lims(dummy_ax, log_x=False, log_y=True) + xmin, xmax = dummy_ax.get_xlim() + ymin, ymax = dummy_ax.get_ylim() + + # prevent dummy axes from showing in jupyter notebook + plt.close(dummy_fig) + + # ensure adequate lower coverage + xmin = min(0, xmin) + ymin = min(0.4, ymin) + + # make room for rugplot elements + xmax *= 1.3 + ymax *= 3 + + return xmin, xmax, ymin, ymax diff --git a/conduitpylib/viz/_compact_xaxis_units.py b/conduitpylib/viz/_compact_xaxis_units.py new file mode 100644 index 0000000000..bcdfe4b198 --- /dev/null +++ b/conduitpylib/viz/_compact_xaxis_units.py @@ -0,0 +1,101 @@ +import re + +from bidict import frozenbidict +from matplotlib import axis +import numpy as np + +from ..utils import round_to_multiple, splice + +_si_prefixes = frozenbidict( + { + "Y": 24, # yotta + "Z": 21, # zetta + "E": 18, # exa + "P": 15, # peta + "T": 12, # tera + "G": 9, # giga + "M": 6, # mega + "k": 3, # kilo + "": 0, # base + "m": -3, # milli + "u": -6, # micro + "n": -9, # nano + "p": -12, # pico + "f": -15, # femto + "a": -18, # atto + "z": -21, # zepto + "y": -24, # yocto + }, +) + + +def compact_xaxis_units( + ax: axis.Axis, + base_unit: str = "s", + regex_pattern_template: str = r"\((.*?){base_unit}\)", +) -> None: + r"""Adjusts the units on the x-axis of a matplotlib axis object to a more + compact form. + + This function finds the multiplier (such as 10^3, 10^6, etc.) used by + matplotlib for the axis, identifies the current unit prefix (like k for + kilo, M for mega), and then consolidates these into a new, more compact unit + prefix. This is particularly useful for graphs where the axis values are + very large or very small, and a more readable unit is desired. + + Parameters + ---------- + ax : axis.Axis + Axis object whose x-axis units are to be compacted inplace. + base_unit : str, default "s" + The base unit (without prefix) for the axis. + regex_pattern_template : str, default r"\((.*?){base_unit}\)" + A regular expression template used to identify and extract the unit + prefix from the axis label. + + The template must contain `{base_unit}` as a + placeholder for the actual base unit. Default matches parenthesized expressions like "(ks)" for kiloseconds. + """ + regex_pattern = regex_pattern_template.format(base_unit=base_unit) + regex = re.compile(regex_pattern) + + # force population of + fig = ax.get_figure() + fig.canvas.draw() + + # get unit multiplier chosen by matplotlib + offset_text = ax.xaxis.get_offset_text() + if offset_text is None: # handle unit (i.e., non) multiplier + return + offset_string = offset_text.get_text() + if offset_string == "": # handle unit (i.e., non) multiplier + return + offset_amount = float(offset_string) + + assert str(offset_amount).count("1") == 1 # power of 10 + + # get unit prefix and multiplier in axis label + old_label_string = ax.get_xlabel() + (old_prefix_match,) = regex.finditer(old_label_string) + old_prefix = old_prefix_match.group(1) + old_multiplier_pow = _si_prefixes[old_prefix] + old_multiplier = 10.0**old_multiplier_pow + + # calculate new prefix from net multiplier + new_multiplier = offset_amount * old_multiplier + approx_pow = round_to_multiple(np.log10(new_multiplier), multiple=1) + new_multiplier_pow = round_to_multiple(approx_pow, multiple=3) + leftover_pow = approx_pow - new_multiplier_pow + + new_prefix = _si_prefixes.inv[new_multiplier_pow] + if leftover_pow: + new_prefix = f"10^{leftover_pow}" + new_prefix + + ax.xaxis.get_offset_text().set(visible=False) # remove offset axis scale + old_prefix_span = old_prefix_match.span(1) + new_label_string = splice( + old_label_string, + old_prefix_span, + new_prefix, + ) + ax.set_xlabel(new_label_string) diff --git a/conduitpylib/viz/_draw_edge_markers.py b/conduitpylib/viz/_draw_edge_markers.py new file mode 100644 index 0000000000..a8cd71993f --- /dev/null +++ b/conduitpylib/viz/_draw_edge_markers.py @@ -0,0 +1,143 @@ +import operator +import typing + +from frozendict import frozendict +import numpy as np + +from matplotlib.axes import Axes as mpl_Axes +from matplotlib.collections import PathCollection as mpl_PathCollection +from matplotlib.container import ErrorbarContainer as mpl_ErrorbarContainer + +from ._align_marker import align_marker + + +def draw_edge_marker( + ax: mpl_Axes, + x: float, + y: float, + annotation_kwargs: typing.Dict, + marker_kwargs: typing.Dict, + offset: float = 0.10, +) -> mpl_Axes: + """ + Draw an edge marker for a single point if it's out of bounds. + + Parameters + ---------- + ax : matplotlib.axes.Axes + The axes object to draw markers on. + x, y : float + The x and y coordinates of the point. + """ + xlim, ylim = ax.get_xlim(), ax.get_ylim() + edge_x, edge_y, markers = x, y, list() + + xwidth = -operator.__sub__(*xlim) + ywidth = -operator.__sub__(*ylim) + xoff = xwidth * offset + yoff = ywidth * offset + if x < xlim[0]: + amount = int((xlim[0] - x) / xwidth) + backslash = "\\" + edge_x = xlim[0] - xoff + markers.append(f"$â—…{backslash * amount}$") + elif x > xlim[1]: + amount = int((x - xlim[1]) / xwidth) + m = rf"$| \!\! \leftrightarrow \!\!|{{\times}}{amount}\rangle\!\rangle\!\rangle$" + edge_x = xlim[1] + xoff + # markers.append(align_marker(m, halign="left")) + markers.append(align_marker(m, halign="right", pad=1.3)) + + if y < ylim[0]: + edge_y = ylim[0] - yoff + markers.append("^") + elif y > ylim[1]: + edge_y = ylim[1] + yoff + markers.append("v") + + for marker in markers: + ax.plot( + edge_x, + edge_y, + **{ + "color": "none", + "clip_on": False, + "linewidth": 1, + "color": "red", + "marker": marker, + "markersize": 20, + "markeredgewidth": 0.5, + "zorder": 10000, + **marker_kwargs, + }, + ) + + return edge_x, edge_y + + +def draw_edge_markers( + ax: mpl_Axes, + annotation_kwargs: typing.Dict = frozendict(), + marker_kwargs: typing.Dict = frozendict(), + offset: float = 0.10, +) -> mpl_Axes: + """ + Draw edge markers for points lying outside the plot's bounds. + + Parameters + ---------- + ax : matplotlib.axes.Axes + The axes object to draw markers on. + """ + xlim, ylim = ax.get_xlim(), ax.get_ylim() + + for collection in ax.collections: + if isinstance(collection, mpl_PathCollection): + # Extract points from the line + new_offsets = [] + for x, y in collection.get_offsets(): + # Draw edge marker if the point is out of bounds + if x < xlim[0] or x > xlim[1] or y < ylim[0] or y > ylim[1]: + x, y = draw_edge_marker( + ax, + x, + y, + annotation_kwargs=annotation_kwargs, + marker_kwargs=marker_kwargs, + offset=offset, + ) + new_offsets.append((x, y)) + + collection.set_offsets(new_offsets) + collection.set(clip_on=False) + + x_width = -operator.__sub__(*xlim) + x_thresh = xlim[1] + x_offset = x_thresh + x_width * offset + for container in ax.containers: + if isinstance(container, mpl_ErrorbarContainer): + # Unpack the container + ( + plotline, # Line2D instance of x, y plot markers and/or line + caplines, # A tuple of Line2D instances of the error bar caps + # A tuple of LineCollection with the horizontal and vertical + # error ranges. + barlinecols, + ) = container + + # Adjust each error bar + for barlinecol in barlinecols: + segments = barlinecol.get_segments() + new_segments = [] + for segment in segments: + (x0, y0), (x1, y1) = segment + if x0 > x_thresh: + assert x0 == x1 + x0, x1 = x_offset, x_offset + print(f"{x0=}, {x1=}") + print(f"{y0=}, {y1=}") + new_segment = [(x0, y0), (x1, y1)] + new_segments.append(new_segment) + + barlinecol.set_segments(new_segments) + barlinecol.set(clip_on=False) diff --git a/conduitpylib/viz/_errplot.py b/conduitpylib/viz/_errplot.py new file mode 100644 index 0000000000..82c13ed23b --- /dev/null +++ b/conduitpylib/viz/_errplot.py @@ -0,0 +1,79 @@ +from matplotlib import pyplot as plt +import numpy as np +import seaborn as sns + +# adapted from https://stackoverflow.com/questions/30385975/seaborn-factor-plot-custom-error-bars +# and https://pandas.pydata.org/pandas-docs/stable/user_guide/visualization.html#visualization-errorbars +def errplot(x, y, err_lb, err_ub, **kwargs): + ax = plt.gca() + data = kwargs.pop("data") + yerr = np.abs( + data[[err_lb, err_ub]].to_numpy() - data[[y, y]].to_numpy() + ).transpose() + plt.axhline( + y=0, + zorder=1, + color="black", + linewidth=2, + ) + data.plot( + x=x, + y=y, + yerr=yerr, + kind="bar", + ax=ax, + zorder=3, + **kwargs, + ).grid( + axis="y", + zorder=0, + ) + + if x is None: + # adapted from https://stackoverflow.com/a/12998531 + plt.tick_params( + axis="x", # changes apply to the x-axis + which="both", # both major and minor ticks are affected + bottom=False, # ticks along the bottom edge are off + top=False, # ticks along the top edge are off + labelbottom=False, + ) + + +def facet_errplot( + *, + data, + x=None, + y, + err_lb, + err_ub, + estimated_statistic, + col=None, + row=None, + size_inches=None, + **kwargs, +): + g = sns.FacetGrid( + data, + col=col if col is not None and data[col].nunique() > 1 else None, + row=row if row is not None and data[row].nunique() > 1 else None, + margin_titles=True, + sharey=False, + ) + g.map_dataframe( + errplot, + x, + y, + err_lb, + err_ub, + **kwargs, + ) + + if size_inches is not None: + plt.gcf().set_size_inches(*size_inches) + + # adapted from https://stackoverflow.com/a/29814281 + plt.gcf().subplots_adjust(top=0.9) + plt.gcf().suptitle( + f"Estimated Statistic = {estimated_statistic}", + ) diff --git a/conduitpylib/viz/_facet_barplot.py b/conduitpylib/viz/_facet_barplot.py new file mode 100644 index 0000000000..681ee525eb --- /dev/null +++ b/conduitpylib/viz/_facet_barplot.py @@ -0,0 +1,20 @@ +import seaborn as sns + + +def facet_barplot(*, data, col=None, row=None, x, y, hue=None): + g = sns.FacetGrid( + data, + col=col if col is not None and data[col].nunique() > 1 else None, + row=row if row is not None and data[row].nunique() > 1 else None, + margin_titles=True, + sharey="row", + ) + g.map_dataframe( + sns.barplot, + x=x, + y=y, + hue=hue, + ) + + # adapted from https://stackoverflow.com/a/48208266 + g.set_axis_labels(x_var=x, y_var=y) diff --git a/conduitpylib/viz/_facet_boxplot.py b/conduitpylib/viz/_facet_boxplot.py new file mode 100644 index 0000000000..c393f8f12b --- /dev/null +++ b/conduitpylib/viz/_facet_boxplot.py @@ -0,0 +1,57 @@ +import seaborn as sns + + +def facet_boxplot(*, data, col=None, row=None, x, y, showfliers=False): + g = sns.FacetGrid( + data, + col=col if col is not None and data[col].nunique() > 1 else None, + row=row if row is not None and data[row].nunique() > 1 else None, + margin_titles=True, + sharey="row", + ) + g.map_dataframe( + sns.boxplot, + x, + y, + showfliers=showfliers, + ) + + +def facet_boxplot_withfliers(*, data, col=None, row=None, x, y, hue=None): + g = sns.FacetGrid( + data, + col=col if col is not None and data[col].nunique() > 1 else None, + row=row if row is not None and data[row].nunique() > 1 else None, + margin_titles=True, + sharey="row", + ) + g.map_dataframe( + sns.boxplot, + x=x, + y=y, + hue=hue, + showfliers=True, + ) + + # adapted from https://stackoverflow.com/a/48208266 + g.set_axis_labels(x_var=x, y_var=y) + + +def facet_boxplot_nofliers(*, data, col=None, row=None, x, y, hue=None): + g = sns.FacetGrid( + data, + col=col if col is not None and data[col].nunique() > 1 else None, + row=row if row is not None and data[row].nunique() > 1 else None, + margin_titles=True, + sharey="row", + ) + g.map_dataframe( + sns.boxplot, + x=x, + y=y, + hue=hue, + showfliers=False, + ) + + # adapted from https://stackoverflow.com/a/48208266 + g.set_axis_labels(x_var=x, y_var=y) diff --git a/conduitpylib/viz/_frame_scatter_subsets.py b/conduitpylib/viz/_frame_scatter_subsets.py new file mode 100644 index 0000000000..16c7672e5b --- /dev/null +++ b/conduitpylib/viz/_frame_scatter_subsets.py @@ -0,0 +1,94 @@ +import itertools as it +import typing + +import numpy as np +import matplotlib as mpl +import matplotlib.patches as mpl_patches +import matplotlib.lines as mpl_lines +import pandas as pd + + +def frame_scatter_subsets( + data: pd.DataFrame, + x: str, + y: str, + frame: str, + ax: typing.Optional[mpl.axes.Axes], + frame_order: typing.Optional[typing.Iterable] = None, + palette: typing.Optional[typing.Iterable] = None, + pad_cm: float = 0.5, + log_y: bool = False, + **kwargs, +) -> mpl.axes.Axes: + """Draw rectangles bounding points associated with each value of `frame` + column. + + Extra kwargs are forwarded to Rectangle initializer. + """ + if ax is None: + ax = mpl.pyplot.gca() + + if palette is None: + palette = [None] + + # Calculate expansion in plot space + x_lb, x_ub = ax.get_xlim() + y_lb, y_ub = ax.get_ylim() + fig_inches_x, fig_inches_y = ax.figure.get_size_inches() + + pad_inches = pad_cm / 2.54 + pad_fraction_x = pad_inches / fig_inches_x + pad_fraction_y = pad_inches / fig_inches_y + + x_padding = (x_ub - x_lb) * pad_fraction_x + if log_y: + y_padding = (np.log10(y_ub) - np.log10(y_lb)) * pad_fraction_y + else: + y_padding = (y_ub - y_lb) * pad_fraction_y + + if frame_order is None: + frame_order = data[frame].unique() + linestyles = it.cycle(["-", "--", ":"]) + hues = it.cycle(palette) + for frame_value, linestyle, hue in zip(frame_order, linestyles, hues): + frame_data = data[data[frame] == frame_value] + + frame_min_x, frame_max_x = ( + frame_data[x].min() - x_padding, + frame_data[x].max() + x_padding, + ) + + frame_min_y, frame_max_y = ( + frame_data[y].min(), + frame_data[y].max(), + ) + + if log_y: + frame_min_y, frame_max_y = ( + 10 ** (np.log10(frame_min_y) - y_padding), + 10 ** (np.log10(frame_max_y) + y_padding), + ) + else: + frame_min_y, frame_max_y = ( + frame_min_y - y_padding, + frame_max_y + y_padding, + ) + + height = frame_max_y - frame_min_y + width = frame_max_x - frame_min_x + rect = mpl_patches.Rectangle( + (frame_min_x, frame_min_y), + width, + height, + linewidth=1, + edgecolor=hue, + facecolor="none", + linestyle=linestyle, + alpha=0.4, + lw=3, + **kwargs, + ) + + ax.add_patch(rect) + + return ax diff --git a/conduitpylib/viz/_get_defaults.py b/conduitpylib/viz/_get_defaults.py new file mode 100644 index 0000000000..b0793cca71 --- /dev/null +++ b/conduitpylib/viz/_get_defaults.py @@ -0,0 +1,29 @@ +import typing + + +def get_default_linestyles() -> typing.List[str]: + return [":", "--", "-", "-."] + + +def get_default_palette() -> typing.List[str]: + # alternate palettes: + # ["#648FFF", "#40B0A6"] or ["#40b07f", "#646eff"] + # return ["#5c5cff", "#64e0ff", "#40B0A6"] + # return ["#91f7ff", "#ff82d4", "#91aaff"] + return [ + "#9467bd", + "#bcbd22", + "#17becf", + "#59a14f", + ] # purple, gold, teal, green + + +def get_default_runtime_indicators() -> typing.List[str]: + return [ + # "Num Messages Per Laden Pull", + "Latency, Log10 Updates", + "ms Per Update", + "Msgs Received Per Sent", + "Bunching", + # "Delivery Failure, %", + ] diff --git a/conduitpylib/viz/_get_notomoji_font_path.py b/conduitpylib/viz/_get_notomoji_font_path.py new file mode 100644 index 0000000000..45ff644fc7 --- /dev/null +++ b/conduitpylib/viz/_get_notomoji_font_path.py @@ -0,0 +1,4 @@ +from pathlib import Path + +def get_notomoji_font_path() -> Path: + return Path(__file__).parent / "assets" / "NotoEmoji-Regular.ttf" diff --git a/conduitpylib/viz/_performance_semantics_facetplot.py b/conduitpylib/viz/_performance_semantics_facetplot.py new file mode 100644 index 0000000000..ccf1d3d82f --- /dev/null +++ b/conduitpylib/viz/_performance_semantics_facetplot.py @@ -0,0 +1,127 @@ +import functools +import operator +import typing +from unittest.mock import patch +import warnings + +from frozendict import frozendict +import pandas as pd +import patchworklib as pw + +from ._calc_performance_semantics_axis_lims import ( + calc_performance_semantics_axis_lims, +) +from ._get_defaults import get_default_linestyles, get_default_palette +from ._performance_semantics_kdeplot import performance_semantics_kdeplot +from ._performance_semantics_scatterplot import ( + performance_semantics_scatterplot, +) + + +@patch.dict(pw.param, {"margin": 0.0}) +def performance_semantics_facetplot( + data: pd.DataFrame, + x: str = "Simstep Period Inlet (ns)", + y: str = "Latency Simsteps Inlet", + hue: typing.Optional[str] = None, + hue_order: typing.Optional[typing.List[str]] = None, + title: str = "", + brick_size_graphs: typing.Tuple[float, float] = (0.51, 0.5), + brick_size_legend: typing.Tuple[float, float] = (0.68, 1.2), + kde_kwargs: typing.Dict = frozendict(), + legend_font_name: typing.Optional[str] = None, + linestyles: typing.Optional[typing.List[str]] = None, + palette: typing.Optional[typing.List[str]] = None, + scatter_kwargs: typing.Dict = frozendict(), + sharexy: bool = True, +) -> pw.Brick: + if hue is not None: + if hue_order is None: + hue_order = sorted(data[hue].unique()) + if not set(hue_order) <= set(data[hue].unique()): + raise ValueError("hue_order has hue categories missing from data.") + elif hue_order is not None: + raise ValueError( + "If hue is None, hue_order must also be None.", + ) + + if linestyles is None: + # default linestyles + linestyles = get_default_linestyles() + if hue_order is not None and len(linestyles) < len(hue_order): + warnings.warn("more hue categories than linestyles") + + if palette is None: + # default palette + palette = get_default_palette() + if hue_order is not None and len(palette) < len(hue_order): + warnings.warn("more hue categories than palette colors") + + xmin, xmax, ymin, ymax = calc_performance_semantics_axis_lims( + data=data, + x=x, + y=y, + hue=hue, + ) + xlim = (xmin, xmax) if sharexy else None + ylim = (ymin, ymax) if sharexy else None + + fig_legend = performance_semantics_scatterplot( + data=data, + x=x, + y=y, + hue=hue, + hue_order=hue_order, + legend="only", + legend_font_name=legend_font_name, + linestyles=linestyles, + palette=palette, + title=title, + xlim=xlim, + ylim=ylim, + ) + brick_legend = pw.load_seaborngrid(fig_legend, figsize=brick_size_legend) + + fig_kde = performance_semantics_kdeplot( + **{ + "data": data, + "x": x, + "y": y, + "hue": hue, + "hue_order": hue_order, + "palette": palette, + "xlim": xlim, + "ylim": ylim, + **kde_kwargs, + }, + ) + brick_kde = pw.load_seaborngrid(fig_kde, figsize=brick_size_graphs) + + fig_scatterplots = [ + performance_semantics_scatterplot( + **{ + "data": data[data[hue] == hue_order[i]], + "x": x, + "y": y, + "hue": None, + "background_color": palette[i], + "legend": "hide", + "linestyles": linestyles[i:], + "palette": palette[i:], + "xlim": xlim, + "ylabel": "", + "ylim": ylim, + **scatter_kwargs, + }, + ) + for i in range(len(hue_order)) + ] + bricks_scatterplots = [ + pw.load_seaborngrid(fig_scatterplot, figsize=brick_size_graphs) + for fig_scatterplot in fig_scatterplots + ] + + bricks = [brick_legend, brick_kde, *bricks_scatterplots] + concat = functools.reduce(operator.or_, bricks) + + return concat diff --git a/conduitpylib/viz/_performance_semantics_kdeplot.py b/conduitpylib/viz/_performance_semantics_kdeplot.py new file mode 100644 index 0000000000..ebdffaaa38 --- /dev/null +++ b/conduitpylib/viz/_performance_semantics_kdeplot.py @@ -0,0 +1,226 @@ +import itertools as it +import typing + + +from matplotlib.collections import ( + PathCollection as mpl_PathCollection, +) +from matplotlib import figure as mpl_figure +from matplotlib.font_manager import FontProperties as mpl_FontProperties +from matplotlib.legend_handler import ( + HandlerPathCollection as mpl_HandlerPathCollection, + HandlerLine2D as mpl_HandlerLine2D, +) +import matplotlib.patches as mpl_patches +from matplotlib import pyplot as plt +import numpy as np +import opytional as opyt +import pandas as pd +import seaborn as sns + +from ..utils import seaborn_monkeypatched_kdecache +from ._compact_xaxis_units import compact_xaxis_units +from ._frame_scatter_subsets import frame_scatter_subsets +from . import _get_defaults as cfg +from ._set_performance_semantics_axis_lims import ( + set_performance_semantics_axis_lims, +) + + +@seaborn_monkeypatched_kdecache() +def performance_semantics_kdeplot( + data: pd.DataFrame, + hue: str, + hue_order: typing.Optional[typing.List[str]] = None, + x: str = "Simstep Period Inlet (ns)", + y: str = "Latency Simsteps Inlet", + background_color: typing.Optional[str] = None, + linestyles: typing.Optional[typing.List[str]] = None, + palette: typing.Optional[typing.List[str]] = None, + size_inches: typing.Tuple[float, float] = (3.5, 2.5), + xlim: typing.Optional[typing.Tuple[float, float]] = None, + ylim: typing.Optional[typing.Tuple[float, float]] = None, +) -> mpl_figure.Figure: + assert set(hue_order) >= set(data[hue]) + + if hue_order is None: + hue_order = sorted(data[hue].unique()) + + if linestyles is None: + linestyles = cfg.get_default_linestyles() + + if palette is None: + palette = cfg.get_default_palette() + + assert len(palette) >= len(hue_order) + + jointgrid = sns.JointGrid( + data=data, + x=x, + y=y, + hue=hue, + hue_order=hue_order, + ratio=8, + height=8, + ) + + sns.kdeplot( + ax=jointgrid.ax_joint, + data=data, + x=x, + y=y, + alpha=0.4, + cut=10, + fill=True, + hue=hue, + hue_order=hue_order, + legend=False, + levels=2, + log_scale=(False, True), + palette=palette, + thresh=1e-12, + zorder=101, + ) + + # set linestyles of marginal KDE outlines + # adapted from https://stackoverflow.com/a/70089200 + for path_collection, linestyle in zip( + jointgrid.ax_joint.collections, + linestyles, + ): + # need faux fill for set_kde_lims. set + color = path_collection.get_facecolor() + path_collection.set_facecolor("none") + path_collection.set_edgecolor("white") + path_collection.set_linestyle(linestyle) + path_collection.set_linewidth(2) + + sns.kdeplot( + ax=jointgrid.ax_joint, + data=data, + x=x, + y=y, + alpha=0.4, + cut=10, + fill=True, + hue=hue, + hue_order=hue_order, + legend=False, + levels=2, + log_scale=(False, True), + palette=palette, + thresh=1e-12, + zorder=99, + ) + + sns.kdeplot( + ax=jointgrid.ax_joint, + data=data, + x=x, + y=y, + alpha=1.0, + cut=10, + fill=False, + hue=hue, + hue_order=hue_order, + legend=False, + levels=2, + log_scale=(False, True), + linewidth=4, + linestyles="solid", + palette=palette, + thresh=1e-12, + zorder=100, + ) + + # set up joint plot axes... + # ... annotate latency simsteps inlet values + jointgrid.refline(y=1, ls="-", zorder=-5) + jointgrid.refline(y=2, ls="--", zorder=-5) + # ... set simsteps inlet to log scale + jointgrid.ax_joint.set_yscale("log") + + # ... set axis labels + jointgrid.ax_joint.set_xlabel("Update Rate (ns)") + jointgrid.ax_joint.set_ylabel("Communication Latency (updates)") + + # PLOT MARGINAL DISTRIBUTIONS + ########################################################################### + # plot marginal distributions as kernel density estimates + jointgrid.plot_marginals( + sns.kdeplot, + alpha=0.1 if hue is not None else 1.0, + fill=True, + lw=3, + palette=palette, + ) + + # now that we have marginals, set figure dimensions + jointgrid.fig.set_size_inches(*size_inches) + + # set linestyles of marginal KDE outlines + # https://stackoverflow.com/a/70089200 + for line, linestyle in it.chain( + zip(reversed(jointgrid.ax_marg_x.collections), linestyles), + zip(reversed(jointgrid.ax_marg_y.collections), linestyles), + ): + line.set_linestyle(linestyle) + line.set_alpha(0.4) + + # do manual rugplot because plot_marginals messes up colormapping + for ax, kwarg in ( + (jointgrid.ax_marg_x, {"x": x}), + (jointgrid.ax_marg_y, {"y": y}), + ): + sns.rugplot( + ax=ax, + color=palette[0], + data=data, + hue=hue, + hue_order=hue_order, + palette=palette, + legend=False, + alpha=0.03, + expand_margins=False, + clip_on=False, + height=-0.8, + **kwarg, + ) + + # FINALIZE + ########################################################################### + if background_color is not None: + jointgrid.fig.patch.set_facecolor(background_color) + jointgrid.fig.patch.set_alpha(0.2) + for ax, key in [ + (jointgrid.ax_joint, "bottom"), + (jointgrid.ax_joint, "left"), + (jointgrid.ax_marg_x, "bottom"), + (jointgrid.ax_marg_y, "left"), + ]: + ax.tick_params( + colors="black", + length=6, + width=2, + which="both", + ) + + for ax in jointgrid.ax_marg_x, jointgrid.ax_marg_y: + ax.get_xaxis().set_visible(False) + ax.get_yaxis().set_visible(False) + + set_performance_semantics_axis_lims( + ax=jointgrid.ax_joint, + data=data, + x=x, + y=y, + hue=hue, + ) + if xlim is not None: + jointgrid.ax_joint.set_xlim(xlim) + if ylim is not None: + jointgrid.ax_joint.set_ylim(ylim) + + compact_xaxis_units(ax=jointgrid.ax_joint, base_unit="s") + + return jointgrid diff --git a/conduitpylib/viz/_performance_semantics_scatterplot.py b/conduitpylib/viz/_performance_semantics_scatterplot.py new file mode 100644 index 0000000000..fb6f53b7be --- /dev/null +++ b/conduitpylib/viz/_performance_semantics_scatterplot.py @@ -0,0 +1,480 @@ +import contextlib +from difflib import restore +import itertools as it +import sys +import typing +import warnings + +from frozendict import frozendict +from matplotlib.collections import ( + PathCollection as mpl_PathCollection, +) +from matplotlib.figure import Figure as mpl_Figure +from matplotlib.font_manager import FontProperties as mpl_FontProperties +from matplotlib.legend_handler import ( + HandlerPathCollection as mpl_HandlerPathCollection, + HandlerLine2D as mpl_HandlerLine2D, +) +import matplotlib.lines as mpl_lines +import matplotlib.patches as mpl_patches +import matplotlib.patheffects as mpl_patheffects +from matplotlib import pyplot as plt +from matplotlib import rc as mpl_rc +import numpy as np +import opytional as opyt +import pandas as pd +import seaborn as sns + +from ..utils import ( + open_dummy_axis, + seaborn_monkeypatched_kdecache, +) +from ._compact_xaxis_units import compact_xaxis_units +from ._draw_edge_markers import draw_edge_markers +from ._DrawBatched import DrawBatched +from ._get_defaults import get_default_linestyles, get_default_palette +from ._set_performance_semantics_axis_lims import ( + set_performance_semantics_axis_lims, +) +from ._frame_scatter_subsets import frame_scatter_subsets + + +@seaborn_monkeypatched_kdecache() +def performance_semantics_scatterplot( + data: pd.DataFrame, + hue: typing.Optional[str] = None, + hue_order: typing.Optional[typing.List[str]] = None, + heat: str = "% Msgs Lost", + heat_norm: typing.Optional[typing.Tuple[float, float]] = (0, 100), + x: str = "Simstep Period Inlet (ns)", + y: str = "Latency Simsteps Inlet", + batch_kwargs: typing.Dict = frozendict(batch_size=sys.maxsize), + background_color: typing.Optional[str] = None, + bunching_smear_alpha: float = 0.2, + bunching_smear_color: str = "green", + edge_marker_kwargs: typing.Dict = frozendict(), + legend: typing.Literal["hide", "only", "show"] = "show", + legend_contents_pad: bool = False, + legend_font_name: typing.Optional[str] = None, + legend_prop: typing.Dict = frozendict(), + linestyles: typing.Optional[typing.List[str]] = None, + outlier_percentile_x: float = 99.0, + palette: typing.Optional[typing.List[str]] = None, + scatter_kwargs: typing.Dict = frozendict(), + show_bunching_smear: bool = True, + size_inches: typing.Tuple[float, float] = (3.5, 2.5), + title: str = "", + xlabel: typing.Optional[str] = None, + xlim: typing.Optional[typing.Tuple[float, float]] = None, + ylabel: typing.Optional[str] = None, + ylim: typing.Optional[typing.Tuple[float, float]] = None, +) -> mpl_Figure: + if hue is not None: + if hue_order is None: + hue_order = sorted(data[hue].unique()) + if not set(hue_order) <= set(data[hue].unique()): + raise ValueError("hue_order has hue categories missing from data.") + elif hue_order is not None: + raise ValueError( + "If hue is None, hue_order must also be None.", + ) + + if linestyles is None: + # default linestyles + linestyles = get_default_linestyles() + if hue_order is not None and len(linestyles) < len(hue_order): + warnings.warn("more hue categories than linestyles") + + if palette is None: + # default palette + palette = get_default_palette() + if hue_order is not None and len(palette) < len(hue_order): + warnings.warn("more hue categories than palette colors") + + jointgrid = sns.JointGrid(data=data, x=x, y=y, hue=hue, ratio=8, height=8) + + if hue is not None and legend != "only": + sns.kdeplot( + ax=jointgrid.ax_joint, + data=data, + x=x, + y=y, + alpha=0.4, + color=palette[0], + cut=10, + fill=True, # remove fill manually below, needed for set_kde_lims + hue=hue, + legend=False, + levels=2, + log_scale=(False, True), + palette=palette, + thresh=1e-12, + zorder=100, + ) + + # set linestyles of marginal KDE outlines + # adapted from https://stackoverflow.com/a/70089200 + for path_collection, linestyle in zip( + jointgrid.ax_joint.collections, + it.cycle(linestyles), + ): + # need faux fill for set_kde_lims. set + color = path_collection.get_facecolor() + path_collection.set_facecolor("none") + path_collection.set_edgecolor(color) + path_collection.set_linestyle(linestyle) + path_collection.set_linewidth(2) + + # PLOT JOINT DISTRIBUTION + ########################################################################### + def scatter_batch(data, zorder): + # PLOT BUNCHING SMEAR ON JOINT DISTRIBUTION + ########################################################################### + # due to unwanted side effects, must come after legend plot + smear_duration_updates = ( + data["Num Messages Per Laden Pull"] / data["Num Messages Per Pull"] + ) - 1 # noqa: fmt + if legend != "only" and show_bunching_smear: + jointgrid.ax_joint.errorbar( + data=data, + x=x, + y=y, + fmt="none", # don't draw dat points + alpha=bunching_smear_alpha, + ecolor=bunching_smear_color, + yerr=np.vstack( + [ + # below-error height + smear_duration_updates / 2, + # above-error height + smear_duration_updates / 2, + ], + ), + # doesn't work, need outer conditional + # visible=(legend != "only"), + # zorder=-5, + # legend=False, + zorder=zorder, + ) + + # PLOT P/S Scatter ON JOINT DISTRIBUTION + ####################################################################### + sns.scatterplot( + ax=jointgrid.ax_joint, + data=data, + x=x, + y=y, + hue=heat, + legend=False, # use dummy below instead + hue_norm=heat_norm, + **{ + "alpha": 0.4, + "markers": "D", + "palette": "flare", + "visible": (legend != "only"), + **scatter_kwargs, + }, + zorder=zorder, + ) + + DrawBatched( + scatter_batch, + data, + **{ + "sort_by": heat, + **batch_kwargs, + }, + )( + data=DrawBatched.DataPlaceholder, + zorder=DrawBatched.ZorderPlaceholder, + ) + + # set up joint plot axes... + # ... annotate latency simsteps inlet values + jointgrid.refline(y=1, ls="-", visible=(legend != "only"), zorder=-5) + jointgrid.refline(y=2, ls="--", visible=(legend != "only"), zorder=-5) + # ... set simsteps inlet to log scale + jointgrid.ax_joint.set_yscale("log") + # ... set axis labels + jointgrid.ax_joint.set_xlabel("Update Rate (ns)") + jointgrid.ax_joint.set_ylabel("Communication Latency (updates)") + + # PLOT MARGINAL DISTRIBUTIONS + ########################################################################### + # plot marginal distributions as kernel density estimates + jointgrid.plot_marginals( + sns.kdeplot, + alpha=0.4 if hue is not None else 1.0, + color=palette[0], + fill=True, + legend=False, + lw=3, + palette=palette, + visible=(legend != "only"), + ) + + # now that we have marginals, set figure dimensions + jointgrid.fig.set_size_inches(*size_inches) + + # set linestyles of marginal KDE outlines + # https://stackoverflow.com/a/70089200 + for line, linestyle in it.chain( + zip(jointgrid.ax_marg_x.collections, it.cycle(linestyles)), + zip(jointgrid.ax_marg_y.collections, it.cycle(linestyles)), + ): + line.set_linestyle(linestyle) + line.set_alpha(0.4) + + # do manual rugplot because plot_marginals messes up colormapping + for ax, kwarg in ( + (jointgrid.ax_marg_x, {"x": x}), + (jointgrid.ax_marg_y, {"y": y}), + ): + sns.rugplot( + ax=ax, + data=data, + hue=heat, + hue_norm=heat_norm, + palette="flare", + legend=False, + alpha=0.03, + expand_margins=False, + clip_on=False, + height=-0.8, + visible=(legend != "only"), + **kwarg, + ) + + # SET UP LEGEND + ########################################################################### + # sns.move_legend( + # jointgrid.ax_joint, + # "upper right", + # bbox_to_anchor=(-0.4, -0.1, 0.1, 1.2), # place legend to left of plot + # frameon=False, + # ) + # manually register category splits into legend... + # ... add quasi-title to legend entry by providing key but no value + + prefix_labels, prefix_handles = [], [] + postfix_labels, postfix_handles = [], [] + empty_patch = mpl_patches.Patch(alpha=0.0, color="white", label="") + + if hue_order is not None: + # prefix_handles.append(empty_patch) + # prefix_labels.append(hue) + # ... then add legend entries manually + for ls, color, label in zip(linestyles, palette, hue_order): + example_patch = mpl_patches.Patch( + facecolor=color, + linestyle=ls, + label=label, + lw=2, + edgecolor=color, + alpha=0.4, + ) + prefix_handles.append(example_patch) + prefix_labels.append(label) + + prefix_handles.append(empty_patch) + prefix_labels.append("") + + # this title is now strung on the end of reference points below + # if heat is not None: + # # don't know why, but have to manually add heat title + # # when scatter style is disabled + # prefix_handles.append(empty_patch) + # prefix_labels.append(heat) + + # add the bunching smear to legend + if show_bunching_smear: + bunching_smear_label = "Bunching Smear" + bunching_smear_handle = mpl_lines.Line2D( + [0], + [0], + color=bunching_smear_color, + # color="white", + # marker="_", + # markersize=20, + # markeredgewidth=2, + # markeredgecolor=bunching_smear_color, + # linestyle=None, + linewidth=4, + label=bunching_smear_label, + path_effects=[ + mpl_patheffects.Stroke(linewidth=6, foreground="w"), + mpl_patheffects.Normal(), + ], + ) + postfix_labels.extend([" ", bunching_smear_label]) + postfix_handles.extend([empty_patch, bunching_smear_handle]) + + with open_dummy_axis() as dummy_ax: + if heat is not None: + # use dummy data to force full legend representation of heat + num_legend_samples = 4 + sns.scatterplot( + ax=dummy_ax, + x=np.full(num_legend_samples, 0), + y=np.full(num_legend_samples, 0), + hue=np.linspace(*heat_norm, num_legend_samples, dtype=int), + hue_norm=heat_norm, + alpha=0.0, + legend="full", + markers="D", + palette="flare", + ) + + ( + handles, + labels, + ) = dummy_ax.get_legend_handles_labels() + + if labels: + first, *rest = map("{} %".format, labels) + labels = [f"{first} Msgs Lost", *rest] + + if legend_contents_pad: + pad_handles = [empty_patch] + pad_labels = [""] + else: + pad_handles = [] + pad_labels = [] + + handles = [ + *pad_handles, + *prefix_handles, + *handles, + *postfix_handles, + *pad_handles, + ] + labels = [ + *pad_labels, + *prefix_labels, + *labels, + *postfix_labels, + *pad_labels, + ] + + # helper function to disable transparency on legend entries + # adapted from https://stackoverflow.com/a/59629242 + def remove_alpha(handle, orig) -> None: + handle.update_from(orig) + handle.set_alpha(1.0) + + # create legend + bbox_to_anchor = ( + (-0.4, -0.15, 0.1, 1.25) # place legend to left of plot + if legend != "only" + else None + ) + legend_ = jointgrid.ax_joint.legend( + handles, + labels, + handler_map={ + mpl_PathCollection: mpl_HandlerPathCollection( + update_func=remove_alpha + ), + plt.Line2D: mpl_HandlerLine2D(update_func=remove_alpha), + }, # apply helper function + borderaxespad=0, + bbox_to_anchor=bbox_to_anchor, + prop=legend_prop, + ) # apply new handles and labels to plot + # legend_._legend_box.set_height(legend_.get_bbox_to_anchor().height) + + # title background adapted from https://stackoverflow.com/a/76103441 + + mpl_rc("text", usetex=False) + thinspace = '"' + legend_.set_title( + f"{thinspace}{title}{thinspace}", + # use custom font with monochrome emojis + # because vanilla matplotlib backend doesn't support color bitmap fonts + prop=mpl_FontProperties(fname=legend_font_name, size=30), + ) + legend_._legend_title_box._text.set_bbox( + {"facecolor": "gray", "alpha": 0.3, "lw": 3}, + ) + legend_._legend_title_box._text.width = 10 + legend_._legend_box.sep = 20 + + if legend == "hide": + legend_.set_visible(False) + + # FINALIZE + ########################################################################### + for ax in jointgrid.ax_marg_x, jointgrid.ax_marg_y: + ax.get_xaxis().set_visible(False) + ax.get_yaxis().set_visible(False) + ax.patch.set_alpha(0.0) + + set_performance_semantics_axis_lims( + ax=jointgrid.ax_joint, + data=data, + x=x, + y=y, + hue=hue, + outlier_percentile_x=outlier_percentile_x, + ) + if xlim is not None: + jointgrid.ax_joint.set_xlim(xlim) + if ylim is not None: + jointgrid.ax_joint.set_ylim(ylim) + + draw_edge_markers(jointgrid.ax_joint, **edge_marker_kwargs) + + if xlabel is not None: + jointgrid.ax_joint.set_xlabel(xlabel) + if ylabel is not None: + jointgrid.ax_joint.set_ylabel(ylabel) + + # patchworklib strips out background colors... + # jointgrid.fig.patch.set_facecolor(background_color) + # jointgrid.fig.patch.set_alpha(0.2) + # ... so just set axis colors instead + for ax, key in [ + (jointgrid.ax_joint, "bottom"), + (jointgrid.ax_joint, "left"), + (jointgrid.ax_marg_x, "bottom"), + (jointgrid.ax_marg_y, "left"), + ]: + spine = ax.spines[key] + spine.set_visible(True) + spine.set_color(opyt.or_value(background_color, "black")) + spine.set(zorder=-100) + if background_color is not None: + spine.set_lw(5) + # 'both' refers to minor and major axes + ax.tick_params( + colors=opyt.or_value(background_color, "black"), + length=6, + width=2, + which="both", + ) + + with contextlib.suppress(AttributeError): + jointgrid.ax_joint.ticklabel_format( + axis="x", style="sci", scilimits=(-2, 3) + ) + with contextlib.suppress(AttributeError): + jointgrid.ax_joint.ticklabel_format( + axis="y", style="sci", scilimits=(-2, 3) + ) + compact_xaxis_units(ax=jointgrid.ax_joint, base_unit="s") + + if legend == "only": + # outfig = plt.figure() + # outax = outfig.add_subplot(111) + # outax.legend() + # outax.legend_ = legend + # plt.close(jointgrid.fig) + # return outfig + jointgrid.ax_joint.set_axis_off() + jointgrid.ax_marg_x.set_axis_off() + jointgrid.ax_marg_y.set_axis_off() + jointgrid.fig.set_figwidth(1) + jointgrid.fig.tight_layout() + return jointgrid + else: + return jointgrid diff --git a/conduitpylib/viz/_quantile_regplot.py b/conduitpylib/viz/_quantile_regplot.py new file mode 100644 index 0000000000..a87f3986a2 --- /dev/null +++ b/conduitpylib/viz/_quantile_regplot.py @@ -0,0 +1,44 @@ +from matplotlib import pyplot as plt +import numpy as np +import seaborn as sns +import statsmodels.formula.api as smf + + +def quantile_regplot(fit_reg=True, color=None, *args, **kwargs): + x, y, data = kwargs["x"], kwargs["y"], kwargs["data"] + sns.regplot( + *args, + **kwargs, + fit_reg=False, + color=color, + ) + + if fit_reg: + model = smf.quantreg(f"Q('{y}') ~ Q('{x}')", data) + res = model.fit(q=0.5) + m = res.params[f"Q('{x}')"] + b = res.params["Intercept"] + + m_ci = res.conf_int().loc[f"Q('{x}')"].tolist() + b_ci = res.conf_int().loc["Intercept"].tolist() + + center_x = np.mean([data[x].min(), data[x].max()]) + center_y = m * center_x + b + + xs = sorted(set(data[x]) | {center_x}) + ys = [m * x_ + b for x_ in xs] + y1 = [min(m_ * (x_ - center_x) + center_y for m_ in m_ci) for x_ in xs] + y2 = [max(m_ * (x_ - center_x) + center_y for m_ in m_ci) for x_ in xs] + + plt.gca().plot( + xs, + ys, + color=color, + ) + plt.gca().fill_between( + xs, + y1, + y2, + alpha=0.2, + color=color, + ) diff --git a/conduitpylib/viz/_runtime_indicator_regplot.py b/conduitpylib/viz/_runtime_indicator_regplot.py new file mode 100644 index 0000000000..cd5b2f4f23 --- /dev/null +++ b/conduitpylib/viz/_runtime_indicator_regplot.py @@ -0,0 +1,64 @@ +import typing + +from matplotlib import patheffects as mpl_pe +from matplotlib import pyplot as plt +import pandas as pd +import seaborn as sns + +from ._apply_symmetric_axes import apply_symmetric_axes + + +def runtime_indicator_regplot( + data: pd.DataFrame, + x: str = "Longitudinal", + y: str = "Snapshot", + color: str = "purple", + ax: typing.Optional[plt.Axes] = None, +) -> plt.Axes: + if ax is None: + ax = plt.gca() + + sns.regplot( + data=data, + x=x, + y=y, + ax=ax, + line_kws={ + "color": "darkorchid", + "zorder": 100, + "lw": 3, + "ls": "--", + "path_effects": [ + mpl_pe.Stroke(linewidth=6, foreground="white"), + mpl_pe.Stroke(linewidth=4, foreground="slateblue"), + ], + }, + scatter_kws={ + "alpha": 0.4, + "s": 10, + "lw": 0.5, + "ec": "gold", + "color": color, + "marker": "D", + }, + ) + sns.kdeplot( + data=data, + x=x, + y=y, + ax=ax, + fill=False, + thresh=1e-6, + cut=10, + color="#fdff52", + lw=0.5, + ) + # ax.set_title(f"{indicator}") + ax.axline((0, 0), slope=1, color="black", linestyle=":", lw=2) + + ax.grid(True) + ax.set_axisbelow(True) + + apply_symmetric_axes(ax) + + return ax diff --git a/conduitpylib/viz/_runtime_indicators_concat_regplot_pairplot.py b/conduitpylib/viz/_runtime_indicators_concat_regplot_pairplot.py new file mode 100644 index 0000000000..160974fbe3 --- /dev/null +++ b/conduitpylib/viz/_runtime_indicators_concat_regplot_pairplot.py @@ -0,0 +1,27 @@ +from unittest.mock import patch + +import patchworklib as pw +import seaborn as sns + + +@patch.dict("patchworklib.param", {"margin": 0}) +def runtime_indicators_concat_regplot_pairplot( + regplot_facetgrid: sns.FacetGrid, + pairplot_grid: sns.PairGrid, +) -> pw.Brick: + f = regplot_facetgrid + g = pairplot_grid + + f.fig.set_size_inches(*g.fig.get_size_inches()) + + for gax, fax in zip(g.axes.flat, f.axes.flat): + # Set the position of the corresponding subplot + fax.set_position(gax.get_position()) + axlim = gax.get_xlim() + fax.set_ylim(*axlim) + fax.set_xlim(*axlim) + + fbrick = pw.load_seaborngrid(f) + gbrick = pw.load_seaborngrid(g) + + return fbrick / gbrick diff --git a/conduitpylib/viz/_runtime_indicators_pairplot.py b/conduitpylib/viz/_runtime_indicators_pairplot.py new file mode 100644 index 0000000000..51ca2cd988 --- /dev/null +++ b/conduitpylib/viz/_runtime_indicators_pairplot.py @@ -0,0 +1,49 @@ +import typing + +import pandas as pd +import seaborn as sns + +from ._get_defaults import get_default_runtime_indicators + + +def runtime_indicators_pairplot( + data: pd.DataFrame, + hue: typing.Optional[str] = None, + hue_order: typing.Optional[str] = None, + indicators: typing.Optional[typing.List[str]] = None, + palette: typing.Optional[typing.List[str]] = None, +) -> sns.PairGrid: + + if palette is None: + palette = sns.color_palette() + + if hue is not None: + if hue_order is None: + hue_order = sorted(data[hue].unique()) + if not set(hue_order) <= set(data[hue].unique()): + raise ValueError("hue_order has hue categories missing from data.") + elif hue_order is not None: + raise ValueError("If hue is None, hue_order must also be None.") + + if indicators is None: + indictors = get_default_runtime_indicators() + + # ideas for enhancement: + # - flip a facetgrid axis so the diagonal is ascending + # - color code axes by indicator + g = sns.PairGrid( + data, + diag_sharey=False, + vars=indictors, + hue=hue, + hue_order=hue_order, + ) + g.map_diag(sns.kdeplot, common_norm=True, palette=palette) + g.map_lower(sns.kdeplot, fill=True, thresh=1e-5, palette=palette) + g.map_upper(sns.scatterplot, palette=palette) + + handles = g._legend_data.values() + labels = g._legend_data.keys() + g.axes.flat[0].legend(handles=handles, labels=labels) + + return g diff --git a/conduitpylib/viz/_runtime_indicators_regplot.py b/conduitpylib/viz/_runtime_indicators_regplot.py new file mode 100644 index 0000000000..230be62ebd --- /dev/null +++ b/conduitpylib/viz/_runtime_indicators_regplot.py @@ -0,0 +1,89 @@ +import typing + +from matplotlib import pyplot as plt +import pandas as pd +import seaborn as sns + +from ._get_defaults import get_default_runtime_indicators +from ._runtime_indicator_regplot import runtime_indicator_regplot + + +def runtime_indicators_regplot( + data: pd.DataFrame, + compare: str = "Instrumentation", + comparison_x: str = "Longitudinal", + comparison_y: str = "Snapshot", + indicators: typing.Optional[typing.List[str]] = None, + palette: typing.Optional[typing.List[str]] = None, +) -> sns.FacetGrid: + if indicators is None: + indicators = get_default_runtime_indicators() + + if palette is None: + palette = sns.color_palette() + + melted = pd.melt( + data, + id_vars=[compare, "Process Instance UUID"], + value_vars=indicators, + var_name="Runtime Indicator", + value_name="Value", + ) + pivoted = pd.pivot( + melted, + index=["Process Instance UUID", "Runtime Indicator"], + columns=compare, + values="Value", + ).reset_index() + + # hack to make a FacetGrid + facetgrid = sns.FacetGrid( + data=pivoted, + col="Runtime Indicator", + col_order=indicators, + sharex=False, + sharey=False, + ) + facetgrid.set_titles(col_template="{col_name}") + + facetgrid.map_dataframe( + runtime_indicator_regplot, + x=comparison_x, + y=comparison_y, + color="purple", + ) + + # add colorcoded axis backgrounds + for i, ax in enumerate(facetgrid.axes.flat): + xmin, xmax = ax.get_xlim() + ymin, ymax = ax.get_ylim() + dx = (xmax - xmin) / 8 + dy = (ymax - ymin) / 6 + if i == 0: + ax.add_patch( + plt.Rectangle( + (xmin - dx, ymin), + dx, + ymax - ymin, + facecolor=palette[0], + alpha=0.3, + clip_on=False, + linewidth=0, + zorder=-999, + ) + ) + + ax.add_patch( + plt.Rectangle( + (xmin, ymin - dy), + xmax - xmin, + dy, + facecolor=palette[1], + alpha=0.3, + clip_on=False, + linewidth=0, + zorder=-999, + ) + ) + + return facetgrid diff --git a/conduitpylib/viz/_set_kde_lims.py b/conduitpylib/viz/_set_kde_lims.py new file mode 100644 index 0000000000..00a4cad4fd --- /dev/null +++ b/conduitpylib/viz/_set_kde_lims.py @@ -0,0 +1,73 @@ +import itertools as it + +from matplotlib import axes as mpl_axes +import numpy as np + + +def _iter_path_collection(path_collection): + yield from path_collection.get_paths()[0].vertices + + +def set_kde_lims( + ax: mpl_axes.Axes, + log_x: bool = False, + log_y: bool = False, + pad_cm: float = 0.0, +) -> None: + fig_inches_x, fig_inches_y = ax.figure.get_size_inches() + pad_inches = pad_cm / 2.54 + pad_fraction_x = pad_inches / fig_inches_x + pad_fraction_y = pad_inches / fig_inches_y + assert 0 <= pad_fraction_x <= 1 and 0 <= pad_fraction_y <= 1 + + assert len(ax.collections) + vertices = lambda: it.chain(*map(_iter_path_collection, ax.collections)) + frame_xmin = min(vertex[0] for vertex in vertices()) + frame_xmax = max(vertex[0] for vertex in vertices()) + frame_ymin = min(vertex[1] for vertex in vertices()) + frame_ymax = max(vertex[1] for vertex in vertices()) + assert frame_xmin <= frame_xmax and frame_ymin <= frame_ymax + + x_lb, x_ub = ax.get_xlim() + if log_x: + x_padding = (np.log10(x_ub) - np.log10(x_lb)) * pad_fraction_x + assert x_padding >= 0 + frame_xmin, frame_xmax = ( + 10 ** (np.log10(frame_xmin) - x_padding), + 10 ** (np.log10(frame_xmax) + x_padding), + ) + assert frame_xmin > 0 + else: + x_padding = (x_ub - x_lb) * pad_fraction_x + assert x_padding >= 0 + frame_xmin -= x_padding + frame_xmax += x_padding + + y_lb, y_ub = ax.get_ylim() + if log_y: + y_padding = (np.log10(y_ub) - np.log10(y_lb)) * pad_fraction_y + assert y_padding >= 0 + frame_ymin, frame_ymax = ( + 10 ** (np.log10(frame_ymin) - y_padding), + 10 ** (np.log10(frame_ymax) + y_padding), + ) + assert frame_ymin > 0 + else: + y_padding = (y_ub - y_lb) * pad_fraction_y + assert y_padding >= 0 + frame_ymin -= y_padding + frame_ymax += (y_padding,) + + # Set the plot limits + assert frame_xmin <= frame_xmax + assert frame_ymin <= frame_ymax + + # don't shrink viewport + ax.set_xlim( + min(frame_xmin, ax.get_xlim()[0]), + max(frame_xmax, ax.get_xlim()[1]), + ) + ax.set_ylim( + min(frame_ymin, ax.get_ylim()[0]), + max(frame_ymax, ax.get_ylim()[1]), + ) diff --git a/conduitpylib/viz/_set_performance_semantics_axis_lims.py b/conduitpylib/viz/_set_performance_semantics_axis_lims.py new file mode 100644 index 0000000000..8c3dd2cc80 --- /dev/null +++ b/conduitpylib/viz/_set_performance_semantics_axis_lims.py @@ -0,0 +1,27 @@ +import typing + +from matplotlib import pyplot as plt +import pandas as pd + +from ._calc_performance_semantics_axis_lims import ( + calc_performance_semantics_axis_lims, +) + + +def set_performance_semantics_axis_lims( + ax: plt.Axes, + data: pd.DataFrame, + x: str = "Simstep Period Inlet (ns)", + y: str = "Latency Simsteps Inlet", + hue: typing.Optional[str] = None, + **kwargs, +) -> None: + xmin, xmax, ymin, ymax = calc_performance_semantics_axis_lims( + data=data, + x=x, + y=y, + hue=hue, + **kwargs, + ) + ax.set_xlim(left=xmin, right=xmax) + ax.set_ylim(bottom=ymin, top=ymax) diff --git a/conduitpylib/viz/_unsplit_regression.py b/conduitpylib/viz/_unsplit_regression.py new file mode 100644 index 0000000000..2b48c8927c --- /dev/null +++ b/conduitpylib/viz/_unsplit_regression.py @@ -0,0 +1,62 @@ +import matplotlib +from matplotlib import pyplot as plt +import seaborn as sns + +from ._quantile_regplot import quantile_regplot + + +def unsplit_regression(*args, regplot, **kwargs): + del kwargs["color"] + regplot( + *args, + **kwargs, + color="black", + fit_reg=False, + ) + regplot( + *args, + **kwargs, + color="purple", + scatter=False, + ) + + # adapted from https://www.scivision.dev/matplotlib-force-integer-labeling-of-axis/ + plt.gca().xaxis.set_major_locator( + matplotlib.ticker.MaxNLocator( + integer=True, + ), + ) + + +def facet_unsplit_regression( + *, data, col=None, row=None, x, y, regression, **kwargs +): + g = sns.FacetGrid( + data, + col=col if col is not None and data[col].nunique() > 1 else None, + row=row if row is not None and data[row].nunique() > 1 else None, + margin_titles=True, + sharey=False, + ) + g.map_dataframe( + unsplit_regression, + regplot={ + "Ordinary Least Squares Regression": quantile_regplot, + "Quantile Regression": quantile_regplot, + }[regression], + x=x, + y=y, + **kwargs, + ) + + # adapted from https://stackoverflow.com/a/48208266 + g.set_axis_labels(x_var=x, y_var=y) + + # adapted from https://stackoverflow.com/a/29814281 + plt.gcf().subplots_adjust(top=0.8) + plt.gcf().suptitle( + { + "Quantile Regression": "Quantile Regression over Medians", + "Ordinary Least Squares Regression": "Quantile Regression over Means", + }[regression], + ) diff --git a/conduitpylib/viz/assets/Noto-LICENSE b/conduitpylib/viz/assets/Noto-LICENSE new file mode 100644 index 0000000000..81286a08df --- /dev/null +++ b/conduitpylib/viz/assets/Noto-LICENSE @@ -0,0 +1,94 @@ +Copyright 2015 Google Inc. + +This Font Software is licensed under the SIL Open Font License, +Version 1.1. + +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font +creation efforts of academic and linguistic communities, and to +provide a free and open framework in which fonts may be shared and +improved in partnership with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply to +any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software +components as distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, +deleting, or substituting -- in part or in whole -- any of the +components of the Original Version, by changing formats or by porting +the Font Software to a new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, +modify, redistribute, and sell modified and unmodified copies of the +Font Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, in +Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the +corresponding Copyright Holder. This restriction only applies to the +primary font name as presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created using +the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/conduitpylib/viz/assets/NotoEmoji-Regular.ttf b/conduitpylib/viz/assets/NotoEmoji-Regular.ttf new file mode 100644 index 0000000000..cf1c701051 Binary files /dev/null and b/conduitpylib/viz/assets/NotoEmoji-Regular.ttf differ diff --git a/conduitpylib/wrangle/__init__.py b/conduitpylib/wrangle/__init__.py new file mode 100644 index 0000000000..21082a7de5 --- /dev/null +++ b/conduitpylib/wrangle/__init__.py @@ -0,0 +1,26 @@ +from ._diff_snapshot_states import diff_snapshot_states +from ._find_treat_idx_mapped_col import find_treat_idx_mapped_col +from ._make_regression_row import make_regression_row +from ._merge_inlet_outlet_data import merge_inlet_outlet_data +from ._retrieve_and_prepare_delta_dataframes import ( + retrieve_and_prepare_delta_dataframes, +) +from ._wrangle_instrumentation_longform import wrangle_instrumentation_longform +from ._wrangle_instrumentation_wideform import wrangle_instrumentation_wideform +from ._wrangle_longitudinal_deltas import wrangle_longitudinal_deltas +from ._wrangle_snapshot_deltas import wrangle_snapshot_deltas +from ._wrangle_world_sums import wrangle_world_sums + + +__all__ = [ + "diff_snapshot_states", + "find_treat_idx_mapped_col", + "make_regression_row", + "merge_inlet_outlet_data", + "retrieve_and_prepare_delta_dataframes", + "wrangle_instrumentation_longform", + "wrangle_instrumentation_wideform", + "wrangle_longitudinal_deltas", + "wrangle_snapshot_deltas", + "wrangle_world_sums", +] diff --git a/conduitpylib/wrangle/_diff_snapshot_states.py b/conduitpylib/wrangle/_diff_snapshot_states.py new file mode 100644 index 0000000000..b4806586f1 --- /dev/null +++ b/conduitpylib/wrangle/_diff_snapshot_states.py @@ -0,0 +1,90 @@ +import typing + +import numpy as np +import pandas as pd + +from ..utils import pickle_cache + + +# this operation is slow, so put it on its own to minimize cache disruption +@pickle_cache() +def diff_snapshot_states( + merged_df: pd.DataFrame, + max_snapshot: int, + treatment_column: typing.Optional[str] = None, +) -> pd.DataFrame: + df_blurry_snapshots = merged_df[ + merged_df["Has Execution Blur"].astype(bool) + # exclude excess, unintended snapshots from runs that took a while to shut down + # (i.e., from at the 6 minute mark and beyond) + & (merged_df["Snapshot"] <= max_snapshot) + ] + + df_snapshot_diffs = df_blurry_snapshots.groupby( + [ + "Process Instance UUID", + "Snapshot", + # subsequent items aren't meaningful to groupby + # but are just included so they pass through untouched + "Async Mode", + "Num Nodes", + "Allocated Tasks Per Node", + "Cpus Per Node", + "Num Processes", + "Log Num Processes", + "Num Simels Per Cpu", + "Replicate", + "proc", + "Hostname", + "Num Inlets", + "Num Outlets", + "Execution Instance UUID", + "Num Threads", + *( + [treatment_column] if treatment_column is not None else [] + ), + ], + as_index=False, + ).aggregate( + { + "Has Execution Blur": "size", # dummy column for assert below + "Num Puts Attempted": np.ptp, + "Num Try Puts Attempted": np.ptp, + "Num Blocking Puts": np.ptp, + "Num Try Puts That Succeeded": np.ptp, + "Num Puts That Succeeded Eventually": np.ptp, + "Num Blocking Puts That Succeeded Immediately": np.ptp, + "Num Puts That Succeeded Immediately": np.ptp, + "Num Puts That Blocked": np.ptp, + "Num Dropped Puts": np.ptp, + "Num Reads Performed": np.ptp, + "Num Reads That Were Fresh": np.ptp, + "Num Reads That Were Stale": np.ptp, + "Num Revisions Pulled": np.ptp, + "Num Try Pulls Attempted": np.ptp, + "Num Blocking Pulls": np.ptp, + "Num Blocking Pulls That Blocked": np.ptp, + "Num Revisions From Try Pulls": np.ptp, + "Num Revisions From Blocking Pulls": np.ptp, + "Num Pulls Attempted": np.ptp, + "Num Pulls That Were Laden Eventually": np.ptp, + "Num Blocking Pulls That Were Laden Immediately": np.ptp, + "Num Blocking Pulls That Were Laden Eventually": np.ptp, + "Num Pulls That Were Laden Immediately": np.ptp, + "Num Try Pulls That Were Laden": np.ptp, + "Num Try Pulls That Were Unladen": np.ptp, + "Net Flux Through Duct": np.ptp, + "Num Round Trip Touches Inlet": np.ptp, + "Num Round Trip Touches Outlet": np.ptp, + # why are these missing? + # 'Row Initial Timepoint (ns) Inlet' : np.ptp, + # 'Row Initial Timepoint (ns) Outlet' : np.ptp, + "Row Final Timepoint (ns) Inlet": np.ptp, + "Row Final Timepoint (ns) Outlet": np.ptp, + "Runtime Seconds Elapsed Inlet": np.mean, + "Runtime Seconds Elapsed Outlet": np.mean, + } + ) + assert {2} == set(df_snapshot_diffs["Has Execution Blur"]) + + return df_snapshot_diffs diff --git a/conduitpylib/wrangle/_find_treat_idx_mapped_col.py b/conduitpylib/wrangle/_find_treat_idx_mapped_col.py new file mode 100644 index 0000000000..146aeb23f9 --- /dev/null +++ b/conduitpylib/wrangle/_find_treat_idx_mapped_col.py @@ -0,0 +1,16 @@ +import pandas as pd + + +def find_treat_idx_mapped_col(df: pd.DataFrame) -> str: + # thanks, chatgpt + regex_pattern = r"(?:\b\d+\s*=\s*.+?\b)(?:\s*\|\s*(?:\b\d+\s*=\s*.+?\b))*" + + # Use a list comprehension to find matching columns + matching_columns = df.columns[df.columns.str.match(regex_pattern)] + + if len(matching_columns) == 1: + return matching_columns[0] + else: + raise ValueError( + f"Too many or too few matching columns: {matching_columns}" + ) diff --git a/conduitpylib/wrangle/_make_regression_row.py b/conduitpylib/wrangle/_make_regression_row.py new file mode 100644 index 0000000000..da724d80de --- /dev/null +++ b/conduitpylib/wrangle/_make_regression_row.py @@ -0,0 +1,132 @@ +import pathlib + +from iterdub import iterdub as ib +from iterpop import iterpop as ip +from keyname import keyname as kn +import numpy as np +from slugify import slugify +import statsmodels.formula.api as smf + + +def make_regression_row( + *, data, independent_variable, dependent_variable, regression, row_filter +): + filtered_data = data[data.apply(eval(row_filter), axis=1)] + + regfun = { + "Quantile Regression over Means": smf.quantreg, + "Quantile Regression over Medians": smf.quantreg, + }[regression] + model = regfun( + f"Q('{dependent_variable}') ~ Q('{independent_variable}')", + filtered_data, + ) + fit_model = model.fit() + + slope = fit_model.params[f"Q('{independent_variable}')"] + intercept = fit_model.params["Intercept"] + + slope_ci_lb, slope_ci_ub = ( + fit_model.conf_int().loc[f"Q('{independent_variable}')"].tolist() + ) + intercept_ci_lb, intercept_ci_ub = ( + fit_model.conf_int().loc["Intercept"].tolist() + ) + + p = fit_model.pvalues.loc[f"Q('{independent_variable}')"] + + # normalize to "control", i.e., lowest num processes observed + effect_size_normalization_data = data[ + data[independent_variable] == data[independent_variable].min() + ][dependent_variable] + effect_size_normalization_factor = { + "Quantile Regression over Means": lambda x: x.mean(), + "Quantile Regression over Medians": lambda x: x.median(), + }[regression](effect_size_normalization_data) + relative_effect_size = slope / effect_size_normalization_factor + relative_effect_size_ci_lb = slope_ci_lb / effect_size_normalization_factor + relative_effect_size_ci_ub = slope_ci_ub / effect_size_normalization_factor + relative_effect_size_ci_width = ( + relative_effect_size_ci_ub - relative_effect_size_ci_lb + ) + + is_significant = p < 0.05 if np.isfinite(p) else None + + res = { + "Independent Variable": independent_variable, + "Dependent Variable": dependent_variable, + "Dependent Variable Slug": slugify(dependent_variable), + "Cpus Per Node": ib.dub(data["Cpus Per Node"]), + "Num Simels Per Cpu": ip.pophomogeneous(data["Num Simels Per Cpu"]), + "Slope Estimate": slope, + "Slope Estimate 95% CI Lower Bound": slope_ci_lb, + "Slope Estimate 95% CI Upper Bound": slope_ci_ub, + "Absolute Effect Size": slope, + "Absolute Effect Size 95% CI Lower Bound": slope_ci_lb, + "Absolute Effect Size 95% CI Upper Bound": slope_ci_ub, + "Absolute Effect Size 95% CI Width": slope_ci_ub - slope_ci_lb, + "Relative Effect Size": relative_effect_size, + "Relative Effect Size 95% CI Lower Bound": relative_effect_size_ci_lb, + "Relative Effect Size 95% CI Upper Bound": relative_effect_size_ci_ub, + "Relative Effect Size 95% CI Width": relative_effect_size_ci_width, + "Intercept Estimate": intercept, + "Intercept Estimate 95% CI Lower Bound": intercept_ci_lb, + "Intercept Estimate 95% CI Upper Bound": intercept_ci_ub, + "R^2": fit_model.rsquared, + "p": fit_model.pvalues.loc[f"Q('{independent_variable}')"], + "Significant?": is_significant, + "Significant Effect Sign": ( + "-" + if is_significant and slope < 0 + else "+" + if is_significant and slope > 0 + else "0" + if is_significant is not None + else None + ), + "n": len(filtered_data), + "Filter": row_filter, + "Num Processes": ib.dub(filtered_data["Num Processes"]), + "Num Processes Prettyprint": ( + "/".join( + filtered_data["Num Processes"] + .sort_values() + .astype(str) + .unique() + ) + ), + "Regression Model": regression, + "Regression Model Slug": slugify(regression), + "Statistic": { + "Quantile Regression over Means": "mean", + "Quantile Regression over Medians": "median", + }[regression], + } + + # dump regression summary to file + summary_filename = kn.pack( + { + **{ + "a": "regression_summary", + "ext": ".txt", + }, + **{ + slugify(k): slugify(str(v)) + for k, v in res.items() + if k + in [ + "Independent Variable", + "Dependent Variable", + "Cpus Per Node", + "Num Simels Per Cpu", + "Regression Model", + ] + }, + } + ) + + pathlib.Path("outplots").mkdir(parents=True, exist_ok=True) + with open(f"outplots/{summary_filename}", "w") as file: + print(fit_model.summary(), file=file) + + return res diff --git a/conduitpylib/wrangle/_merge_inlet_outlet_data.py b/conduitpylib/wrangle/_merge_inlet_outlet_data.py new file mode 100644 index 0000000000..e89094328c --- /dev/null +++ b/conduitpylib/wrangle/_merge_inlet_outlet_data.py @@ -0,0 +1,118 @@ +from keyname import keyname as kn +import numpy as np +import pandas as pd + +from ..utils import consolidate_merge, pickle_cache + + +@pickle_cache() +def merge_inlet_outlet_data( + df_inlet: pd.DataFrame, + df_outlet: pd.DataFrame, +) -> pd.DataFrame: + df = consolidate_merge( + df_inlet, + df_outlet, + on=["Process Instance UUID", "Update"], + suffixes=(" Inlet", " Outlet"), + how="outer", + ) + if all( + df_inlet["Runtime Seconds Elapsed"] + == df_outlet["Runtime Seconds Elapsed"] + ): + df["Runtime Seconds Elapsed Inlet"] = df["Runtime Seconds Elapsed"] + df["Runtime Seconds Elapsed Outlet"] = df["Runtime Seconds Elapsed"] + else: + df["Runtime Seconds Elapsed Inlet"] = df_inlet[ + "Runtime Seconds Elapsed" + ] + df["Runtime Seconds Elapsed Outlet"] = df_outlet[ + "Runtime Seconds Elapsed" + ] + + conv = { + "Num Inlets": "int64", + "Num Outlets": "int64", + "Num Puts Attempted": "int64", + "Num Try Puts Attempted": "int64", + "Num Blocking Puts": "int64", + "Num Try Puts That Succeeded": "int64", + "Num Puts That Succeeded Eventually": "int64", + "Num Blocking Puts That Succeeded Immediately": "int64", + "Num Puts That Succeeded Immediately": "int64", + "Num Puts That Blocked": "int64", + "Num Dropped Puts": "int64", + "Num Round Trip Touches Inlet": "int64", + "Net Flux Through Duct": "int64", + "proc": "int64", + "Snapshot": "int64", + "Has Execution Blur": "bool", + "Replicate": "int64", + "Async Mode": "int64", + "Num Threads": "int64", + "Num Processes": "int64", + "SLURM_NNODES": "int64", + "SLURM_NTASKS": "int64", + "SLURM_CPUS_ON_NODE": "int64", + } + for key, value in conv.items(): + df[key] = df[key].astype(value, errors="ignore") + + try: + df["Hostname"] = df.apply( + lambda row: kn.unpack(row["Source File Inlet"])["_hostname"], + axis=1, + ) + except Exception: + try: + df["Hostname"] = df.apply( + lambda row: kn.unpack(row["Source File Outlet"])["_hostname"], + axis=1, + ) + except Exception: + df["Hostname"] = df["Process Instance UUID"] + + df["Num Nodes"] = df["SLURM_NNODES"] + df["Num Tasks"] = df["SLURM_NTASKS"] + df["Num Simels Per Cpu"] = df["Num Simulation Elements Per Cpu"] + df["Num Cpus"] = df["Num Threads"] * df["Num Processes"] + df["Allocated Tasks Per Node"] = df["Num Tasks"] // df["Num Nodes"] + df["Cpus Per Node"] = df["Allocated Tasks Per Node"] + df["Delivery Time Inlet"] = (df["Num Puts Attempted"] - 1) / df[ + "Num Round Trip Touches Inlet" + ] + df["Delivery Time Outlet"] = (df["Num Pulls Attempted"] - 1) / df[ + "Num Round Trip Touches Outlet" + ] + df["Intermittancy"] = df["Num Pulls That Were Laden Immediately"] / df[ + ["Net Flux Through Duct", "Num Pulls Attempted"] + ].min(axis=1) + df["Inlet-Seconds Elapsed"] = ( + df["Num Inlets"] * df["Runtime Seconds Elapsed Inlet"] + ) + df["Outlet-Seconds Elapsed"] = ( + df["Num Outlets"] * df["Runtime Seconds Elapsed Outlet"] + ) + df["Latency Simsteps Inlet"] = df["Delivery Time Inlet"] + df["Latency Simsteps Outlet"] = df["Delivery Time Inlet"] + df["Simstep Period Inlet (s)"] = ( + df["Inlet-Seconds Elapsed"] / df["Num Puts Attempted"] + ) + df["Simstep Period Outlet (s)"] = ( + df["Outlet-Seconds Elapsed"] / df["Num Pulls Attempted"] + ) + df["Latency Walltime Inlet (s)"] = ( + df["Latency Simsteps Inlet"] * df["Simstep Period Inlet (s)"] + ) + df["Latency Walltime Outlet (s)"] = ( + df["Latency Simsteps Outlet"] * df["Simstep Period Outlet (s)"] + ) + if "Num Processes" in df: + df["Log Num Processes"] = np.log(df["Num Processes"]) / np.log(4) + if "Amount Compute Work" in df: + df["Log Compute Work"] = ( + np.log(np.maximum(df["Amount Compute Work"], 1)) / np.log(64) + ).astype("int64") + + return df diff --git a/conduitpylib/wrangle/_retrieve_and_prepare_delta_dataframes.py b/conduitpylib/wrangle/_retrieve_and_prepare_delta_dataframes.py new file mode 100644 index 0000000000..2224540a5f --- /dev/null +++ b/conduitpylib/wrangle/_retrieve_and_prepare_delta_dataframes.py @@ -0,0 +1,73 @@ +import typing + +from nbmetalog import nbmetalog as nbm +import pandas as pd + +from ._find_treat_idx_mapped_col import find_treat_idx_mapped_col +from ._merge_inlet_outlet_data import merge_inlet_outlet_data +from ._wrangle_longitudinal_deltas import wrangle_longitudinal_deltas +from ._wrangle_snapshot_deltas import wrangle_snapshot_deltas + + +def retrieve_and_prepare_delta_dataframes( + df_inlet_url: str = "https://osf.io/jgpnv/download", + df_outlet_url: str = "https://osf.io/ncdfq/download", + apply: typing.Callable = lambda df: df, + treatment_column: typing.Optional[str] = None, + return_merge_df: bool = False, +) -> typing.Union[pd.DataFrame, typing.Tuple[pd.DataFrame, pd.DataFrame]]: + df_inlet = pd.read_csv( + df_inlet_url, + compression="gzip", + ) + nbm.print_dataframe_summary(*eval(nbm.nvp_expr("df_inlet"))) + + df_outlet = pd.read_csv( + df_outlet_url, + compression="gzip", + ) + nbm.print_dataframe_summary(*eval(nbm.nvp_expr("df_outlet"))) + + if df_inlet["Process Instance UUID"].isna().any(): + print( + f"""df_inlet has { + df_inlet["Process Instance UUID"].isna().sum() + } na rows, dropping them""" + ) + if df_outlet["Process Instance UUID"].isna().any(): + print( + f"""df_outlet has { + df_outlet["Process Instance UUID"].isna().sum() + } na rows, dropping them""" + ) + + df_inlet.dropna(subset=["Process Instance UUID"], inplace=True) + df_outlet.dropna(subset=["Process Instance UUID"], inplace=True) + + merge_df = apply(merge_inlet_outlet_data(df_inlet, df_outlet)) + + res = ( + wrangle_longitudinal_deltas(merge_df), + wrangle_snapshot_deltas(merge_df, treatment_column=treatment_column), + ) + + for df_ in res: + if treatment_column is not None: + treat_idx_mapper = { + val: idx + for idx, val in enumerate(df_[treatment_column].unique()) + } + treat_idx_mapped_title = " | ".join( + f"{idx} = {val}" for val, idx in treat_idx_mapper.items() + ) + df_[treat_idx_mapped_title] = df_.apply( + lambda row: treat_idx_mapper[row[treatment_column]], + axis=1, + ) + + assert find_treat_idx_mapped_col(df_) == treat_idx_mapped_title + + if not return_merge_df: + return res + else: + return (merge_df, res) diff --git a/conduitpylib/wrangle/_wrangle_instrumentation_longform.py b/conduitpylib/wrangle/_wrangle_instrumentation_longform.py new file mode 100644 index 0000000000..594793a8cd --- /dev/null +++ b/conduitpylib/wrangle/_wrangle_instrumentation_longform.py @@ -0,0 +1,19 @@ +import pandas as pd + + +def wrangle_instrumentation_longform( + longitudinal_df: pd.DataFrame, + snapshot_df: pd.DataFrame, + column_name: str = "Instrumentation", + snapshot_moniker: str = "Snapshot", + longitudinal_moniker: str = "Longitudinal", +) -> pd.DataFrame: + snapshot = snapshot_df.copy().groupby("Process Instance UUID").sample(n=1) + snapshot[column_name] = snapshot_moniker + + longitudinal = ( + longitudinal_df.copy().groupby("Process Instance UUID").sample(n=1) + ) + longitudinal[column_name] = longitudinal_moniker + + return pd.concat([snapshot, longitudinal]) diff --git a/conduitpylib/wrangle/_wrangle_instrumentation_wideform.py b/conduitpylib/wrangle/_wrangle_instrumentation_wideform.py new file mode 100644 index 0000000000..4a056135f3 --- /dev/null +++ b/conduitpylib/wrangle/_wrangle_instrumentation_wideform.py @@ -0,0 +1,55 @@ +import pandas as pd + +from ..viz._get_defaults import get_default_runtime_indicators + + +def wrangle_instrumentation_wideform( + longitudinal_df: pd.DataFrame, + snapshot_df: pd.DataFrame, + column_name: str = "Instrumentation", + snapshot_moniker: str = "Snapshot", + longitudinal_moniker: str = "Longitudinal", +) -> pd.DataFrame: + assert not snapshot_df.columns.duplicated().any() + + roll = snapshot_df.copy().groupby("Process Instance UUID").sample(n=1) + roll[column_name] = snapshot_moniker + assert not roll.columns.duplicated().any() + + assert not longitudinal_df.columns.duplicated().any() + fin = longitudinal_df.copy().groupby("Process Instance UUID").sample(n=1) + fin[column_name] = longitudinal_moniker + assert not fin.columns.duplicated().any() + + suffixes = (f" {snapshot_moniker}", f" {longitudinal_moniker}") + cat = pd.merge( + roll, + fin, + how="right", + on=["Process Instance UUID"], + suffixes=suffixes, + ).reset_index(drop=True) + assert not cat.columns.duplicated().any() + + keep_columns = [] + rename_columns = {} + + col_pairs = [ + (c, c.replace(*suffixes)) + for c in cat.columns + if suffixes[0] in c and c.replace(*suffixes) in cat.columns + ] + + for c1, c2 in col_pairs: + assert c1 != c2 + keep_columns.append(c1) + assert len(cat[c1]) == len(cat[c2]) + root = c1.replace(suffixes[0], "") + if ( + cat[c1] != cat[c2] + ).any() or root in get_default_runtime_indicators(): + keep_columns.append(c2) + else: + rename_columns[c1] = root + + return cat[keep_columns].rename(columns=rename_columns).copy() diff --git a/conduitpylib/wrangle/_wrangle_longitudinal_deltas.py b/conduitpylib/wrangle/_wrangle_longitudinal_deltas.py new file mode 100644 index 0000000000..a4eb7cc9e6 --- /dev/null +++ b/conduitpylib/wrangle/_wrangle_longitudinal_deltas.py @@ -0,0 +1,93 @@ +import numpy as np +import pandas as pd + +from ..utils import pickle_cache + + +@pickle_cache() +def wrangle_longitudinal_deltas(merged_df: pd.DataFrame) -> pd.DataFrame: + # https://stackoverflow.com/a/40629420 + df_finalized_observations = merged_df.sort_values( + "Update", ascending=False + ).drop_duplicates(["Process Instance UUID"]) + + df_finalized_observations["Dominance"] = ( + df_finalized_observations["Num Try Puts Attempted"] + / df_finalized_observations["Net Flux Through Duct"] + ) + + df_finalized_observations[ + "Fraction Messages Dropped" + ] = df_finalized_observations["Fraction Try Puts Dropped"] + + df_finalized_observations["Simstep Period Inlet (ns)"] = ( + df_finalized_observations["Simstep Period Inlet (s)"] * 1e9 + ) + + df_finalized_observations["Num Messages Per Pull"] = ( + df_finalized_observations["Net Flux Through Duct"] + / df_finalized_observations["Num Pulls Attempted"] + ) + + df_finalized_observations["Num Messages Per Laden Pull"] = ( + df_finalized_observations["Net Flux Through Duct"] + / df_finalized_observations["Num Pulls That Were Laden Immediately"] + ) + + df_finalized_observations["Bunchiness"] = ( + df_finalized_observations["Num Messages Per Laden Pull"] + / df_finalized_observations["Num Messages Per Pull"] + ) - 1 + + df_finalized_observations[ + "Delivery Clumpiness" + ] = df_finalized_observations["Bunchiness"] + df_finalized_observations["Intermittancy"] = df_finalized_observations[ + "Bunchiness" + ] + df_finalized_observations["Bunching"] = df_finalized_observations[ + "Bunchiness" + ] + + df_finalized_observations["Latency, Log10 Updates"] = np.log10( + df_finalized_observations["Latency Simsteps Inlet"] + ) + + df_finalized_observations["ms Per Update"] = ( + df_finalized_observations["Simstep Period Inlet (ns)"] / 10**6 + ) + + df_finalized_observations["Delivery Failure, %"] = ( + df_finalized_observations["Fraction Messages Dropped"] * 100 + ) + df_finalized_observations["% Msgs Lost"] = df_finalized_observations[ + "Delivery Failure, %" + ] + + df_finalized_observations[ + "Msgs Received Per Sent" + ] = df_finalized_observations["Num Messages Per Pull"] + + df_finalized_observations[ + "Messages Received Per Second" + ] = df_finalized_observations["Net Flux Through Duct"] / ( + df_finalized_observations["Outlet-Seconds Elapsed"] + ) + df_finalized_observations[ + "Messages Sent Per Second" + ] = df_finalized_observations["Num Try Puts Attempted"] / ( + df_finalized_observations["Inlet-Seconds Elapsed"] + ) + + df_finalized_observations[ + "Messages Received Per Second" + ] = df_finalized_observations["Net Flux Through Duct"] / ( + df_finalized_observations["Outlet-Seconds Elapsed"] + ) + df_finalized_observations[ + "Messages Sent Per Second" + ] = df_finalized_observations["Num Try Puts Attempted"] / ( + df_finalized_observations["Inlet-Seconds Elapsed"] + ) + + return df_finalized_observations diff --git a/conduitpylib/wrangle/_wrangle_snapshot_deltas.py b/conduitpylib/wrangle/_wrangle_snapshot_deltas.py new file mode 100644 index 0000000000..b437708a00 --- /dev/null +++ b/conduitpylib/wrangle/_wrangle_snapshot_deltas.py @@ -0,0 +1,168 @@ +import typing + +import numpy as np +import pandas as pd + +from ..utils import pickle_cache + +from ._diff_snapshot_states import diff_snapshot_states + + +@pickle_cache() +def wrangle_snapshot_deltas( + merged_df: pd.DataFrame, + max_snapshot: int = 5, + treatment_column: typing.Optional[str] = None, +) -> pd.DataFrame: + df_snapshot_diffs = diff_snapshot_states( + merged_df, max_snapshot, treatment_column=treatment_column + ) + + df_snapshot_diffs["Fraction Messages Delivered"] = ( + df_snapshot_diffs["Num Try Puts That Succeeded"] + / df_snapshot_diffs["Num Try Puts Attempted"] + ) + df_snapshot_diffs["Delivery Success Rate"] = ( + df_snapshot_diffs["Num Try Puts That Succeeded"] + / df_snapshot_diffs["Num Try Puts Attempted"] + ) + df_snapshot_diffs["Delivery Failure Rate"] = ( + 1 - df_snapshot_diffs["Delivery Success Rate"] + ) + df_snapshot_diffs["Fraction Messages Dropped"] = df_snapshot_diffs[ + "Delivery Failure Rate" + ] + df_snapshot_diffs["Fraction Try Pulls That Were Laden"] = ( + df_snapshot_diffs["Num Try Pulls That Were Laden"] + / df_snapshot_diffs["Num Try Pulls Attempted"] + ) + + df_snapshot_diffs["Round Trip Touches Per Attempted Put"] = ( + df_snapshot_diffs["Num Round Trip Touches Inlet"] + ) / df_snapshot_diffs["Num Try Puts Attempted"] + + df_snapshot_diffs["Round Trip Touches Per Attempted Pull"] = ( + df_snapshot_diffs["Num Round Trip Touches Outlet"] + ) / df_snapshot_diffs["Num Try Pulls Attempted"] + + df_snapshot_diffs["Round Trip Touches Per Runtime Nanosecond"] = ( + df_snapshot_diffs["Num Round Trip Touches Outlet"] + ) / df_snapshot_diffs["Row Final Timepoint (ns) Outlet"] + + df_snapshot_diffs["Latency Simsteps Inlet"] = ( + df_snapshot_diffs["Num Puts Attempted"] + / df_snapshot_diffs["Num Round Trip Touches Inlet"] + ) + df_snapshot_diffs["Latency Simsteps Outlet"] = ( + df_snapshot_diffs["Num Pulls Attempted"] + / df_snapshot_diffs["Num Round Trip Touches Outlet"] + ) + df_snapshot_diffs["Inlet-Nanoseconds Elapsed"] = ( + df_snapshot_diffs["Num Inlets"] + * df_snapshot_diffs["Row Final Timepoint (ns) Inlet"] + ) + df_snapshot_diffs["Outlet-Nanoseconds Elapsed"] = ( + df_snapshot_diffs["Num Outlets"] + * df_snapshot_diffs["Row Final Timepoint (ns) Outlet"] + ) + df_snapshot_diffs["Simsteps Elapsed Inlet"] = ( + df_snapshot_diffs["Num Puts Attempted"] + / df_snapshot_diffs["Num Inlets"] + ) + df_snapshot_diffs["Simsteps Elapsed Outlet"] = ( + df_snapshot_diffs["Num Pulls Attempted"] + / df_snapshot_diffs["Num Outlets"] + ) + df_snapshot_diffs["Simstep Period Inlet (ns)"] = ( + df_snapshot_diffs["Inlet-Nanoseconds Elapsed"] + / df_snapshot_diffs["Num Puts Attempted"] + ) + df_snapshot_diffs["Simstep Period Outlet (ns)"] = ( + df_snapshot_diffs["Outlet-Nanoseconds Elapsed"] + / df_snapshot_diffs["Num Pulls Attempted"] + ) + df_snapshot_diffs["Latency Walltime Inlet (ns)"] = ( + df_snapshot_diffs["Latency Simsteps Inlet"] + * df_snapshot_diffs["Simstep Period Inlet (ns)"] + ) + df_snapshot_diffs["Latency Walltime Outlet (ns)"] = ( + df_snapshot_diffs["Latency Simsteps Outlet"] + * df_snapshot_diffs["Simstep Period Outlet (ns)"] + ) + + df_snapshot_diffs = df_snapshot_diffs.astype( + { + "Num Inlets": "int64", + "Num Outlets": "int64", + "proc": "int64", + "Snapshot": "int64", + "Replicate": "int64", + "Async Mode": "int64", + "Num Threads": "int64", + "Num Processes": "int64", + "Allocated Tasks Per Node": "int64", + "Cpus Per Node": "int64", + "Num Nodes": "int64", + "Num Simels Per Cpu": "int64", + } + ) + + df_snapshot_diffs["Dominance"] = ( + df_snapshot_diffs["Num Try Puts Attempted"] + / df_snapshot_diffs["Net Flux Through Duct"] + ) + + df_snapshot_diffs["Num Messages Per Pull"] = ( + df_snapshot_diffs["Net Flux Through Duct"] + / df_snapshot_diffs["Num Pulls Attempted"] + ) + + df_snapshot_diffs["Num Messages Per Laden Pull"] = ( + df_snapshot_diffs["Net Flux Through Duct"] + / df_snapshot_diffs["Num Pulls That Were Laden Immediately"] + ) + + df_snapshot_diffs["Bunchiness"] = ( + df_snapshot_diffs["Num Messages Per Laden Pull"] + / df_snapshot_diffs["Num Messages Per Pull"] + ) - 1 + df_snapshot_diffs["Delivery Clumpiness"] = df_snapshot_diffs["Bunchiness"] + df_snapshot_diffs["Intermittancy"] = df_snapshot_diffs["Bunchiness"] + df_snapshot_diffs["Bunching"] = df_snapshot_diffs["Bunchiness"] + + df_snapshot_diffs["Latency, Log10 Updates"] = np.log10( + df_snapshot_diffs["Latency Simsteps Inlet"] + ) + + df_snapshot_diffs["ms Per Update"] = ( + df_snapshot_diffs["Simstep Period Inlet (ns)"] / 10**6 + ) + + df_snapshot_diffs["Delivery Failure, %"] = ( + df_snapshot_diffs["Fraction Messages Dropped"] * 100 + ) + df_snapshot_diffs["% Msgs Lost"] = df_snapshot_diffs["Delivery Failure, %"] + + df_snapshot_diffs["Msgs Received Per Sent"] = df_snapshot_diffs[ + "Num Messages Per Pull" + ] + + df_snapshot_diffs["Messages Received Per Second"] = df_snapshot_diffs[ + "Net Flux Through Duct" + ] / (df_snapshot_diffs["Outlet-Nanoseconds Elapsed"] / 10**9) + df_snapshot_diffs["Messages Sent Per Second"] = df_snapshot_diffs[ + "Num Try Puts Attempted" + ] / (df_snapshot_diffs["Inlet-Nanoseconds Elapsed"] / 10**9) + + df_snapshot_diffs["Any Messages Dropped"] = df_snapshot_diffs[ + "Fraction Messages Dropped" + ].astype(bool) + + df_snapshot_diffs["Messages Received Per Second"] = df_snapshot_diffs[ + "Net Flux Through Duct" + ] / (df_snapshot_diffs["Outlet-Nanoseconds Elapsed"] / 10**9) + df_snapshot_diffs["Messages Sent Per Second"] = df_snapshot_diffs[ + "Num Try Puts Attempted" + ] / (df_snapshot_diffs["Inlet-Nanoseconds Elapsed"] / 10**9) + + return df_snapshot_diffs diff --git a/conduitpylib/wrangle/_wrangle_world_sums.py b/conduitpylib/wrangle/_wrangle_world_sums.py new file mode 100644 index 0000000000..236f6a9415 --- /dev/null +++ b/conduitpylib/wrangle/_wrangle_world_sums.py @@ -0,0 +1,95 @@ +import pandas as pd + +from ._find_treat_idx_mapped_col import find_treat_idx_mapped_col + + +def wrangle_world_sums(df_finalized_observations: pd.DataFrame) -> pd.DataFrame: + df_world_sum = df_finalized_observations.groupby( + [ + "Replicate", + "Async Mode", + "Num Processes", + "Num Nodes", + "Num Simels Per Cpu", + "Allocated Tasks Per Node", + "Cpus Per Node", + "Allocation", + find_treat_idx_mapped_col(df_finalized_observations), + ], + as_index=False, + ).sum() + + df_world_sum["Fraction Messages Utilized"] = ( + df_world_sum["Num Reads That Were Fresh"] + / df_world_sum["Num Try Puts Attempted"] + ) + df_world_sum["Fraction Messages Delivered"] = ( + df_world_sum["Num Try Puts That Succeeded"] + / df_world_sum["Num Try Puts Attempted"] + ) + df_world_sum["Delivery Failure Rate"] = ( + 1.0 - df_world_sum["Fraction Messages Delivered"] + ) + df_world_sum["Fraction Messages Dropped"] = df_world_sum[ + "Delivery Failure Rate" + ] + df_world_sum["Fraction Try Pulls That Were Laden"] = ( + df_world_sum["Num Try Pulls That Were Laden"] + / df_world_sum["Num Try Pulls Attempted"] + ) + df_world_sum["Round Trip Touches Per Attempted Pull"] = ( + df_world_sum["Num Round Trip Touches Outlet"] + / df_world_sum["Num Try Pulls Attempted"] + ) + df_world_sum["Round Trip Touches Per Attempted Put"] = ( + df_world_sum["Num Round Trip Touches Inlet"] + / df_world_sum["Num Try Puts Attempted"] + ) + df_world_sum["Num Inflight Messages"] = ( + 2.0 / df_world_sum["Round Trip Touches Per Attempted Put"] - 1 + ) + df_world_sum["Fraction Duct Flux Stepped Through"] = ( + df_world_sum["Num Revisions Pulled"] + / df_world_sum["Net Flux Through Duct"] + ) + df_world_sum["Fraction Duct Flux Jumped Over"] = ( + 1.0 - df_world_sum["Fraction Duct Flux Stepped Through"] + ) + df_world_sum["Round Trip Touches Per Runtime Second"] = ( + df_world_sum["Num Round Trip Touches Inlet"] + / df_world_sum["Runtime Seconds Elapsed Inlet"] + ) + df_world_sum["Latency Simsteps Inlet"] = ( + df_world_sum["Num Puts Attempted"] - 1 + ) / df_world_sum["Num Round Trip Touches Inlet"] + df_world_sum["Latency Simsteps Outlet"] = ( + df_world_sum["Num Pulls Attempted"] - 1 + ) / df_world_sum["Num Round Trip Touches Outlet"] + + df_world_sum["Bunchiness"] = ( + df_world_sum["Num Messages Per Laden Pull"] + / df_world_sum["Num Messages Per Pull"] + ) - 1 + df_world_sum["Delivery Clumpiness"] = df_world_sum["Bunchiness"] + df_world_sum["Intermittancy"] = df_world_sum["Bunchiness"] + df_world_sum["Bunching"] = df_world_sum["Bunchiness"] + + + df_world_sum["Simstep Period Inlet (s)"] = ( + df_world_sum["Inlet-Seconds Elapsed"] + / df_world_sum["Num Puts Attempted"] + ) + df_world_sum["Simstep Period Outlet (s)"] = ( + df_world_sum["Outlet-Seconds Elapsed"] + / df_world_sum["Num Pulls Attempted"] + ) + df_world_sum["Latency Walltime Inlet (s)"] = ( + df_world_sum["Latency Simsteps Inlet"] + * df_world_sum["Simstep Period Inlet (s)"] + ) + df_world_sum["Latency Walltime Outlet (s)"] = ( + df_world_sum["Latency Simsteps Outlet"] + * df_world_sum["Simstep Period Outlet (s)"] + ) + + return df_world_sum diff --git a/third-party/requirements.in b/third-party/requirements.in index 973215cb49..c644a00e70 100644 --- a/third-party/requirements.in +++ b/third-party/requirements.in @@ -1,16 +1,17 @@ git+git://github.com/mmore500/osfclient@9e2e14d#egg=osfclient -keyname==0.4.1 +keyname==0.5.6 +numpy==1.23.5 pandas==1.1.2 frozendict==1.2 -seaborn==0.11.0 -matplotlib==3.3.2 +# 0.13.0 with override of pandas version pin +git+git://github.com/mmore500/seaborn@bb07f1fa8e55a3cbd4c1e5c08c20c9dd014860d8#egg=seaborn +matplotlib==3.5.2 python-slugify==4.0.1 iterpop==0.3.4 exdown==0.7.0 networkx==2.5 -numpy==1.19.5 scipy==1.5.4 -teeplot==0.3.0 +teeplot==0.6.1 nbmetalog==0.2.6 nbconvert==5.6.1 jupyter_client==6.1.12 @@ -23,3 +24,12 @@ iterdub==0.2.0 python-slugify==4.0.1 statsmodels==0.12.2 nose==1.3.7 +git+git://github.com/mmore500/patchworklib@010008a90d859f54efdaec19b250f8e6c978d51a#egg=patchworklib +# unused depndency, loosend version pin for installability +git+git://github.com/mmore500/plotnine@3daf0e81c0405b1eeebb9daf53489f423eefef2f#egg=plotnine +mizani==0.7.3 # unused dependency pinned to force installability +bidict==0.21.4 +opytional==0.1.0 +# prevent true +pip-tools==5.5.0 +pip==20.1 diff --git a/third-party/requirements.txt b/third-party/requirements.txt index e7a2048da6..539fc3ca6d 100644 --- a/third-party/requirements.txt +++ b/third-party/requirements.txt @@ -2,35 +2,42 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile requirements.in +# pip-compile --pip-args='--no-warn-conflicts' requirements.in # attrs==21.2.0 # via jsonschema backcall==0.2.0 # via ipython +bidict==0.21.4 + # via -r requirements.in bleach==4.1.0 # via nbconvert certifi==2020.12.5 - # via - # matplotlib - # requests + # via requests chardet==4.0.0 # via requests click==7.1.2 - # via keyname + # via + # keyname + # pip-tools cycler==0.10.0 # via matplotlib decorator==4.4.2 # via # ipython # networkx + # retry # traitlets defusedxml==0.7.1 # via nbconvert +dill==0.3.7 + # via patchworklib entrypoints==0.3 # via nbconvert exdown==0.7.0 # via -r requirements.in +fonttools==4.44.0 + # via matplotlib frozendict==1.2 # via -r requirements.in hurry.filesize==0.9 @@ -71,7 +78,7 @@ jupyter_client==6.1.12 # via # -r requirements.in # ipykernel -keyname==0.4.1 +keyname==0.5.6 # via # -r requirements.in # nbmetalog @@ -80,13 +87,22 @@ kiwisolver==1.3.1 # via matplotlib markupsafe==2.0.1 # via jinja2 -matplotlib==3.3.2 +matplotlib==3.5.2 # via # -r requirements.in + # mizani + # patchworklib + # plotnine # seaborn # teeplot mistune==0.8.4 # via nbconvert +mizani==0.7.3 + # via + # -r requirements.in + # plotnine +more-itertools==10.1.0 + # via keyname nb-clean==2.0.2 # via -r requirements.in nbconvert==5.6.1 @@ -101,25 +117,37 @@ networkx==2.5 # via -r requirements.in nose==1.3.7 # via -r requirements.in -numpy==1.19.5 +numpy==1.23.5 # via # -r requirements.in # iterdub # matplotlib + # mizani # pandas + # patchworklib # patsy + # plotnine # scipy # seaborn # statsmodels # teeplot +opytional==0.1.0 + # via -r requirements.in git+git://github.com/mmore500/osfclient@9e2e14d#egg=osfclient # via -r requirements.in packaging==21.0 - # via bleach + # via + # bleach + # matplotlib +palettable==3.3.3 + # via mizani pandas==1.1.2 # via # -r requirements.in + # mizani # nbmetalog + # patchworklib + # plotnine # seaborn # statsmodels # teeplot @@ -127,18 +155,30 @@ pandocfilters==1.4.3 # via nbconvert parso==0.8.2 # via jedi +git+git://github.com/mmore500/patchworklib@010008a90d859f54efdaec19b250f8e6c978d51a#egg=patchworklib + # via -r requirements.in patsy==0.5.1 - # via statsmodels + # via + # plotnine + # statsmodels pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython pillow==8.1.0 # via matplotlib +pip-tools==5.5.0 + # via -r requirements.in +git+git://github.com/mmore500/plotnine@3daf0e81c0405b1eeebb9daf53489f423eefef2f#egg=plotnine + # via + # -r requirements.in + # patchworklib prompt-toolkit==3.0.20 # via ipython ptyprocess==0.7.0 # via pexpect +py==1.11.0 + # via retry pygments==2.10.0 # via # ipython @@ -166,13 +206,17 @@ pyzmq==22.2.1 # via jupyter-client requests==2.25.1 # via osfclient +retry==0.9.2 + # via keyname scipy==1.5.4 # via # -r requirements.in - # seaborn + # plotnine # statsmodels -seaborn==0.11.0 - # via -r requirements.in +git+git://github.com/mmore500/seaborn@bb07f1fa8e55a3cbd4c1e5c08c20c9dd014860d8#egg=seaborn + # via + # -r requirements.in + # patchworklib six==1.15.0 # via # bleach @@ -183,8 +227,10 @@ six==1.15.0 # python-dateutil # traitlets statsmodels==0.12.2 - # via -r requirements.in -teeplot==0.3.0 + # via + # -r requirements.in + # plotnine +teeplot==0.6.1 # via -r requirements.in testpath==0.5.0 # via nbconvert @@ -213,4 +259,5 @@ webencodings==0.5.1 # via bleach # The following packages are considered to be unsafe in a requirements file: +# pip # setuptools