From 3f9dddbcee177fe5b3cc4ab678f00df9fa2cd667 Mon Sep 17 00:00:00 2001 From: franz haas Date: Mon, 2 Feb 2026 19:28:55 +0100 Subject: [PATCH 1/4] - added plugins for ipc and parquet --- argos/repo/registry.py | 10 ++++++++++ argos/repo/rtiplugins/pandasio.py | 18 ++++++++++++++++-- pyproject.toml | 1 + 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/argos/repo/registry.py b/argos/repo/registry.py index 7f8b05d..49de9a2 100644 --- a/argos/repo/registry.py +++ b/argos/repo/registry.py @@ -211,6 +211,16 @@ def getDefaultItems(self): iconColor=ICON_COLOR_PANDAS, globs='*.csv'), + RtiRegItem('Pandas parquet file', + 'argos.repo.rtiplugins.pandasio.PandasParquetFileRti', + iconColor=ICON_COLOR_PANDAS, + globs='*.parquet'), + + RtiRegItem('Pandas ipc file', + 'argos.repo.rtiplugins.pandasio.PandasIpcFileRti', + iconColor=ICON_COLOR_PANDAS, + globs='*.ipc'), + RtiRegItem('NumPy binary file', 'argos.repo.rtiplugins.numpyio.NumpyBinaryFileRti', iconColor=ICON_COLOR_NUMPY, diff --git a/argos/repo/rtiplugins/pandasio.py b/argos/repo/rtiplugins/pandasio.py index 6950d9d..82b816a 100644 --- a/argos/repo/rtiplugins/pandasio.py +++ b/argos/repo/rtiplugins/pandasio.py @@ -333,16 +333,30 @@ def hasChildren(self): def _openResources(self): - """ Uses pandas.read_cs to open the underlying file + """ Uses pandas.read_csv to open the underlying file """ self._ndFrame = pd.read_csv(self._fileName, comment='#') - def _closeResources(self): """ Closes the underlying resources """ self._ndFrame = None +class PandasParquetFileRti(PandasCsvFileRti): + """ Reads a parquet into a Pandas DataFrame. + """ + def _openResources(self): + """ Uses pandas.read_parquet to open the underlying file + """ + self._ndFrame = pd.read_parquet(self._fileName) + +class PandasIpcFileRti(PandasCsvFileRti): + """ Reads a ipc into a Pandas DataFrame.*.csv;*.parquet; + """ + def _openResources(self): + """ Uses pandas.read_ipc to open the underlying file + """ + self._ndFrame = pd.read_ipc(self._fileName) class PandasHdfFileRti(BaseRti): diff --git a/pyproject.toml b/pyproject.toml index f8b35fd..b5c0c86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ all-formats = [ "pillow", "scipy", "pandas", + "pyarrow", "exdir", ] From 700ea1cc87edb358a4d39a5c30fc2d1afecffc6e Mon Sep 17 00:00:00 2001 From: franz haas Date: Fri, 6 Feb 2026 19:53:55 +0100 Subject: [PATCH 2/4] pandas... not polars... sry --- argos/repo/rtiplugins/pandasio.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/argos/repo/rtiplugins/pandasio.py b/argos/repo/rtiplugins/pandasio.py index 82b816a..1386083 100644 --- a/argos/repo/rtiplugins/pandasio.py +++ b/argos/repo/rtiplugins/pandasio.py @@ -356,8 +356,7 @@ class PandasIpcFileRti(PandasCsvFileRti): def _openResources(self): """ Uses pandas.read_ipc to open the underlying file """ - self._ndFrame = pd.read_ipc(self._fileName) - + self._ndFrame = pd.read_feather(self._fileName) class PandasHdfFileRti(BaseRti): """ Reads Pandas data stored in a HDF-5 file. From 4b40f62e8ecc1969520f0532addf1a1363cf0aae Mon Sep 17 00:00:00 2001 From: franz haas Date: Fri, 6 Feb 2026 19:55:15 +0100 Subject: [PATCH 3/4] added test data generation --- tests/create_test_data/binTables/binTables.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tests/create_test_data/binTables/binTables.py diff --git a/tests/create_test_data/binTables/binTables.py b/tests/create_test_data/binTables/binTables.py new file mode 100644 index 0000000..5e43153 --- /dev/null +++ b/tests/create_test_data/binTables/binTables.py @@ -0,0 +1,18 @@ +import polars as pl + +def main(): + df = pl.DataFrame({ + "a": [1, 2, 3], + "b": ["x", "y", "z"], + "d": [1.0, 2.0, 3.0], + }) + df.write_parquet("example1.parquet") + df.write_ipc("example1.ipc") + df = df.with_columns(pl.col("b").cast(pl.Categorical)) + df.write_parquet("example2.parquet") + df.write_ipc("example2.ipc") + +if __name__ == "__main__": + main() + + \ No newline at end of file From c834560d556fb482ec2c2ba6e8060473db1e8ffb Mon Sep 17 00:00:00 2001 From: franz haas Date: Sat, 14 Feb 2026 15:16:19 +0100 Subject: [PATCH 4/4] improved by allwing to work with 128bit integers --- argos/repo/rtiplugins/pandasio.py | 5 +++-- pyproject.toml | 1 + tests/create_test_data/binTables/binTables.py | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/argos/repo/rtiplugins/pandasio.py b/argos/repo/rtiplugins/pandasio.py index 1386083..6946b09 100644 --- a/argos/repo/rtiplugins/pandasio.py +++ b/argos/repo/rtiplugins/pandasio.py @@ -24,6 +24,7 @@ import logging import numpy as np import pandas as pd +import polars as pl from pandas.core.generic import NDFrame @@ -348,7 +349,7 @@ class PandasParquetFileRti(PandasCsvFileRti): def _openResources(self): """ Uses pandas.read_parquet to open the underlying file """ - self._ndFrame = pd.read_parquet(self._fileName) + self._ndFrame = pl.read_parquet(self._fileName).with_columns(pl.col(pl.Int128).cast(pl.Utf8), pl.col(pl.UInt128).cast(pl.Utf8)).to_pandas() class PandasIpcFileRti(PandasCsvFileRti): """ Reads a ipc into a Pandas DataFrame.*.csv;*.parquet; @@ -356,7 +357,7 @@ class PandasIpcFileRti(PandasCsvFileRti): def _openResources(self): """ Uses pandas.read_ipc to open the underlying file """ - self._ndFrame = pd.read_feather(self._fileName) + self._ndFrame = pl.read_ipc(self._fileName).with_columns(pl.col(pl.Int128).cast(pl.Utf8), pl.col(pl.UInt128).cast(pl.Utf8)).to_pandas() class PandasHdfFileRti(BaseRti): """ Reads Pandas data stored in a HDF-5 file. diff --git a/pyproject.toml b/pyproject.toml index b5c0c86..688248d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ dependencies = [ "pyqtgraph >= 0.12.4", # Otherwise LinePlot gives The truth value of an array with more than one element is ambiguous. "pgcolorbar >= 1.1.3", "cmlib >= 1.1.3", # Needed, even if no plugins are installed. + "polars>=1.8.2", ] [project.optional-dependencies] diff --git a/tests/create_test_data/binTables/binTables.py b/tests/create_test_data/binTables/binTables.py index 5e43153..366df2e 100644 --- a/tests/create_test_data/binTables/binTables.py +++ b/tests/create_test_data/binTables/binTables.py @@ -11,6 +11,9 @@ def main(): df = df.with_columns(pl.col("b").cast(pl.Categorical)) df.write_parquet("example2.parquet") df.write_ipc("example2.ipc") + df = df.with_columns(pl.col("a").cast(pl.Int128)) + df.write_parquet("example3.parquet") + df.write_ipc("example3.ipc") if __name__ == "__main__": main()