diff --git a/argos/repo/registry.py b/argos/repo/registry.py index e44fa98..082feb3 100644 --- a/argos/repo/registry.py +++ b/argos/repo/registry.py @@ -211,6 +211,16 @@ def getDefaultItems(self): iconColor=ICON_COLOR_PANDAS, globs='*.csv'), + RtiRegItem('Pandas parquet file', + 'argos.repo.rtiplugins.pandasio.PandasParquetFileRti', + iconColor=ICON_COLOR_PANDAS, + globs='*.parquet'), + + RtiRegItem('Pandas ipc file', + 'argos.repo.rtiplugins.pandasio.PandasIpcFileRti', + iconColor=ICON_COLOR_PANDAS, + globs='*.ipc'), + RtiRegItem('NumPy binary file', 'argos.repo.rtiplugins.numpyio.NumpyBinaryFileRti', iconColor=ICON_COLOR_NUMPY, diff --git a/argos/repo/rtiplugins/pandasio.py b/argos/repo/rtiplugins/pandasio.py index bb4e369..e9fe685 100644 --- a/argos/repo/rtiplugins/pandasio.py +++ b/argos/repo/rtiplugins/pandasio.py @@ -24,6 +24,7 @@ import logging import numpy as np import pandas as pd +import polars as pl from pandas.core.generic import NDFrame @@ -335,17 +336,30 @@ def hasChildren(self): def _openResources(self): - """ Uses pandas.read_cs to open the underlying file + """ Uses pandas.read_csv to open the underlying file """ self._ndFrame = pd.read_csv(self._fileName, comment='#') - def _closeResources(self): """ Closes the underlying resources """ self._ndFrame = None +class PandasParquetFileRti(PandasCsvFileRti): + """ Reads a parquet into a Pandas DataFrame. + """ + def _openResources(self): + """ Uses pandas.read_parquet to open the underlying file + """ + self._ndFrame = pl.read_parquet(self._fileName).with_columns(pl.col(pl.Int128).cast(pl.Utf8), pl.col(pl.UInt128).cast(pl.Utf8)).to_pandas() +class PandasIpcFileRti(PandasCsvFileRti): + """ Reads a ipc into a Pandas DataFrame.*.csv;*.parquet; + """ + def _openResources(self): + """ Uses pandas.read_ipc to open the underlying file + """ + self._ndFrame = pl.read_ipc(self._fileName).with_columns(pl.col(pl.Int128).cast(pl.Utf8), pl.col(pl.UInt128).cast(pl.Utf8)).to_pandas() class PandasHdfFileRti(BaseRti): """ Reads Pandas data stored in a HDF-5 file. diff --git a/pyproject.toml b/pyproject.toml index 361ba96..21d6587 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ dependencies = [ "pyqtgraph >= 0.12.4", # Otherwise LinePlot gives The truth value of an array with more than one element is ambiguous. "pgcolorbar >= 1.1.3", "cmlib >= 1.1.3", # Needed, even if no plugins are installed. + "polars>=1.8.2", ] [project.optional-dependencies] @@ -78,6 +79,7 @@ all-formats = [ "pillow", "scipy", "pandas", + "pyarrow", "exdir", ] diff --git a/tests/create_test_data/binTables/binTables.py b/tests/create_test_data/binTables/binTables.py new file mode 100644 index 0000000..366df2e --- /dev/null +++ b/tests/create_test_data/binTables/binTables.py @@ -0,0 +1,21 @@ +import polars as pl + +def main(): + df = pl.DataFrame({ + "a": [1, 2, 3], + "b": ["x", "y", "z"], + "d": [1.0, 2.0, 3.0], + }) + df.write_parquet("example1.parquet") + df.write_ipc("example1.ipc") + df = df.with_columns(pl.col("b").cast(pl.Categorical)) + df.write_parquet("example2.parquet") + df.write_ipc("example2.ipc") + df = df.with_columns(pl.col("a").cast(pl.Int128)) + df.write_parquet("example3.parquet") + df.write_ipc("example3.ipc") + +if __name__ == "__main__": + main() + + \ No newline at end of file