From 35a6a8a74ef0b4c30b5f77310a179bd833089cdd Mon Sep 17 00:00:00 2001 From: Rui Ji Date: Fri, 9 Jun 2023 16:49:17 +0800 Subject: [PATCH 1/3] fix some minor issues on formatting --- python/xorbits/_mars/services/cluster/gather.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/xorbits/_mars/services/cluster/gather.py b/python/xorbits/_mars/services/cluster/gather.py index 02ed3773c..d6191af8f 100644 --- a/python/xorbits/_mars/services/cluster/gather.py +++ b/python/xorbits/_mars/services/cluster/gather.py @@ -171,7 +171,7 @@ def gather_node_resource(band_to_resource: Dict[str, Resource] = None, use_gpu=T if not num_gpu: continue res[f"gpu-{idx}"] = { - "gpu_avail": 1 - gpu_card_stat.gpu_usage, + "gpu_avail": num_gpu - (gpu_card_stat.gpu_usage / 100.0), "gpu_total": num_gpu, "gpu_memory_avail": gpu_card_stat.fb_mem_info.available, "gpu_memory_total": gpu_card_stat.fb_mem_info.total, From ce984da050085d02477f2fc37ba14e4a50db4633 Mon Sep 17 00:00:00 2001 From: Rui Ji Date: Sun, 2 Jul 2023 18:29:55 +0800 Subject: [PATCH 2/3] now will automatically execute on Xorbits's scope when calling rolling and then mean() --- python/xorbits/core/adapter.py | 1 + python/xorbits/pandas/groupby.py | 6 +++++ python/xorbits/pandas/tests/test_groupby.py | 27 +++++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 python/xorbits/pandas/tests/test_groupby.py diff --git a/python/xorbits/core/adapter.py b/python/xorbits/core/adapter.py index 96079458d..705aca3d9 100644 --- a/python/xorbits/core/adapter.py +++ b/python/xorbits/core/adapter.py @@ -62,6 +62,7 @@ from .._mars.dataframe.datastore.to_vineyard import ( DataFrameToVineyardChunk as MarsDataFrameToVineyardChunk, ) +from .._mars.dataframe.groupby.rolling import GroupByRolling as MarsGroupByRolling from .._mars.dataframe.indexing.at import DataFrameAt as MarsDataFrameAt from .._mars.dataframe.indexing.iat import DataFrameIat as MarsDataFrameIat from .._mars.dataframe.indexing.iloc import DataFrameIloc as MarsDataFrameIloc diff --git a/python/xorbits/pandas/groupby.py b/python/xorbits/pandas/groupby.py index c28b94b7b..2bd035f71 100644 --- a/python/xorbits/pandas/groupby.py +++ b/python/xorbits/pandas/groupby.py @@ -15,6 +15,7 @@ import pandas from ..core import DataRef, DataType +from ..core.adapter import MarsGetAttrProxy, MarsGroupByRolling, register_converter from ..core.data import register_cls_to_type from ..core.utils.docstring import attach_module_callable_docstring @@ -37,3 +38,8 @@ class SeriesGroupBy(DataRef): attach_module_callable_docstring( SeriesGroupBy, pandas, pandas.core.groupby.SeriesGroupBy ) + + +@register_converter(from_cls_list=[MarsGroupByRolling]) +class GroupByRolling(MarsGetAttrProxy): + pass diff --git a/python/xorbits/pandas/tests/test_groupby.py b/python/xorbits/pandas/tests/test_groupby.py new file mode 100644 index 000000000..f8d5f6e65 --- /dev/null +++ b/python/xorbits/pandas/tests/test_groupby.py @@ -0,0 +1,27 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pandas as pd + +from ... import pandas as xpd + + +def test_groupby_rolling(setup): + df = pd.DataFrame({"Group": ["A", "A", "B", "B", "B"], "Value": [1, 2, 3, 4, 5]}) + xdf = xpd.DataFrame(df) + + rolling_x_mean = xdf.groupby("Group")["Value"].rolling(window=2).mean().to_pandas() + rolling_mean = df.groupby("Group")["Value"].rolling(window=2).mean() + + pd.testing.assert_series_equal(rolling_x_mean, rolling_mean) From 7265b9a3a965b4e7bf7105ec9c14492d6ffe0ddc Mon Sep 17 00:00:00 2001 From: Rui Ji Date: Sun, 2 Jul 2023 18:29:55 +0800 Subject: [PATCH 3/3] now will automatically execute on Xorbits's scope when calling rolling and then mean() --- python/xorbits/core/adapter.py | 1 + python/xorbits/pandas/groupby.py | 6 +++++ python/xorbits/pandas/tests/test_groupby.py | 27 +++++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 python/xorbits/pandas/tests/test_groupby.py diff --git a/python/xorbits/core/adapter.py b/python/xorbits/core/adapter.py index 96079458d..705aca3d9 100644 --- a/python/xorbits/core/adapter.py +++ b/python/xorbits/core/adapter.py @@ -62,6 +62,7 @@ from .._mars.dataframe.datastore.to_vineyard import ( DataFrameToVineyardChunk as MarsDataFrameToVineyardChunk, ) +from .._mars.dataframe.groupby.rolling import GroupByRolling as MarsGroupByRolling from .._mars.dataframe.indexing.at import DataFrameAt as MarsDataFrameAt from .._mars.dataframe.indexing.iat import DataFrameIat as MarsDataFrameIat from .._mars.dataframe.indexing.iloc import DataFrameIloc as MarsDataFrameIloc diff --git a/python/xorbits/pandas/groupby.py b/python/xorbits/pandas/groupby.py index c28b94b7b..2bd035f71 100644 --- a/python/xorbits/pandas/groupby.py +++ b/python/xorbits/pandas/groupby.py @@ -15,6 +15,7 @@ import pandas from ..core import DataRef, DataType +from ..core.adapter import MarsGetAttrProxy, MarsGroupByRolling, register_converter from ..core.data import register_cls_to_type from ..core.utils.docstring import attach_module_callable_docstring @@ -37,3 +38,8 @@ class SeriesGroupBy(DataRef): attach_module_callable_docstring( SeriesGroupBy, pandas, pandas.core.groupby.SeriesGroupBy ) + + +@register_converter(from_cls_list=[MarsGroupByRolling]) +class GroupByRolling(MarsGetAttrProxy): + pass diff --git a/python/xorbits/pandas/tests/test_groupby.py b/python/xorbits/pandas/tests/test_groupby.py new file mode 100644 index 000000000..f8d5f6e65 --- /dev/null +++ b/python/xorbits/pandas/tests/test_groupby.py @@ -0,0 +1,27 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pandas as pd + +from ... import pandas as xpd + + +def test_groupby_rolling(setup): + df = pd.DataFrame({"Group": ["A", "A", "B", "B", "B"], "Value": [1, 2, 3, 4, 5]}) + xdf = xpd.DataFrame(df) + + rolling_x_mean = xdf.groupby("Group")["Value"].rolling(window=2).mean().to_pandas() + rolling_mean = df.groupby("Group")["Value"].rolling(window=2).mean() + + pd.testing.assert_series_equal(rolling_x_mean, rolling_mean)