Skip to content

Commit ce024d9

Browse files
authored
perf: align dask chunks with stored chunks and keep conversions lazy (#527)
* perf: align dask chunks with stored chunks and keep conversions lazy - Open existing cutouts with chunks="auto" by default (also after prepare), so dask chunks align with the chunks stored in the NetCDF file. Before, each stored chunk was decompressed many times. - Interpolate the CSP efficiency per chunk with RegularGridInterpolator. - Keep pv lazy for trigon_model="other" and tracking="tilted_horizontal". - Vectorise line_rating over all (line, cell) pairs instead of one dask task per line. * add PR link to release notes * simplify line_rating aggregation and fix mypy in test Keep rows, cols = I.nonzero() from master and use np.unique for the first pair of each line instead of CSR internals. Pass the panel config as a plain dict in the laziness test.
1 parent 0d4ff8f commit ce024d9

8 files changed

Lines changed: 224 additions & 32 deletions

File tree

‎RELEASE_NOTES.rst‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,27 @@ Upcoming Release
1717
To use the features already you have to install the ``master`` branch, e.g.
1818
``pip install git+https://github.com/pypsa/atlite``.
1919

20+
**Performance**
21+
22+
* Open existing cutouts with ``chunks="auto"`` by default instead of ``{"time": 100}``,
23+
also after ``Cutout.prepare``. Dask chunks are now aligned with the chunks stored in
24+
the NetCDF file. Before, dask chunks that split the stored chunks made the compressed
25+
data decompress many times. This makes all conversions much faster for cutouts with
26+
large stored chunks. An explicit ``chunks`` argument is
27+
still respected. An explicit ``chunks="auto"`` now also
28+
respects the stored chunks. Larger chunks can increase peak memory. To reduce it,
29+
pass ``chunks``
30+
(https://github.com/PyPSA/atlite/pull/527).
31+
* Keep ``csp`` lazy and chunked. The efficiency interpolation previously merged all time
32+
steps into one chunk and added the solar position as four extra coordinates
33+
(https://github.com/PyPSA/atlite/pull/527).
34+
* Keep ``pv`` and ``irradiation`` lazy for ``trigon_model="other"`` and
35+
``tracking="tilted_horizontal"``. Both computed intermediate results eagerly before.
36+
The check for negative diffuse irradiation now only runs at log level ``DEBUG``
37+
(https://github.com/PyPSA/atlite/pull/527).
38+
* Compute ``line_rating`` for all lines in one vectorised dask operation instead of one
39+
task per line. Before, each line read and decompressed the cutout data again
40+
(https://github.com/PyPSA/atlite/pull/527).
2041
**Documentation**
2142

2243
* Fix broken links in docs (https://github.com/PyPSA/atlite/pull/537).

‎atlite/convert.py‎

Lines changed: 47 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,10 @@
1616
import numpy as np
1717
import pandas as pd
1818
import xarray as xr
19-
from dask import compute, delayed
2019
from dask.array import absolute, arccos, cos, maximum, mod, radians, sin, sqrt
2120
from dask.diagnostics import ProgressBar
2221
from numpy import pi
22+
from scipy.interpolate import RegularGridInterpolator
2323
from scipy.sparse import csr_matrix
2424

2525
from atlite import hydro as hydrom
@@ -1415,8 +1415,18 @@ def convert_csp(ds, installation):
14151415
raise ValueError(f'Unknown CSP technology option "{tech}".')
14161416

14171417
# Determine solar_position dependend efficiency for each grid cell and time step
1418-
efficiency = installation["efficiency"].interp(
1419-
altitude=solar_position["altitude"], azimuth=solar_position["azimuth"]
1418+
eff = installation["efficiency"].transpose("altitude", "azimuth")
1419+
interp = RegularGridInterpolator(
1420+
(eff["altitude"].values, eff["azimuth"].values),
1421+
eff.values,
1422+
bounds_error=False,
1423+
)
1424+
efficiency = xr.apply_ufunc(
1425+
lambda alt, az: interp((alt, az)),
1426+
solar_position["altitude"],
1427+
solar_position["azimuth"],
1428+
dask="parallelized",
1429+
output_dtypes=[eff.dtype],
14201430
)
14211431

14221432
# Thermal system output
@@ -1732,8 +1742,8 @@ def convert_line_rating(
17321742
-------
17331743
xr.DataArray or numpy.ndarray
17341744
Maximum current per time step in ampere. When *ds* is an
1735-
``xr.Dataset`` the result is aggregated across intersecting cells
1736-
via ``.min("spatial")``.
1745+
``xr.Dataset`` with a ``spatial`` dimension, the result is aggregated
1746+
across intersecting cells via ``.min("spatial")``.
17371747
"""
17381748
Ta = ds["temperature"]
17391749
Tfilm = (Ta + Ts) / 2
@@ -1791,7 +1801,9 @@ def convert_line_rating(
17911801
qs = alpha * Q * A * sin(Phi_s)
17921802

17931803
Imax = sqrt((qc + qr - qs) / R)
1794-
return Imax.min("spatial") if isinstance(Imax, xr.DataArray) else Imax
1804+
if isinstance(Imax, xr.DataArray) and "spatial" in Imax.dims:
1805+
return Imax.min("spatial")
1806+
return Imax
17951807

17961808

17971809
def line_azimuth_degrees(shape: Any) -> float:
@@ -1900,7 +1912,8 @@ def line_rating(
19001912
if not isinstance(shapes, gpd.GeoSeries):
19011913
shapes = gpd.GeoSeries(shapes).rename_axis("dim_0")
19021914

1903-
I = cutout.intersectionmatrix(shapes)
1915+
# CSR format orders the (line, cell) pairs by line
1916+
I = cutout.intersectionmatrix(shapes).tocsr()
19041917
rows, cols = I.nonzero()
19051918

19061919
data = cutout.data.stack(spatial=["y", "x"])
@@ -1918,19 +1931,31 @@ def line_rating(
19181931
assert df.notnull().all().all(), "Nan values encountered."
19191932
assert df.columns.equals(pd.Index(["psi", "R", "D", "Ts", "epsilon", "alpha"]))
19201933

1921-
dummy = xr.DataArray(np.full(len(data.time), np.nan), coords=(data.time,))
1922-
res = []
1923-
for i in range(len(df)):
1924-
cells_i = cols[rows == i]
1925-
if cells_i.size:
1926-
ds = data.isel(spatial=cells_i)
1927-
res.append(delayed(convert_line_rating)(ds, *df.iloc[i].values))
1928-
else:
1929-
res.append(dummy)
1930-
if show_progress:
1931-
with ProgressBar(minimum=2):
1932-
(computed,) = compute(res, **dask_kwargs)
1933-
else:
1934-
(computed,) = compute(res, **dask_kwargs)
1934+
# Compute the rating of each (line, cell) pair
1935+
pairs = data.isel(spatial=cols).rename(spatial="pair").chunk(pair=-1, time=100)
1936+
pair_params = {
1937+
k: xr.DataArray(v.values, dims="pair") for k, v in df.iloc[rows].items()
1938+
}
1939+
Imax = convert_line_rating(pairs, **pair_params)
1940+
1941+
# Take the minimum over the cells of each line, ignoring NaN like ``.min()``.
1942+
# The pairs of each line are contiguous and start at ``starts``.
1943+
lines, starts = np.unique(rows, return_index=True)
1944+
Imax = xr.apply_ufunc(
1945+
lambda a: np.fmin.reduceat(a, starts, axis=-1),
1946+
Imax,
1947+
input_core_dims=[["pair"]],
1948+
output_core_dims=[["line"]],
1949+
dask="parallelized",
1950+
output_dtypes=[Imax.dtype],
1951+
dask_gufunc_kwargs={
1952+
"output_sizes": {"line": len(lines)},
1953+
"allow_rechunk": False,
1954+
},
1955+
)
19351956

1936-
return xr.concat(computed, dim=df.index).assign_attrs(units="A")
1957+
# Lines without intersecting cells get NaN
1958+
Imax = Imax.assign_coords(line=lines).reindex(line=range(len(df)))
1959+
dim = df.index.name or "concat_dim"
1960+
Imax = Imax.rename(line=dim).assign_coords({dim: df.index}).transpose(dim, "time")
1961+
return maybe_progressbar(Imax, show_progress, **dask_kwargs).assign_attrs(units="A")

‎atlite/cutout.py‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -125,9 +125,11 @@ def __init__(self, path: PathLike, **cutoutparams: Any) -> None:
125125
dt : str, optional
126126
Frequency of the time coordinate. The default is 'h'. Valid are all
127127
pandas offset aliases.
128-
chunks : dict
128+
chunks : dict or str
129129
Chunks when opening NetCDF files. For cutout preparation it is recommended
130-
to chunk only along the time dimension. Defaults to {'time': 100}
130+
to chunk only along the time dimension. Defaults to "auto" for existing
131+
cutouts, which aligns the chunks with the chunks stored in the file, and
132+
to {'time': 100} for new cutouts.
131133
data : xr.Dataset
132134
User provided cutout data. Save the cutout using `Cutout.to_file()`
133135
afterwards.
@@ -162,7 +164,7 @@ def __init__(self, path: PathLike, **cutoutparams: Any) -> None:
162164
163165
"""
164166
path = Path(path).with_suffix(".nc")
165-
chunks = cutoutparams.pop("chunks", {"time": 100})
167+
chunks = cutoutparams.pop("chunks", "auto" if path.is_file() else {"time": 100})
166168
if isinstance(chunks, dict):
167169
storable_chunks = {f"chunksize_{k}": v for k, v in (chunks or {}).items()}
168170
else:
@@ -171,8 +173,7 @@ def __init__(self, path: PathLike, **cutoutparams: Any) -> None:
171173
# Three cases. First, cutout exists -> take the data.
172174
# Second, data is given -> take it. Third, else -> build a new cutout
173175
if path.is_file():
174-
data = xr.open_dataset(str(path))
175-
data = data.chunk(chunks)
176+
data = xr.open_dataset(str(path), chunks=chunks)
176177
data.attrs.update(storable_chunks)
177178
if cutoutparams:
178179
warn(

‎atlite/data.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,6 @@ def cutout_prepare(
324324
cutout.path.unlink()
325325
Path(tmp).rename(cutout.path)
326326

327-
cutout.data = xr.open_dataset(cutout.path, chunks=cutout.chunks)
327+
cutout.data = xr.open_dataset(cutout.path, chunks="auto")
328328

329329
return cutout

‎atlite/pv/irradiation.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -159,10 +159,10 @@ def TiltedDiffuseIrrad(
159159
) * diffuse
160160

161161
if (
162-
logger.isEnabledFor(logging.WARNING)
162+
logger.isEnabledFor(logging.DEBUG)
163163
and ((diffuse_t < 0.0) & (sinaltitude > sin(radians(1.0)))).any()
164164
):
165-
logger.warning("diffuse_t exhibits negative values above altitude threshold.")
165+
logger.debug("diffuse_t exhibits negative values above altitude threshold.")
166166

167167
# fixup: clip all negative values (unclear why it gets negative)
168168
# note: REatlas does not do the fixup

‎atlite/pv/orientation.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -304,12 +304,12 @@ def SurfaceOrientation(
304304
azimuth_difference = xr.where(
305305
azimuth_difference < -pi, 2 * pi + azimuth_difference, azimuth_difference
306306
)
307-
rotation = np.where(
307+
rotation = xr.where(
308308
logical_and(rotation < 0, azimuth_difference > 0),
309309
rotation + pi,
310310
rotation,
311311
)
312-
rotation = np.where(
312+
rotation = xr.where(
313313
logical_and(rotation > 0, azimuth_difference < 0),
314314
rotation - pi,
315315
rotation,

‎test/test_dynamic_line_rating.py‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,14 @@
99
@author: fabian
1010
"""
1111

12+
import geopandas as gpd
1213
import numpy as np
1314
import pandas as pd
1415
import pytest
16+
import xarray as xr
1517
from shapely.geometry import LineString, Point
1618

19+
from atlite import Cutout
1720
from atlite.convert import convert_line_rating, line_azimuth_degrees
1821

1922

@@ -194,3 +197,62 @@ def test_line_azimuth_degrees(start, end, expected):
194197
"""`line_azimuth_degrees` returns degrees consistent with `convert_line_rating`'s `psi`."""
195198
shape = LineString([Point(*start), Point(*end)])
196199
assert np.isclose(line_azimuth_degrees(shape), expected)
200+
201+
202+
@pytest.fixture
203+
def cutout(tmp_path):
204+
rng = np.random.default_rng(0)
205+
coords = {
206+
"time": pd.date_range("2013-06-01", periods=24, freq="h"),
207+
"y": [50.0, 50.25, 50.5],
208+
"x": [5.0, 5.25, 5.5, 5.75],
209+
}
210+
211+
def var(low, high, dims=("time", "y", "x")):
212+
return (dims, rng.uniform(low, high, [len(coords[d]) for d in dims]))
213+
214+
ds = xr.Dataset(
215+
{
216+
"temperature": var(270, 300),
217+
"wnd100m": var(0, 15),
218+
"wnd_azimuth": var(0, 2 * np.pi),
219+
"influx_direct": var(0, 800),
220+
"solar_altitude": var(0, 1.2),
221+
"solar_azimuth": var(0, 2 * np.pi),
222+
"height": var(0, 500, ("y", "x")),
223+
},
224+
coords=coords,
225+
attrs={"module": "era5"},
226+
)
227+
ds["influx_direct"][:, 0, 0] = np.nan
228+
ds.to_netcdf(tmp_path / "cutout.nc")
229+
return Cutout(tmp_path / "cutout.nc")
230+
231+
232+
def test_line_rating_is_minimum_over_intersected_cells(cutout):
233+
shapes = gpd.GeoSeries([
234+
LineString([(5.0, 50.0), (5.5, 50.5)]),
235+
LineString([(5.75, 50.25), (5.25, 50.25)]),
236+
LineString([(9.0, 40.0), (9.5, 40.0)]),
237+
])
238+
R = pd.Series([3e-5, 4e-5, 5e-5])
239+
res = cutout.line_rating(shapes, R)
240+
241+
data = cutout.data.stack(spatial=["y", "x"])
242+
cells = cutout.intersectionmatrix(shapes).tocsr()
243+
for i, shape in enumerate(shapes):
244+
if not cells[i].nnz:
245+
assert res[i].isnull().all()
246+
continue
247+
psi = line_azimuth_degrees(shape)
248+
psi = psi if psi >= 0 else psi + 180
249+
ds = data.isel(spatial=cells[i].indices)
250+
expected = convert_line_rating(ds, psi, R[i])
251+
np.testing.assert_allclose(res[i], expected)
252+
253+
254+
def test_line_rating_without_intersections_is_nan(cutout):
255+
shapes = gpd.GeoSeries([LineString([(9.0, 40.0), (9.5, 40.0)])])
256+
res = cutout.line_rating(shapes, 3e-5)
257+
assert res.shape == (1, 24)
258+
assert res.isnull().all()

‎test/test_performance.py‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# SPDX-FileCopyrightText: Contributors to atlite <https://github.com/pypsa/atlite>
2+
#
3+
# SPDX-License-Identifier: MIT
4+
5+
"""Tests that conversions stay lazy and cutouts read data efficiently."""
6+
7+
import dask
8+
import numpy as np
9+
import pandas as pd
10+
import pytest
11+
import xarray as xr
12+
13+
from atlite import Cutout
14+
from atlite.convert import convert_csp, convert_pv
15+
from atlite.pv.orientation import get_orientation
16+
from atlite.resource import get_cspinstallationconfig, get_solarpanelconfig
17+
18+
19+
def _forbid_compute(*args, **kwargs):
20+
raise RuntimeError("Unexpected eager computation.")
21+
22+
23+
@pytest.fixture
24+
def ds():
25+
rng = np.random.default_rng(0)
26+
time = pd.date_range("2013-06-01", periods=48, freq="h")
27+
coords = {"time": time, "y": [50.0, 51.0], "x": [5.0, 6.0, 7.0]}
28+
shape = (len(time), 2, 3)
29+
30+
def var(low, high):
31+
return (("time", "y", "x"), rng.uniform(low, high, shape))
32+
33+
ds = xr.Dataset(
34+
{
35+
"influx_toa": var(800, 1000),
36+
"influx_direct": var(0, 400),
37+
"influx_diffuse": var(0, 200),
38+
"albedo": var(0, 0.3),
39+
"temperature": var(270, 300),
40+
"solar_altitude": var(-0.5, 1.2),
41+
"solar_azimuth": var(0, 2 * np.pi),
42+
},
43+
coords=coords,
44+
)
45+
ds = ds.assign_coords(lon=("x", coords["x"]), lat=("y", coords["y"]))
46+
return ds.chunk({"time": 24})
47+
48+
49+
@pytest.mark.parametrize(
50+
"kwargs",
51+
[
52+
{"tracking": None, "trigon_model": "other"},
53+
{"tracking": "tilted_horizontal"},
54+
],
55+
)
56+
def test_convert_pv_is_lazy(ds, kwargs):
57+
panel = dict(get_solarpanelconfig("CSi"))
58+
orientation = get_orientation({"slope": 30.0, "azimuth": 180.0})
59+
with dask.config.set(scheduler=_forbid_compute):
60+
da = convert_pv(ds, panel, orientation, **kwargs)
61+
assert isinstance(da.data, dask.array.Array)
62+
63+
64+
def test_convert_csp_keeps_chunks_and_coords(ds):
65+
installation = get_cspinstallationconfig("SAM_solar_tower")
66+
da = convert_csp(ds, installation)
67+
assert da.chunks == ds["influx_direct"].chunks
68+
assert set(da.coords) <= set(ds.coords)
69+
70+
71+
def test_cutout_chunks_align_with_file_chunks(tmp_path):
72+
path = tmp_path / "cutout.nc"
73+
time = pd.date_range("2013-01-01", periods=300, freq="h")
74+
ds = xr.Dataset(
75+
{"temperature": (("time", "y", "x"), np.zeros((300, 2, 3)))},
76+
coords={"time": time, "y": [50.0, 51.0], "x": [5.0, 6.0, 7.0]},
77+
attrs={"module": "era5"},
78+
)
79+
encoding = {"temperature": {"chunksizes": (150, 2, 3), "zlib": True}}
80+
ds.to_netcdf(path, encoding=encoding)
81+
82+
chunks = Cutout(path).data["temperature"].chunks
83+
assert all(c % 150 == 0 for c in chunks[0])

0 commit comments

Comments
 (0)