Skip to content
Open
Show file tree
Hide file tree
Changes from 12 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/conda-package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -91,13 +91,13 @@ jobs:
- name: Build conda package
id: build_conda_pkg
continue-on-error: true
run: conda build --no-test --python ${{ matrix.python }} --numpy 2.0 ${{ env.channels-list }} conda-recipe
run: conda-build --no-test --python ${{ matrix.python }} --numpy 2.0 ${{ env.channels-list }} conda-recipe
env:
MAX_BUILD_CMPL_MKL_VERSION: '2026.0a0'

- name: ReBuild conda package
if: steps.build_conda_pkg.outcome == 'failure'
run: conda build --no-test --python ${{ matrix.python }} --numpy 2.0 ${{ env.channels-list }} conda-recipe
run: conda-build --no-test --python ${{ matrix.python }} --numpy 2.0 ${{ env.channels-list }} conda-recipe
env:
MAX_BUILD_CMPL_MKL_VERSION: '2026.0a0'

Expand Down
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Added

* Added support for buffer protocol objects as advanced index keys in `dpnp.ndarray` [#2889](https://github.com/IntelPython/dpnp/pull/2889)

### Changed

### Deprecated
Expand All @@ -16,6 +18,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Fixed

* Fixed incorrect in-place advanced indexing for 4D arrays when using `range` or `list` as index keys [#2872](https://github.com/IntelPython/dpnp/pull/2872)

### Security


Expand Down
52 changes: 43 additions & 9 deletions dpnp/dpnp_array.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@

import warnings

import numpy

import dpnp
import dpnp.tensor as dpt
import dpnp.tensor._type_utils as dtu
Expand All @@ -46,24 +48,56 @@
from .exceptions import AxisError


def _unwrap_index_element(x):
"""
Unwrap a single index element for the tensor indexing layer.

Converts dpnp arrays to usm_ndarray and array-like objects (range, list,
buffer protocol objects) to numpy arrays for NumPy-compatible advanced
indexing.

"""

if isinstance(x, dpt.usm_ndarray):
return x
if isinstance(x, dpnp_array):
return x.get_array()
if isinstance(x, range):
return numpy.asarray(x, dtype=numpy.intp)
if isinstance(x, list):
# keep boolean lists as boolean
arr = numpy.asarray(x)
# cast empty lists (float64 in NumPy) to intp
# for correct tensor indexing
if arr.size == 0:
arr = arr.astype(numpy.intp)
return arr
if isinstance(x, numpy.ndarray):
return x
# convert buffer protocol objects (array.array, memoryview, etc.)
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can all that use cases covered converting x to numpy.ndarray?
Do we need so complicated if-logic?

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

here is CuPy implementation of a similar idea for a reference:
https://github.com/cupy/cupy/blob/main/cupy/_core/_routines_indexing.pyx#L247

can take a similar approach too in some cases (best to leave converting to usm_ndarray to the tensor layer, though, for queue propagation reasons)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think we need to fully follow the same CuPy implementation in _prepare_slice_list, because it seems CuPy mimics faulty NumPy behavior sometimes.

Not sure if I got the comment fully correctly, but in general I meant something like:

    if x is None or x is Ellipsis or isinstance(x, (dpt.usm_ndarray, slice, numpy.ndarray)):
        return x
    if isinstance(x, dpnp_array):
        return x.get_array()

    try:
        x = numpy.asarray(x)
    except ... :
        ...
        raise ...
    ...
    return x

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yep, agreed mostly, though problem is, we don't really want to send integers or boolean Python scalars to numpy arrays, which is why we may need something like:

        elif numpy.isscalar(s):
            if not isinstance(s, (bool, numpy.bool_)):
                # keep scalar int
                continue

which is in cupy implementation. We could handle range, list, tuple through numpy though

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don not think try/except around numpy.asarray is necessary because invalid index types either raise naturally or produce arrays with unsupported dtypes that the tensor layer rejects with IndexError

Regarding elif numpy.isscalar(s) I replaced it with isinstance(x, (int, numpy.generic)) because np.isscalar(memoryview(array.array("l", [0, 1, 2]))) returns True
bool check is not needed since bool is subclass of int

try:
Comment thread
ndgrigorian marked this conversation as resolved.
Outdated
mv = memoryview(x)
except TypeError:
return x
# 0-d buffers are handled by the tensor layer
if mv.ndim > 0:
return numpy.asarray(x)
return x


def _get_unwrapped_index_key(key):
"""
Get an unwrapped index key.

Return a key where each nested instance of DPNP array is unwrapped into
USM ndarray for further processing in DPCTL advanced indexing functions.
USM ndarray, and array-like objects (range, list) are converted to numpy
arrays for further processing in advanced indexing functions.

"""

if isinstance(key, tuple):
if any(isinstance(x, dpnp_array) for x in key):
# create a new tuple from the input key with unwrapped DPNP arrays
return tuple(
x.get_array() if isinstance(x, dpnp_array) else x for x in key
)
elif isinstance(key, dpnp_array):
return key.get_array()
return key
return tuple(_unwrap_index_element(x) for x in key)
return _unwrap_index_element(key)


# pylint: disable=too-many-public-methods
Expand Down
6 changes: 0 additions & 6 deletions dpnp/tensor/_slicing.pxi
Original file line number Diff line number Diff line change
Expand Up @@ -104,12 +104,6 @@ cdef bint _is_boolean(object x) except *:
return f in "?"
else:
return False
if callable(getattr(x, "__bool__", None)):
try:
x.__bool__()
except (TypeError, ValueError):
return False
return True
return False


Expand Down
89 changes: 89 additions & 0 deletions dpnp/tests/test_indexing.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import array
import functools

import dpctl
Expand Down Expand Up @@ -353,6 +354,94 @@ def test_indexing_array_negative_strides(self):
arr[slices] = 10
assert_equal(arr, 10.0, strict=False)

@pytest.mark.parametrize(
"idx",
[
(range(2), range(2)),
([0, 1], [0, 1]),
],
ids=["range", "list"],
)
def test_array_like_index_getitem(self, idx):
np_a = numpy.arange(36).reshape(2, 2, 3, 3)
dp_a = dpnp.arange(36).reshape(2, 2, 3, 3)
assert_array_equal(dp_a[idx], np_a[idx])

@pytest.mark.parametrize(
"idx",
[
(range(2), range(2)),
([0, 1], [0, 1]),
],
ids=["range", "list"],
)
def test_array_like_index_setitem(self, idx):
np_a = numpy.arange(36).reshape(2, 2, 3, 3)
dp_a = dpnp.arange(36).reshape(2, 2, 3, 3)
np_a[idx] = 0
dp_a[idx] = 0
assert_array_equal(dp_a, np_a)

def test_array_like_index_inplace_add(self):
np_a = numpy.arange(36).reshape(2, 2, 3, 3)
dp_a = dpnp.arange(36).reshape(2, 2, 3, 3)
np_tmp = -numpy.ones((2, 3, 3), dtype=numpy.intp)
dp_tmp = -dpnp.ones((2, 3, 3), dtype=numpy.intp)

np_a[range(2), range(2)] += 2 * np_tmp
dp_a[range(2), range(2)] += 2 * dp_tmp
assert_array_equal(dp_a, np_a)

@pytest.mark.parametrize(
"idx",
[
range(2),
[0, 1],
range(0),
[],
],
ids=["range", "list", "empty_range", "empty_list"],
)
def test_array_like_single_index(self, idx):
np_a = numpy.arange(24).reshape(2, 3, 4)
dp_a = dpnp.arange(24).reshape(2, 3, 4)
assert_array_equal(dp_a[idx], np_a[idx])

def test_buffer_protocol_getitem(self):
inds = array.array("l")
inds.frombytes(numpy.arange(3).tobytes())
np_a = numpy.arange(12).reshape(3, 4)
dp_a = dpnp.arange(12).reshape(3, 4)
assert_array_equal(dp_a[inds], np_a[inds])

def test_buffer_protocol_paired_index(self):
inds = array.array("l")
inds.frombytes(numpy.arange(3).tobytes())
np_a = numpy.arange(12).reshape(3, 4)
dp_a = dpnp.arange(12).reshape(3, 4)
assert_array_equal(dp_a[inds, inds], np_a[inds, inds])

def test_buffer_protocol_setitem(self):
inds = array.array("l")
inds.frombytes(numpy.arange(3).tobytes())
np_a = numpy.arange(12).reshape(3, 4)
dp_a = dpnp.arange(12).reshape(3, 4)
np_a[inds, inds] = 0
dp_a[inds, inds] = 0
assert_array_equal(dp_a, np_a)

def test_memoryview_getitem(self):
inds = memoryview(array.array("l", [0, 1, 2]))
np_a = numpy.arange(12).reshape(3, 4)
dp_a = dpnp.arange(12).reshape(3, 4)
assert_array_equal(dp_a[inds], np_a[inds])

def test_bytearray_getitem(self):
inds = bytearray(b"\x00\x01\x02")
np_a = numpy.arange(10)
dp_a = dpnp.arange(10)
assert_array_equal(dp_a[inds], np_a[inds])


class TestIx:
@pytest.mark.parametrize(
Expand Down
Loading