diff --git a/dpctl_ext/tensor/CMakeLists.txt b/dpctl_ext/tensor/CMakeLists.txt index 864e34ddaba..eff5e755264 100644 --- a/dpctl_ext/tensor/CMakeLists.txt +++ b/dpctl_ext/tensor/CMakeLists.txt @@ -63,6 +63,16 @@ set(_tensor_impl_sources ${CMAKE_CURRENT_SOURCE_DIR}/libtensor/source/repeat.cpp ${CMAKE_CURRENT_SOURCE_DIR}/libtensor/source/clip.cpp ) +set(_accumulator_sources + ${CMAKE_CURRENT_SOURCE_DIR}/libtensor/source/accumulators/accumulators_common.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/libtensor/source/accumulators/cumulative_logsumexp.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/libtensor/source/accumulators/cumulative_prod.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/libtensor/source/accumulators/cumulative_sum.cpp +) +set(_tensor_accumulation_impl_sources + ${CMAKE_CURRENT_SOURCE_DIR}/libtensor/source/tensor_accumulation.cpp + ${_accumulator_sources} +) set(_static_lib_trgt simplify_iteration_space) @@ -85,6 +95,12 @@ add_sycl_to_target(TARGET ${python_module_name} SOURCES ${_tensor_impl_sources}) target_link_libraries(${python_module_name} PRIVATE ${_static_lib_trgt}) list(APPEND _py_trgts ${python_module_name}) +set(python_module_name _tensor_accumulation_impl) +pybind11_add_module(${python_module_name} MODULE ${_tensor_accumulation_impl_sources}) +add_sycl_to_target(TARGET ${python_module_name} SOURCES ${_tensor_accumulation_impl_sources}) +target_link_libraries(${python_module_name} PRIVATE ${_static_lib_trgt}) +list(APPEND _py_trgts ${python_module_name}) + set(_clang_prefix "") if(WIN32) set(_clang_prefix "/clang:") @@ -97,14 +113,14 @@ set(_no_fast_math_sources ${CMAKE_CURRENT_SOURCE_DIR}/libtensor/source/clip.cpp ${CMAKE_CURRENT_SOURCE_DIR}/libtensor/source/where.cpp ) -#list( -#APPEND _no_fast_math_sources -# ${_elementwise_sources} -# ${_reduction_sources} -# ${_sorting_sources} -# ${_linalg_sources} -# ${_accumulator_sources} -#) +list( + APPEND _no_fast_math_sources + # ${_elementwise_sources} + # ${_reduction_sources} + # ${_sorting_sources} + # ${_linalg_sources} + ${_accumulator_sources} +) foreach(_src_fn ${_no_fast_math_sources}) get_source_file_property(_cmpl_options_prop ${_src_fn} COMPILE_OPTIONS) diff --git a/dpctl_ext/tensor/__init__.py b/dpctl_ext/tensor/__init__.py index 9d4013e146a..72c7536ed47 100644 --- a/dpctl_ext/tensor/__init__.py +++ b/dpctl_ext/tensor/__init__.py @@ -27,6 +27,7 @@ # ***************************************************************************** +from ._accumulation import cumulative_logsumexp, cumulative_prod, cumulative_sum from ._clip import clip from ._copy_utils import ( asnumpy, @@ -92,6 +93,9 @@ "concat", "copy", "clip", + "cumulative_logsumexp", + "cumulative_prod", + "cumulative_sum", "empty", "empty_like", "extract", diff --git a/dpctl_ext/tensor/_accumulation.py b/dpctl_ext/tensor/_accumulation.py new file mode 100644 index 00000000000..2dfe9656e19 --- /dev/null +++ b/dpctl_ext/tensor/_accumulation.py @@ -0,0 +1,470 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +import dpctl +import dpctl.tensor as dpt +from dpctl.utils import ExecutionPlacementError, SequentialOrderManager + +# TODO: revert to `import dpctl.tensor...` +# when dpnp fully migrates dpctl/tensor +import dpctl_ext.tensor as dpt_ext +import dpctl_ext.tensor._tensor_accumulation_impl as tai +import dpctl_ext.tensor._tensor_impl as ti + +from ._numpy_helper import normalize_axis_index +from ._type_utils import ( + _default_accumulation_dtype, + _default_accumulation_dtype_fp_types, + _to_device_supported_dtype, +) + + +def _accumulate_common( + x, + axis, + dtype, + include_initial, + out, + _accumulate_fn, + _accumulate_include_initial_fn, + _dtype_supported, + _default_accumulation_type_fn, +): + if not isinstance(x, dpt.usm_ndarray): + raise TypeError(f"Expected dpctl.tensor.usm_ndarray, got {type(x)}") + appended_axis = False + if x.ndim == 0: + x = x[dpt.newaxis] + appended_axis = True + nd = x.ndim + if axis is None: + if nd > 1: + raise ValueError( + "`axis` cannot be `None` for array of dimension `{}`".format(nd) + ) + axis = 0 + else: + axis = normalize_axis_index(axis, nd, "axis") + sh = x.shape + res_sh = ( + sh[:axis] + (sh[axis] + 1,) + sh[axis + 1 :] if include_initial else sh + ) + a1 = axis + 1 + if a1 == nd: + perm = list(range(nd)) + arr = x + else: + perm = [i for i in range(nd) if i != axis] + [ + axis, + ] + arr = dpt_ext.permute_dims(x, perm) + q = x.sycl_queue + inp_dt = x.dtype + res_usm_type = x.usm_type + if dtype is None: + res_dt = _default_accumulation_type_fn(inp_dt, q) + else: + res_dt = dpt.dtype(dtype) + res_dt = _to_device_supported_dtype(res_dt, q.sycl_device) + + # checking now avoids unnecessary allocations + implemented_types = _dtype_supported(inp_dt, res_dt) + if dtype is None and not implemented_types: + raise RuntimeError( + "Automatically determined accumulation data type does not " + "have direct implementation" + ) + orig_out = out + if out is not None: + if not isinstance(out, dpt.usm_ndarray): + raise TypeError( + f"output array must be of usm_ndarray type, got {type(out)}" + ) + if not out.flags.writable: + raise ValueError("provided `out` array is read-only") + out_sh = out.shape + # append an axis to `out` if scalar + if appended_axis and not include_initial: + out = out[dpt.newaxis, ...] + orig_out = out + final_res_sh = res_sh[1:] + else: + final_res_sh = res_sh + if not out_sh == final_res_sh: + raise ValueError( + "The shape of input and output arrays are inconsistent. " + f"Expected output shape is {final_res_sh}, got {out_sh}" + ) + if res_dt != out.dtype: + raise ValueError( + f"Output array of type {res_dt} is needed, " f"got {out.dtype}" + ) + if dpctl.utils.get_execution_queue((q, out.sycl_queue)) is None: + raise ExecutionPlacementError( + "Input and output allocation queues are not compatible" + ) + # permute out array dims if necessary + if a1 != nd: + out = dpt_ext.permute_dims(out, perm) + orig_out = out + if ti._array_overlap(x, out) and implemented_types: + out = dpt_ext.empty_like(out) + else: + out = dpt_ext.empty( + res_sh, dtype=res_dt, usm_type=res_usm_type, sycl_queue=q + ) + if a1 != nd: + out = dpt_ext.permute_dims(out, perm) + + _manager = SequentialOrderManager[q] + depends = _manager.submitted_events + if implemented_types: + if not include_initial: + ht_e, acc_ev = _accumulate_fn( + src=arr, + trailing_dims_to_accumulate=1, + dst=out, + sycl_queue=q, + depends=depends, + ) + else: + ht_e, acc_ev = _accumulate_include_initial_fn( + src=arr, dst=out, sycl_queue=q, depends=depends + ) + _manager.add_event_pair(ht_e, acc_ev) + if not (orig_out is None or out is orig_out): + # Copy the out data from temporary buffer to original memory + ht_e_cpy, cpy_e = ti._copy_usm_ndarray_into_usm_ndarray( + src=out, dst=orig_out, sycl_queue=q, depends=[acc_ev] + ) + _manager.add_event_pair(ht_e_cpy, cpy_e) + out = orig_out + else: + if _dtype_supported(res_dt, res_dt): + tmp = dpt_ext.empty( + arr.shape, dtype=res_dt, usm_type=res_usm_type, sycl_queue=q + ) + ht_e_cpy, cpy_e = ti._copy_usm_ndarray_into_usm_ndarray( + src=arr, dst=tmp, sycl_queue=q, depends=depends + ) + _manager.add_event_pair(ht_e_cpy, cpy_e) + if not include_initial: + ht_e, acc_ev = _accumulate_fn( + src=tmp, + trailing_dims_to_accumulate=1, + dst=out, + sycl_queue=q, + depends=[cpy_e], + ) + else: + ht_e, acc_ev = _accumulate_include_initial_fn( + src=tmp, + dst=out, + sycl_queue=q, + depends=[cpy_e], + ) + _manager.add_event_pair(ht_e, acc_ev) + else: + buf_dt = _default_accumulation_type_fn(inp_dt, q) + tmp = dpt_ext.empty( + arr.shape, dtype=buf_dt, usm_type=res_usm_type, sycl_queue=q + ) + ht_e_cpy, cpy_e = ti._copy_usm_ndarray_into_usm_ndarray( + src=arr, dst=tmp, sycl_queue=q, depends=depends + ) + _manager.add_event_pair(ht_e_cpy, cpy_e) + tmp_res = dpt_ext.empty( + res_sh, dtype=buf_dt, usm_type=res_usm_type, sycl_queue=q + ) + if a1 != nd: + tmp_res = dpt_ext.permute_dims(tmp_res, perm) + if not include_initial: + ht_e, acc_ev = _accumulate_fn( + src=tmp, + trailing_dims_to_accumulate=1, + dst=tmp_res, + sycl_queue=q, + depends=[cpy_e], + ) + else: + ht_e, acc_ev = _accumulate_include_initial_fn( + src=tmp, + dst=tmp_res, + sycl_queue=q, + depends=[cpy_e], + ) + _manager.add_event_pair(ht_e, acc_ev) + ht_e_cpy2, cpy_e2 = ti._copy_usm_ndarray_into_usm_ndarray( + src=tmp_res, dst=out, sycl_queue=q, depends=[acc_ev] + ) + _manager.add_event_pair(ht_e_cpy2, cpy_e2) + + if appended_axis: + out = dpt_ext.squeeze(out) + if a1 != nd: + inv_perm = sorted(range(nd), key=lambda d: perm[d]) + out = dpt_ext.permute_dims(out, inv_perm) + + return out + + +def cumulative_sum( + x, /, *, axis=None, dtype=None, include_initial=False, out=None +): + """ + cumulative_sum(x, /, *, axis=None, dtype=None, include_initial=False, + out=None) + + Calculates the cumulative sum of elements in the input array `x`. + + Args: + x (usm_ndarray): + input array. + axis (Optional[int]): + axis along which cumulative sum must be computed. + If `None`, the sum is computed over the entire array. + If `x` is a one-dimensional array, providing an `axis` is optional; + however, if `x` has more than one dimension, providing an `axis` + is required. + Default: `None`. + dtype (Optional[dtype]): + data type of the returned array. If `None`, the default data + type is inferred from the "kind" of the input array data type. + + * If `x` has a real- or complex-valued floating-point data + type, the returned array will have the same data type as + `x`. + * If `x` has signed integral data type, the returned array + will have the default signed integral type for the device + where input array `x` is allocated. + * If `x` has unsigned integral data type, the returned array + will have the default unsigned integral type for the device + where input array `x` is allocated. + * If `x` has a boolean data type, the returned array will + have the default signed integral type for the device + where input array `x` is allocated. + + If the data type (either specified or resolved) differs from the + data type of `x`, the input array elements are cast to the + specified data type before computing the cumulative sum. + Default: `None`. + include_initial (bool): + boolean indicating whether to include the initial value (i.e., the + additive identity, zero) as the first value along the provided axis + in the output. Default: `False`. + out (Optional[usm_ndarray]): + the array into which the result is written. + The data type of `out` must match the expected shape and the + expected data type of the result or (if provided) `dtype`. + If `None` then a new array is returned. Default: `None`. + + Returns: + usm_ndarray: + an array containing cumulative sums. The returned array has the data + type as described in the `dtype` parameter description above. + + The returned array shape is determined as follows: + + * If `include_initial` is `False`, the returned array will + have the same shape as `x` + * If `include_initial` is `True`, the returned array will + have the same shape as `x` except the axis along which the + cumulative sum is calculated, which will have size `N+1` + + where `N` is the size of the axis the cumulative sums are computed + along. + """ + return _accumulate_common( + x, + axis, + dtype, + include_initial, + out, + tai._cumsum_over_axis, + tai._cumsum_final_axis_include_initial, + tai._cumsum_dtype_supported, + _default_accumulation_dtype, + ) + + +def cumulative_prod( + x, /, *, axis=None, dtype=None, include_initial=False, out=None +): + """ + cumulative_prod(x, /, *, axis=None, dtype=None, include_initial=False, + out=None) + + Calculates the cumulative product of elements in the input array `x`. + + Args: + x (usm_ndarray): + input array. + axis (Optional[int]): + axis along which cumulative product must be computed. + If `None`, the product is computed over the entire array. + If `x` is a one-dimensional array, providing an `axis` is optional; + however, if `x` has more than one dimension, providing an `axis` + is required. + Default: `None`. + dtype (Optional[dtype]): + data type of the returned array. If `None`, the default data + type is inferred from the "kind" of the input array data type. + + * If `x` has a real- or complex-valued floating-point data + type, the returned array will have the same data type as + `x`. + * If `x` has signed integral data type, the returned array + will have the default signed integral type for the device + where input array `x` is allocated. + * If `x` has unsigned integral data type, the returned array + will have the default unsigned integral type for the device + where input array `x` is allocated. + * If `x` has a boolean data type, the returned array will + have the default signed integral type for the device + where input array `x` is allocated. + + If the data type (either specified or resolved) differs from the + data type of `x`, the input array elements are cast to the + specified data type before computing the cumulative product. + Default: `None`. + include_initial (bool): + boolean indicating whether to include the initial value (i.e., the + additive identity, zero) as the first value along the provided + axis in the output. Default: `False`. + out (Optional[usm_ndarray]): + the array into which the result is written. + The data type of `out` must match the expected shape and the + expected data type of the result or (if provided) `dtype`. + If `None` then a new array is returned. Default: `None`. + + Returns: + usm_ndarray: + an array containing cumulative products. The returned array has + the data type as described in the `dtype` parameter description + above. + + The returned array shape is determined as follows: + + * If `include_initial` is `False`, the returned array will + have the same shape as `x` + * If `include_initial` is `True`, the returned array will + have the same shape as `x` except the axis along which the + cumulative product is calculated, which will have size `N+1` + + where `N` is the size of the axis the cumulative products are + computed along. + """ + return _accumulate_common( + x, + axis, + dtype, + include_initial, + out, + tai._cumprod_over_axis, + tai._cumprod_final_axis_include_initial, + tai._cumprod_dtype_supported, + _default_accumulation_dtype, + ) + + +def cumulative_logsumexp( + x, /, *, axis=None, dtype=None, include_initial=False, out=None +): + """ + cumulative_logsumexp(x, /, *, axis=None, dtype=None, include_initial=False, + out=None) + + Calculates the cumulative logsmumexp of elements in the input array `x`. + + Args: + x (usm_ndarray): + input array. + axis (Optional[int]): + axis along which cumulative logsumexp must be computed. + If `None`, the logsumexp is computed over the entire array. + If `x` is a one-dimensional array, providing an `axis` is optional; + however, if `x` has more than one dimension, providing an `axis` + is required. + Default: `None`. + dtype (Optional[dtype]): + data type of the returned array. If `None`, the default data + type is inferred from the "kind" of the input array data type. + + * If `x` has a real- or complex-valued floating-point data + type, the returned array will have the same data type as + `x`. + * If `x` has signed integral data type, the returned array + will have the default signed integral type for the device + where input array `x` is allocated. + * If `x` has unsigned integral data type, the returned array + will have the default unsigned integral type for the device + where input array `x` is allocated. + * If `x` has a boolean data type, the returned array will + have the default signed integral type for the device + where input array `x` is allocated. + + If the data type (either specified or resolved) differs from the + data type of `x`, the input array elements are cast to the + specified data type before computing the cumulative logsumexp. + Default: `None`. + include_initial (bool): + boolean indicating whether to include the initial value (i.e., the + additive identity, zero) as the first value along the provided axis + in the output. Default: `False`. + out (Optional[usm_ndarray]): + the array into which the result is written. + The data type of `out` must match the expected shape and the + expected data type of the result or (if provided) `dtype`. + If `None` then a new array is returned. Default: `None`. + + Returns: + usm_ndarray: + an array containing cumulative logsumexp results. The returned + array has the data type as described in the `dtype` parameter + description above. + + The returned array shape is determined as follows: + + * If `include_initial` is `False`, the returned array will + have the same shape as `x` + * If `include_initial` is `True`, the returned array will + have the same shape as `x` except the axis along which the + cumulative logsumexp is calculated, which will have size + `N+1` + """ + return _accumulate_common( + x, + axis, + dtype, + include_initial, + out, + tai._cumlogsumexp_over_axis, + tai._cumlogsumexp_final_axis_include_initial, + tai._cumlogsumexp_dtype_supported, + _default_accumulation_dtype_fp_types, + ) diff --git a/dpctl_ext/tensor/libtensor/source/accumulators/accumulate_over_axis.hpp b/dpctl_ext/tensor/libtensor/source/accumulators/accumulate_over_axis.hpp new file mode 100644 index 00000000000..4dd00620a26 --- /dev/null +++ b/dpctl_ext/tensor/libtensor/source/accumulators/accumulate_over_axis.hpp @@ -0,0 +1,462 @@ +//***************************************************************************** +// Copyright (c) 2026, Intel Corporation +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// - Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// - Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +//***************************************************************************** +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines functions of dpctl.tensor._tensor_accumulation_impl +// extensions +//===----------------------------------------------------------------------===// + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "dpnp4pybind11.hpp" +#include +#include + +#include "kernels/accumulators.hpp" +#include "simplify_iteration_space.hpp" +#include "utils/memory_overlap.hpp" +#include "utils/offset_utils.hpp" +#include "utils/output_validation.hpp" +#include "utils/sycl_alloc_utils.hpp" +#include "utils/type_dispatch.hpp" + +namespace dpctl::tensor::py_internal +{ + +namespace py = pybind11; +namespace td_ns = dpctl::tensor::type_dispatch; + +template +std::pair + py_accumulate_over_axis(const dpctl::tensor::usm_ndarray &src, + const int trailing_dims_to_accumulate, + const dpctl::tensor::usm_ndarray &dst, + sycl::queue &exec_q, + std::vector const &depends, + const strided_fnT &strided_dispatch_table, + const contig_fnT &contig_dispatch_table) +{ + int src_nd = src.get_ndim(); + int dst_nd = dst.get_ndim(); + if (src_nd != dst_nd) { + throw py::value_error("The input and output arrays must have " + "the same array ranks"); + } + int iter_nd = src_nd - trailing_dims_to_accumulate; + if (trailing_dims_to_accumulate <= 0 || iter_nd < 0) { + throw py::value_error( + "trailing_dims_to_accumulate must be positive, but no " + "greater than rank of the input array"); + } + + const py::ssize_t *src_shape_ptr = src.get_shape_raw(); + const py::ssize_t *dst_shape_ptr = dst.get_shape_raw(); + + bool same_shapes = true; + std::size_t iter_nelems(1); + for (int i = 0; same_shapes && (i < iter_nd); ++i) { + auto src_shape_i = src_shape_ptr[i]; + same_shapes = same_shapes && (src_shape_i == dst_shape_ptr[i]); + iter_nelems *= static_cast(src_shape_i); + } + + std::size_t acc_nelems(1); + for (int i = iter_nd; same_shapes && (i < src_nd); ++i) { + auto dst_shape_i = dst_shape_ptr[i]; + same_shapes = same_shapes && (src_shape_ptr[i] == dst_shape_i); + acc_nelems *= static_cast(dst_shape_i); + } + + if (!same_shapes) { + throw py::value_error( + "Destination shape does not match the input shape"); + } + + if (!dpctl::utils::queues_are_compatible(exec_q, {src, dst})) { + throw py::value_error( + "Execution queue is not compatible with allocation queues"); + } + + dpctl::tensor::validation::CheckWritable::throw_if_not_writable(dst); + + if ((iter_nelems == 0) || (acc_nelems == 0)) { + return std::make_pair(sycl::event(), sycl::event()); + } + + auto const &overlap = dpctl::tensor::overlap::MemoryOverlap(); + if (overlap(src, dst)) { + throw py::value_error("Arrays index overlapping segments of memory"); + } + + dpctl::tensor::validation::AmpleMemory::throw_if_not_ample( + dst, acc_nelems * iter_nelems); + + const char *src_data = src.get_data(); + char *dst_data = dst.get_data(); + + int src_typenum = src.get_typenum(); + int dst_typenum = dst.get_typenum(); + + const auto &array_types = td_ns::usm_ndarray_types(); + int src_typeid = array_types.typenum_to_lookup_id(src_typenum); + int dst_typeid = array_types.typenum_to_lookup_id(dst_typenum); + + bool is_src_c_contig = src.is_c_contiguous(); + bool is_dst_c_contig = dst.is_c_contiguous(); + + std::vector host_task_events; + + if ((is_src_c_contig && is_dst_c_contig) && iter_nd == 0) { + auto fn = contig_dispatch_table[src_typeid][dst_typeid]; + if (fn == nullptr) { + throw std::runtime_error("Datatypes are not supported"); + } + + sycl::event acc_ev = fn(exec_q, acc_nelems, src_data, dst_data, + host_task_events, depends); + + return std::make_pair( + dpctl::utils::keep_args_alive(exec_q, {src, dst}, {acc_ev}), + acc_ev); + } + + auto src_shape_vec = src.get_shape_vector(); + auto src_strides_vec = src.get_strides_vector(); + auto dst_strides_vec = dst.get_strides_vector(); + + int acc_nd = trailing_dims_to_accumulate; + + using shT = std::vector; + shT acc_shape(std::begin(src_shape_vec) + iter_nd, std::end(src_shape_vec)); + + shT acc_src_strides(std::begin(src_strides_vec) + iter_nd, + std::end(src_strides_vec)); + + shT acc_dst_strides(std::begin(dst_strides_vec) + iter_nd, + std::end(dst_strides_vec)); + + shT iter_shape(std::begin(src_shape_vec), + std::begin(src_shape_vec) + iter_nd); + + shT iter_src_strides(std::begin(src_strides_vec), + std::begin(src_strides_vec) + iter_nd); + + shT iter_dst_strides(std::begin(dst_strides_vec), + std::begin(dst_strides_vec) + iter_nd); + + shT simplified_iter_shape; + shT simplified_iter_src_strides; + shT simplified_iter_dst_strides; + py::ssize_t iter_src_offset(0); + py::ssize_t iter_dst_offset(0); + + if (iter_nd == 0) { + iter_nd = 1; + simplified_iter_shape.push_back(1); + simplified_iter_src_strides.push_back(0); + simplified_iter_dst_strides.push_back(0); + } + else { + simplify_iteration_space( + iter_nd, src_shape_ptr, iter_src_strides, iter_dst_strides, + // output + simplified_iter_shape, simplified_iter_src_strides, + simplified_iter_dst_strides, iter_src_offset, iter_dst_offset); + } + + // Strided implementation + auto strided_fn = strided_dispatch_table[src_typeid][dst_typeid]; + if (strided_fn == nullptr) { + throw std::runtime_error("Datatypes are not supported"); + } + + using dpctl::tensor::offset_utils::device_allocate_and_pack; + auto ptr_size_event_tuple = device_allocate_and_pack( + exec_q, host_task_events, simplified_iter_shape, + simplified_iter_src_strides, simplified_iter_dst_strides, acc_shape, + acc_src_strides, acc_dst_strides); + auto packed_shapes_and_strides_owner = + std::move(std::get<0>(ptr_size_event_tuple)); + const auto ©_shapes_strides_ev = std::get<2>(ptr_size_event_tuple); + const py::ssize_t *packed_shapes_and_strides = + packed_shapes_and_strides_owner.get(); + + const py::ssize_t *iter_shape_and_strides = packed_shapes_and_strides; + const py::ssize_t *acc_shapes_and_strides = + packed_shapes_and_strides + 3 * simplified_iter_shape.size(); + + std::vector all_deps; + all_deps.reserve(depends.size() + 1); + all_deps.insert(all_deps.end(), copy_shapes_strides_ev); + all_deps.insert(all_deps.end(), depends.begin(), depends.end()); + + sycl::event acc_ev = strided_fn( + exec_q, iter_nelems, acc_nelems, src_data, iter_nd, + iter_shape_and_strides, iter_src_offset, iter_dst_offset, acc_nd, + acc_shapes_and_strides, dst_data, host_task_events, all_deps); + + sycl::event temp_cleanup_ev = dpctl::tensor::alloc_utils::async_smart_free( + exec_q, {acc_ev}, packed_shapes_and_strides_owner); + host_task_events.push_back(temp_cleanup_ev); + + return std::make_pair( + dpctl::utils::keep_args_alive(exec_q, {src, dst}, host_task_events), + acc_ev); +} + +template +std::pair py_accumulate_final_axis_include_initial( + const dpctl::tensor::usm_ndarray &src, + const dpctl::tensor::usm_ndarray &dst, + sycl::queue &exec_q, + std::vector const &depends, + const strided_fnT &strided_dispatch_table, + const contig_fnT &contig_dispatch_table) +{ + int src_nd = src.get_ndim(); + int dst_nd = dst.get_ndim(); + + if (src_nd != dst_nd) { + throw py::value_error("The input and output arrays must have " + "the same array ranks"); + } + + static constexpr int acc_nd = 1; + + int iter_nd = src_nd - acc_nd; + if (iter_nd < 0) { + throw py::value_error("accumulation axis must not be greater than rank " + "of the input array"); + } + + const py::ssize_t *src_shape_ptr = src.get_shape_raw(); + const py::ssize_t *dst_shape_ptr = dst.get_shape_raw(); + + bool same_shapes = true; + std::size_t iter_nelems(1); + for (int i = 0; same_shapes && (i < iter_nd); ++i) { + auto src_shape_i = src_shape_ptr[i]; + same_shapes = same_shapes && (src_shape_i == dst_shape_ptr[i]); + iter_nelems *= static_cast(src_shape_i); + } + + std::size_t acc_nelems(1); + for (int i = iter_nd; same_shapes && (i < src_nd); ++i) { + auto dst_shape_i = dst_shape_ptr[i]; + same_shapes = same_shapes && (src_shape_ptr[i] + 1 == dst_shape_i); + acc_nelems *= static_cast(dst_shape_i); + } + + if (!same_shapes) { + throw py::value_error( + "Destination shape does not match the input shape"); + } + + if (!dpctl::utils::queues_are_compatible(exec_q, {src, dst})) { + throw py::value_error( + "Execution queue is not compatible with allocation queues"); + } + + dpctl::tensor::validation::CheckWritable::throw_if_not_writable(dst); + + if ((iter_nelems == 0) || (acc_nelems == 0)) { + return std::make_pair(sycl::event(), sycl::event()); + } + + auto const &overlap = dpctl::tensor::overlap::MemoryOverlap(); + if (overlap(src, dst)) { + throw py::value_error("Arrays index overlapping segments of memory"); + } + + dpctl::tensor::validation::AmpleMemory::throw_if_not_ample( + dst, acc_nelems * iter_nelems); + + const char *src_data = src.get_data(); + char *dst_data = dst.get_data(); + + int src_typenum = src.get_typenum(); + int dst_typenum = dst.get_typenum(); + + const auto &array_types = td_ns::usm_ndarray_types(); + int src_typeid = array_types.typenum_to_lookup_id(src_typenum); + int dst_typeid = array_types.typenum_to_lookup_id(dst_typenum); + + bool is_src_c_contig = src.is_c_contiguous(); + bool is_dst_c_contig = dst.is_c_contiguous(); + + std::vector host_task_events; + + if ((is_src_c_contig && is_dst_c_contig) && iter_nd == 0) { + auto fn = contig_dispatch_table[src_typeid][dst_typeid]; + if (fn == nullptr) { + throw std::runtime_error("Datatypes are not supported"); + } + + sycl::event acc_ev = fn(exec_q, acc_nelems, src_data, dst_data, + host_task_events, depends); + + return std::make_pair( + dpctl::utils::keep_args_alive(exec_q, {src, dst}, {acc_ev}), + acc_ev); + } + + auto src_shape_vec = src.get_shape_vector(); + auto src_strides_vec = src.get_strides_vector(); + auto dst_strides_vec = dst.get_strides_vector(); + + using shT = std::vector; + shT acc_shape(std::begin(src_shape_vec) + iter_nd, std::end(src_shape_vec)); + + shT acc_src_strides(std::begin(src_strides_vec) + iter_nd, + std::end(src_strides_vec)); + + shT acc_dst_strides(std::begin(dst_strides_vec) + iter_nd, + std::end(dst_strides_vec)); + + shT iter_shape(std::begin(src_shape_vec), + std::begin(src_shape_vec) + iter_nd); + + shT iter_src_strides(std::begin(src_strides_vec), + std::begin(src_strides_vec) + iter_nd); + + shT iter_dst_strides(std::begin(dst_strides_vec), + std::begin(dst_strides_vec) + iter_nd); + + shT simplified_iter_shape; + shT simplified_iter_src_strides; + shT simplified_iter_dst_strides; + py::ssize_t iter_src_offset(0); + py::ssize_t iter_dst_offset(0); + + if (iter_nd == 0) { + iter_nd = 1; + simplified_iter_shape.push_back(1); + simplified_iter_src_strides.push_back(0); + simplified_iter_dst_strides.push_back(0); + } + else { + simplify_iteration_space( + iter_nd, src_shape_ptr, iter_src_strides, iter_dst_strides, + // output + simplified_iter_shape, simplified_iter_src_strides, + simplified_iter_dst_strides, iter_src_offset, iter_dst_offset); + } + + // Strided implementation + auto strided_fn = strided_dispatch_table[src_typeid][dst_typeid]; + if (strided_fn == nullptr) { + throw std::runtime_error("Datatypes are not supported"); + } + + using dpctl::tensor::offset_utils::device_allocate_and_pack; + auto ptr_size_event_tuple = device_allocate_and_pack( + exec_q, host_task_events, simplified_iter_shape, + simplified_iter_src_strides, simplified_iter_dst_strides, acc_shape, + acc_src_strides, acc_dst_strides); + auto packed_shapes_and_strides_owner = + std::move(std::get<0>(ptr_size_event_tuple)); + const auto ©_shapes_strides_ev = std::get<2>(ptr_size_event_tuple); + const py::ssize_t *packed_shapes_and_strides = + packed_shapes_and_strides_owner.get(); + + const py::ssize_t *iter_shape_and_strides = packed_shapes_and_strides; + const py::ssize_t *acc_shapes_and_strides = + packed_shapes_and_strides + 3 * simplified_iter_shape.size(); + + std::vector all_deps; + all_deps.reserve(depends.size() + 1); + all_deps.insert(all_deps.end(), copy_shapes_strides_ev); + all_deps.insert(all_deps.end(), depends.begin(), depends.end()); + + sycl::event acc_ev = strided_fn( + exec_q, iter_nelems, acc_nelems, src_data, iter_nd, + iter_shape_and_strides, iter_src_offset, iter_dst_offset, acc_nd, + acc_shapes_and_strides, dst_data, host_task_events, all_deps); + + sycl::event temp_cleanup_ev = dpctl::tensor::alloc_utils::async_smart_free( + exec_q, {acc_ev}, packed_shapes_and_strides_owner); + host_task_events.push_back(temp_cleanup_ev); + + return std::make_pair( + dpctl::utils::keep_args_alive(exec_q, {src, dst}, host_task_events), + acc_ev); +} + +/*! @brief Template implementing Python API for querying accumulation + * type support */ +template +bool py_accumulate_dtype_supported(const py::dtype &input_dtype, + const py::dtype &output_dtype, + const fnT &dispatch_table) +{ + int arg_tn = + input_dtype.num(); // NumPy type numbers are the same as in dpctl + int out_tn = + output_dtype.num(); // NumPy type numbers are the same as in dpctl + int arg_typeid = -1; + int out_typeid = -1; + + auto array_types = td_ns::usm_ndarray_types(); + + try { + arg_typeid = array_types.typenum_to_lookup_id(arg_tn); + out_typeid = array_types.typenum_to_lookup_id(out_tn); + } catch (const std::exception &e) { + throw py::value_error(e.what()); + } + + if (arg_typeid < 0 || arg_typeid >= td_ns::num_types || out_typeid < 0 || + out_typeid >= td_ns::num_types) + { + throw std::runtime_error("Reduction type support check: lookup failed"); + } + + // remove_all_extents gets underlying type of table + using fn_ptrT = typename std::remove_all_extents::type; + fn_ptrT fn = nullptr; + + fn = dispatch_table[arg_typeid][out_typeid]; + + return (fn != nullptr); +} + +} // namespace dpctl::tensor::py_internal diff --git a/dpctl_ext/tensor/libtensor/source/accumulators/accumulators_common.cpp b/dpctl_ext/tensor/libtensor/source/accumulators/accumulators_common.cpp new file mode 100644 index 00000000000..5e07e81b7ad --- /dev/null +++ b/dpctl_ext/tensor/libtensor/source/accumulators/accumulators_common.cpp @@ -0,0 +1,55 @@ +//***************************************************************************** +// Copyright (c) 2026, Intel Corporation +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// - Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// - Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +//***************************************************************************** +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines functions of dpctl.tensor._tensor_accumulation_impl +// extensions +//===----------------------------------------------------------------------===// + +#include + +#include "cumulative_logsumexp.hpp" +#include "cumulative_prod.hpp" +#include "cumulative_sum.hpp" + +namespace py = pybind11; + +namespace dpctl::tensor::py_internal +{ + +/*! @brief Add accumulators to Python module */ +void init_accumulator_functions(py::module_ m) +{ + init_cumulative_logsumexp(m); + init_cumulative_prod(m); + init_cumulative_sum(m); +} + +} // namespace dpctl::tensor::py_internal diff --git a/dpctl_ext/tensor/libtensor/source/accumulators/accumulators_common.hpp b/dpctl_ext/tensor/libtensor/source/accumulators/accumulators_common.hpp new file mode 100644 index 00000000000..c33a040a7fa --- /dev/null +++ b/dpctl_ext/tensor/libtensor/source/accumulators/accumulators_common.hpp @@ -0,0 +1,46 @@ +//***************************************************************************** +// Copyright (c) 2026, Intel Corporation +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// - Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// - Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +//***************************************************************************** +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines functions of dpctl.tensor._tensor_accumulation_impl +// extensions +//===----------------------------------------------------------------------===// + +#pragma once +#include + +namespace py = pybind11; + +namespace dpctl::tensor::py_internal +{ + +extern void init_accumulator_functions(py::module_); + +} // namespace dpctl::tensor::py_internal diff --git a/dpctl_ext/tensor/libtensor/source/accumulators/cumulative_logsumexp.cpp b/dpctl_ext/tensor/libtensor/source/accumulators/cumulative_logsumexp.cpp new file mode 100644 index 00000000000..f1ad170caa5 --- /dev/null +++ b/dpctl_ext/tensor/libtensor/source/accumulators/cumulative_logsumexp.cpp @@ -0,0 +1,348 @@ +//***************************************************************************** +// Copyright (c) 2026, Intel Corporation +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// - Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// - Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +//***************************************************************************** +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines functions of dpctl.tensor._tensor_accumulation_impl +// extensions +//===----------------------------------------------------------------------===// + +#include +#include +#include + +#include + +#include "dpnp4pybind11.hpp" +#include +#include +#include + +#include "accumulate_over_axis.hpp" +#include "kernels/accumulators.hpp" +#include "utils/sycl_utils.hpp" +#include "utils/type_dispatch_building.hpp" + +namespace py = pybind11; + +namespace dpctl::tensor::py_internal +{ + +namespace su_ns = dpctl::tensor::sycl_utils; +namespace td_ns = dpctl::tensor::type_dispatch; + +namespace impl +{ + +using dpctl::tensor::kernels::accumulators::accumulate_1d_contig_impl_fn_ptr_t; +static accumulate_1d_contig_impl_fn_ptr_t + cumlogsumexp_1d_contig_dispatch_table[td_ns::num_types][td_ns::num_types]; + +using dpctl::tensor::kernels::accumulators::accumulate_strided_impl_fn_ptr_t; +static accumulate_strided_impl_fn_ptr_t + cumlogsumexp_strided_dispatch_table[td_ns::num_types][td_ns::num_types]; + +static accumulate_1d_contig_impl_fn_ptr_t + cumlogsumexp_1d_include_initial_contig_dispatch_table[td_ns::num_types] + [td_ns::num_types]; + +static accumulate_strided_impl_fn_ptr_t + cumlogsumexp_include_initial_strided_dispatch_table[td_ns::num_types] + [td_ns::num_types]; + +template +struct TypePairSupportDataForLogSumExpAccumulation +{ + static constexpr bool is_defined = std::disjunction< + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input int8_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input uint8_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input int16_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input uint16_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input int32_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input uint32_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input int64_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input uint64_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input half + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input float + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input double + td_ns::TypePairDefinedEntry, + + // fall-through + td_ns::NotDefinedEntry>::is_defined; +}; + +template +struct CumLogSumExp1DContigFactory +{ + fnT get() + { + if constexpr (TypePairSupportDataForLogSumExpAccumulation< + srcTy, dstTy>::is_defined) + { + using ScanOpT = su_ns::LogSumExp; + static constexpr bool include_initial = false; + if constexpr (std::is_same_v) { + using dpctl::tensor::kernels::accumulators::NoOpTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_1d_contig_impl, ScanOpT, + include_initial>; + return fn; + } + else { + using dpctl::tensor::kernels::accumulators::CastTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_1d_contig_impl, + ScanOpT, include_initial>; + return fn; + } + } + else { + return nullptr; + } + } +}; + +template +struct CumLogSumExp1DIncludeInitialContigFactory +{ + fnT get() + { + if constexpr (TypePairSupportDataForLogSumExpAccumulation< + srcTy, dstTy>::is_defined) + { + using ScanOpT = su_ns::LogSumExp; + static constexpr bool include_initial = true; + if constexpr (std::is_same_v) { + using dpctl::tensor::kernels::accumulators::NoOpTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_1d_contig_impl, ScanOpT, + include_initial>; + return fn; + } + else { + using dpctl::tensor::kernels::accumulators::CastTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_1d_contig_impl, + ScanOpT, include_initial>; + return fn; + } + } + else { + return nullptr; + } + } +}; + +template +struct CumLogSumExpStridedFactory +{ + fnT get() + { + if constexpr (TypePairSupportDataForLogSumExpAccumulation< + srcTy, dstTy>::is_defined) + { + using ScanOpT = su_ns::LogSumExp; + static constexpr bool include_initial = false; + if constexpr (std::is_same_v) { + using dpctl::tensor::kernels::accumulators::NoOpTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_strided_impl, ScanOpT, + include_initial>; + return fn; + } + else { + using dpctl::tensor::kernels::accumulators::CastTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_strided_impl, + ScanOpT, include_initial>; + return fn; + } + } + else { + return nullptr; + } + } +}; + +template +struct CumLogSumExpIncludeInitialStridedFactory +{ + fnT get() + { + if constexpr (TypePairSupportDataForLogSumExpAccumulation< + srcTy, dstTy>::is_defined) + { + using ScanOpT = su_ns::LogSumExp; + static constexpr bool include_initial = true; + if constexpr (std::is_same_v) { + using dpctl::tensor::kernels::accumulators::NoOpTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_strided_impl, ScanOpT, + include_initial>; + return fn; + } + else { + using dpctl::tensor::kernels::accumulators::CastTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_strided_impl, + ScanOpT, include_initial>; + return fn; + } + } + else { + return nullptr; + } + } +}; + +void populate_cumlogsumexp_dispatch_tables(void) +{ + td_ns::DispatchTableBuilder + dtb1; + dtb1.populate_dispatch_table(cumlogsumexp_1d_contig_dispatch_table); + + td_ns::DispatchTableBuilder + dtb2; + dtb2.populate_dispatch_table(cumlogsumexp_strided_dispatch_table); + + td_ns::DispatchTableBuilder + dtb3; + dtb3.populate_dispatch_table( + cumlogsumexp_1d_include_initial_contig_dispatch_table); + + td_ns::DispatchTableBuilder + dtb4; + dtb4.populate_dispatch_table( + cumlogsumexp_include_initial_strided_dispatch_table); + + return; +} + +} // namespace impl + +void init_cumulative_logsumexp(py::module_ m) +{ + using arrayT = dpctl::tensor::usm_ndarray; + using event_vecT = std::vector; + + using impl::populate_cumlogsumexp_dispatch_tables; + populate_cumlogsumexp_dispatch_tables(); + + using impl::cumlogsumexp_1d_contig_dispatch_table; + using impl::cumlogsumexp_strided_dispatch_table; + auto cumlogsumexp_pyapi = [&](const arrayT &src, + int trailing_dims_to_accumulate, + const arrayT &dst, sycl::queue &exec_q, + const event_vecT &depends = {}) { + return py_accumulate_over_axis(src, trailing_dims_to_accumulate, dst, + exec_q, depends, + cumlogsumexp_strided_dispatch_table, + cumlogsumexp_1d_contig_dispatch_table); + }; + m.def("_cumlogsumexp_over_axis", cumlogsumexp_pyapi, "", py::arg("src"), + py::arg("trailing_dims_to_accumulate"), py::arg("dst"), + py::arg("sycl_queue"), py::arg("depends") = py::list()); + + using impl::cumlogsumexp_1d_include_initial_contig_dispatch_table; + using impl::cumlogsumexp_include_initial_strided_dispatch_table; + auto cumlogsumexp_include_initial_pyapi = + [&](const arrayT &src, const arrayT &dst, sycl::queue &exec_q, + const event_vecT &depends = {}) { + return py_accumulate_final_axis_include_initial( + src, dst, exec_q, depends, + cumlogsumexp_include_initial_strided_dispatch_table, + cumlogsumexp_1d_include_initial_contig_dispatch_table); + }; + m.def("_cumlogsumexp_final_axis_include_initial", + cumlogsumexp_include_initial_pyapi, "", py::arg("src"), + py::arg("dst"), py::arg("sycl_queue"), + py::arg("depends") = py::list()); + + auto cumlogsumexp_dtype_supported = [&](const py::dtype &input_dtype, + const py::dtype &output_dtype) { + using dpctl::tensor::py_internal::py_accumulate_dtype_supported; + return py_accumulate_dtype_supported( + input_dtype, output_dtype, cumlogsumexp_strided_dispatch_table); + }; + m.def("_cumlogsumexp_dtype_supported", cumlogsumexp_dtype_supported, "", + py::arg("arg_dtype"), py::arg("out_dtype")); +} + +} // namespace dpctl::tensor::py_internal diff --git a/dpctl_ext/tensor/libtensor/source/accumulators/cumulative_logsumexp.hpp b/dpctl_ext/tensor/libtensor/source/accumulators/cumulative_logsumexp.hpp new file mode 100644 index 00000000000..f1292320bd0 --- /dev/null +++ b/dpctl_ext/tensor/libtensor/source/accumulators/cumulative_logsumexp.hpp @@ -0,0 +1,46 @@ +//***************************************************************************** +// Copyright (c) 2026, Intel Corporation +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// - Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// - Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +//***************************************************************************** +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines functions of dpctl.tensor._tensor_accumulation_impl +// extensions +//===----------------------------------------------------------------------===// + +#pragma once +#include + +namespace py = pybind11; + +namespace dpctl::tensor::py_internal +{ + +extern void init_cumulative_logsumexp(py::module_ m); + +} // namespace dpctl::tensor::py_internal diff --git a/dpctl_ext/tensor/libtensor/source/accumulators/cumulative_prod.cpp b/dpctl_ext/tensor/libtensor/source/accumulators/cumulative_prod.cpp new file mode 100644 index 00000000000..9a9961441d3 --- /dev/null +++ b/dpctl_ext/tensor/libtensor/source/accumulators/cumulative_prod.cpp @@ -0,0 +1,357 @@ +//***************************************************************************** +// Copyright (c) 2026, Intel Corporation +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// - Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// - Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +//***************************************************************************** +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines functions of dpctl.tensor._tensor_accumulation_impl +// extensions +//===----------------------------------------------------------------------===// + +#include +#include +#include +#include + +#include + +#include "dpnp4pybind11.hpp" +#include +#include +#include + +#include "accumulate_over_axis.hpp" +#include "kernels/accumulators.hpp" +#include "utils/type_dispatch_building.hpp" + +namespace py = pybind11; + +namespace dpctl::tensor::py_internal +{ + +namespace td_ns = dpctl::tensor::type_dispatch; + +namespace impl +{ + +using dpctl::tensor::kernels::accumulators::accumulate_1d_contig_impl_fn_ptr_t; +static accumulate_1d_contig_impl_fn_ptr_t + cumprod_1d_contig_dispatch_table[td_ns::num_types][td_ns::num_types]; + +using dpctl::tensor::kernels::accumulators::accumulate_strided_impl_fn_ptr_t; +static accumulate_strided_impl_fn_ptr_t + cumprod_strided_dispatch_table[td_ns::num_types][td_ns::num_types]; + +static accumulate_1d_contig_impl_fn_ptr_t + cumprod_1d_include_initial_contig_dispatch_table[td_ns::num_types] + [td_ns::num_types]; + +static accumulate_strided_impl_fn_ptr_t + cumprod_include_initial_strided_dispatch_table[td_ns::num_types] + [td_ns::num_types]; + +template +struct TypePairSupportDataForProdAccumulation +{ + static constexpr bool is_defined = std::disjunction< + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input int8_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input uint8_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input int16_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input uint16_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input int32_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input uint32_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input int64_t + td_ns::TypePairDefinedEntry, + + // input uint64_t + td_ns::TypePairDefinedEntry, + + // input half + td_ns::TypePairDefinedEntry, + + // input float + td_ns::TypePairDefinedEntry, + + // input double + td_ns::TypePairDefinedEntry, + + // input std::complex + td_ns::TypePairDefinedEntry, + outTy, + std::complex>, + + td_ns::TypePairDefinedEntry, + outTy, + std::complex>, + + // fall-through + td_ns::NotDefinedEntry>::is_defined; +}; + +template +using CumProdScanOpT = std::conditional_t, + sycl::logical_and, + sycl::multiplies>; + +template +struct CumProd1DContigFactory +{ + fnT get() + { + if constexpr (TypePairSupportDataForProdAccumulation::is_defined) + { + using ScanOpT = CumProdScanOpT; + static constexpr bool include_initial = false; + if constexpr (std::is_same_v) { + using dpctl::tensor::kernels::accumulators::NoOpTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_1d_contig_impl, ScanOpT, + include_initial>; + return fn; + } + else { + using dpctl::tensor::kernels::accumulators::CastTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_1d_contig_impl, + ScanOpT, include_initial>; + return fn; + } + } + else { + return nullptr; + } + } +}; + +template +struct CumProd1DIncludeInitialContigFactory +{ + fnT get() + { + if constexpr (TypePairSupportDataForProdAccumulation::is_defined) + { + using ScanOpT = CumProdScanOpT; + static constexpr bool include_initial = true; + if constexpr (std::is_same_v) { + using dpctl::tensor::kernels::accumulators::NoOpTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_1d_contig_impl, ScanOpT, + include_initial>; + return fn; + } + else { + using dpctl::tensor::kernels::accumulators::CastTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_1d_contig_impl, + ScanOpT, include_initial>; + return fn; + } + } + else { + return nullptr; + } + } +}; + +template +struct CumProdStridedFactory +{ + fnT get() + { + if constexpr (TypePairSupportDataForProdAccumulation::is_defined) + { + using ScanOpT = CumProdScanOpT; + static constexpr bool include_initial = false; + if constexpr (std::is_same_v) { + using dpctl::tensor::kernels::accumulators::NoOpTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_strided_impl, ScanOpT, + include_initial>; + return fn; + } + else { + using dpctl::tensor::kernels::accumulators::CastTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_strided_impl, + ScanOpT, include_initial>; + return fn; + } + } + else { + return nullptr; + } + } +}; + +template +struct CumProdIncludeInitialStridedFactory +{ + fnT get() + { + if constexpr (TypePairSupportDataForProdAccumulation::is_defined) + { + using ScanOpT = CumProdScanOpT; + static constexpr bool include_initial = true; + if constexpr (std::is_same_v) { + using dpctl::tensor::kernels::accumulators::NoOpTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_strided_impl, ScanOpT, + include_initial>; + return fn; + } + else { + using dpctl::tensor::kernels::accumulators::CastTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_strided_impl, + ScanOpT, include_initial>; + return fn; + } + } + else { + return nullptr; + } + } +}; + +void populate_cumprod_dispatch_tables(void) +{ + td_ns::DispatchTableBuilder + dtb1; + dtb1.populate_dispatch_table(cumprod_1d_contig_dispatch_table); + + td_ns::DispatchTableBuilder + dtb2; + dtb2.populate_dispatch_table(cumprod_strided_dispatch_table); + + td_ns::DispatchTableBuilder + dtb3; + dtb3.populate_dispatch_table( + cumprod_1d_include_initial_contig_dispatch_table); + + td_ns::DispatchTableBuilder + dtb4; + dtb4.populate_dispatch_table( + cumprod_include_initial_strided_dispatch_table); + + return; +} + +} // namespace impl + +void init_cumulative_prod(py::module_ m) +{ + using arrayT = dpctl::tensor::usm_ndarray; + using event_vecT = std::vector; + + using impl::populate_cumprod_dispatch_tables; + populate_cumprod_dispatch_tables(); + + using impl::cumprod_1d_contig_dispatch_table; + using impl::cumprod_strided_dispatch_table; + auto cumprod_pyapi = [&](const arrayT &src, int trailing_dims_to_accumulate, + const arrayT &dst, sycl::queue &exec_q, + const event_vecT &depends = {}) { + return py_accumulate_over_axis( + src, trailing_dims_to_accumulate, dst, exec_q, depends, + cumprod_strided_dispatch_table, cumprod_1d_contig_dispatch_table); + }; + m.def("_cumprod_over_axis", cumprod_pyapi, "", py::arg("src"), + py::arg("trailing_dims_to_accumulate"), py::arg("dst"), + py::arg("sycl_queue"), py::arg("depends") = py::list()); + + using impl::cumprod_1d_include_initial_contig_dispatch_table; + using impl::cumprod_include_initial_strided_dispatch_table; + auto cumprod_include_initial_pyapi = + [&](const arrayT &src, const arrayT &dst, sycl::queue &exec_q, + const event_vecT &depends = {}) { + return py_accumulate_final_axis_include_initial( + src, dst, exec_q, depends, + cumprod_include_initial_strided_dispatch_table, + cumprod_1d_include_initial_contig_dispatch_table); + }; + m.def("_cumprod_final_axis_include_initial", cumprod_include_initial_pyapi, + "", py::arg("src"), py::arg("dst"), py::arg("sycl_queue"), + py::arg("depends") = py::list()); + + auto cumprod_dtype_supported = [&](const py::dtype &input_dtype, + const py::dtype &output_dtype) { + using dpctl::tensor::py_internal::py_accumulate_dtype_supported; + return py_accumulate_dtype_supported(input_dtype, output_dtype, + cumprod_strided_dispatch_table); + }; + m.def("_cumprod_dtype_supported", cumprod_dtype_supported, "", + py::arg("arg_dtype"), py::arg("out_dtype")); +} + +} // namespace dpctl::tensor::py_internal diff --git a/dpctl_ext/tensor/libtensor/source/accumulators/cumulative_prod.hpp b/dpctl_ext/tensor/libtensor/source/accumulators/cumulative_prod.hpp new file mode 100644 index 00000000000..e14bb2c4436 --- /dev/null +++ b/dpctl_ext/tensor/libtensor/source/accumulators/cumulative_prod.hpp @@ -0,0 +1,46 @@ +//***************************************************************************** +// Copyright (c) 2026, Intel Corporation +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// - Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// - Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +//***************************************************************************** +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines functions of dpctl.tensor._tensor_accumulation_impl +// extensions +//===----------------------------------------------------------------------===// + +#pragma once +#include + +namespace py = pybind11; + +namespace dpctl::tensor::py_internal +{ + +extern void init_cumulative_prod(py::module_ m); + +} // namespace dpctl::tensor::py_internal diff --git a/dpctl_ext/tensor/libtensor/source/accumulators/cumulative_sum.cpp b/dpctl_ext/tensor/libtensor/source/accumulators/cumulative_sum.cpp new file mode 100644 index 00000000000..3a0ed6cf3ab --- /dev/null +++ b/dpctl_ext/tensor/libtensor/source/accumulators/cumulative_sum.cpp @@ -0,0 +1,355 @@ +//***************************************************************************** +// Copyright (c) 2026, Intel Corporation +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// - Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// - Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +//***************************************************************************** +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines functions of dpctl.tensor._tensor_accumulation_impl +// extensions +//===----------------------------------------------------------------------===// + +#include +#include +#include +#include + +#include + +#include "dpnp4pybind11.hpp" +#include +#include +#include + +#include "accumulate_over_axis.hpp" +#include "kernels/accumulators.hpp" +#include "utils/type_dispatch_building.hpp" + +namespace py = pybind11; + +namespace dpctl::tensor::py_internal +{ + +namespace td_ns = dpctl::tensor::type_dispatch; + +namespace impl +{ + +using dpctl::tensor::kernels::accumulators::accumulate_1d_contig_impl_fn_ptr_t; +static accumulate_1d_contig_impl_fn_ptr_t + cumsum_1d_contig_dispatch_table[td_ns::num_types][td_ns::num_types]; + +using dpctl::tensor::kernels::accumulators::accumulate_strided_impl_fn_ptr_t; +static accumulate_strided_impl_fn_ptr_t + cumsum_strided_dispatch_table[td_ns::num_types][td_ns::num_types]; + +static accumulate_1d_contig_impl_fn_ptr_t + cumsum_1d_include_initial_contig_dispatch_table[td_ns::num_types] + [td_ns::num_types]; + +static accumulate_strided_impl_fn_ptr_t + cumsum_include_initial_strided_dispatch_table[td_ns::num_types] + [td_ns::num_types]; + +template +struct TypePairSupportDataForSumAccumulation +{ + static constexpr bool is_defined = std::disjunction< + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input int8_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input uint8_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input int16_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input uint16_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input int32_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input uint32_t + td_ns::TypePairDefinedEntry, + td_ns::TypePairDefinedEntry, + + // input int64_t + td_ns::TypePairDefinedEntry, + + // input uint64_t + td_ns::TypePairDefinedEntry, + + // input half + td_ns::TypePairDefinedEntry, + + // input float + td_ns::TypePairDefinedEntry, + + // input double + td_ns::TypePairDefinedEntry, + + // input std::complex + td_ns::TypePairDefinedEntry, + outTy, + std::complex>, + + td_ns::TypePairDefinedEntry, + outTy, + std::complex>, + + // fall-through + td_ns::NotDefinedEntry>::is_defined; +}; + +template +using CumSumScanOpT = std:: + conditional_t, sycl::logical_or, sycl::plus>; + +template +struct CumSum1DContigFactory +{ + fnT get() + { + if constexpr (TypePairSupportDataForSumAccumulation::is_defined) + { + using ScanOpT = CumSumScanOpT; + static constexpr bool include_initial = false; + if constexpr (std::is_same_v) { + using dpctl::tensor::kernels::accumulators::NoOpTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_1d_contig_impl, ScanOpT, + include_initial>; + return fn; + } + else { + using dpctl::tensor::kernels::accumulators::CastTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_1d_contig_impl, + ScanOpT, include_initial>; + return fn; + } + } + else { + return nullptr; + } + } +}; + +template +struct CumSum1DIncludeInitialContigFactory +{ + fnT get() + { + if constexpr (TypePairSupportDataForSumAccumulation::is_defined) + { + using ScanOpT = CumSumScanOpT; + static constexpr bool include_initial = true; + if constexpr (std::is_same_v) { + using dpctl::tensor::kernels::accumulators::NoOpTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_1d_contig_impl, ScanOpT, + include_initial>; + return fn; + } + else { + using dpctl::tensor::kernels::accumulators::CastTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_1d_contig_impl, + ScanOpT, include_initial>; + return fn; + } + } + else { + return nullptr; + } + } +}; + +template +struct CumSumStridedFactory +{ + fnT get() + { + if constexpr (TypePairSupportDataForSumAccumulation::is_defined) + { + using ScanOpT = CumSumScanOpT; + static constexpr bool include_initial = false; + if constexpr (std::is_same_v) { + using dpctl::tensor::kernels::accumulators::NoOpTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_strided_impl, ScanOpT, + include_initial>; + return fn; + } + else { + using dpctl::tensor::kernels::accumulators::CastTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_strided_impl, + ScanOpT, include_initial>; + return fn; + } + } + else { + return nullptr; + } + } +}; + +template +struct CumSumIncludeInitialStridedFactory +{ + fnT get() + { + if constexpr (TypePairSupportDataForSumAccumulation::is_defined) + { + using ScanOpT = CumSumScanOpT; + static constexpr bool include_initial = true; + if constexpr (std::is_same_v) { + using dpctl::tensor::kernels::accumulators::NoOpTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_strided_impl, ScanOpT, + include_initial>; + return fn; + } + else { + using dpctl::tensor::kernels::accumulators::CastTransformer; + fnT fn = dpctl::tensor::kernels::accumulators:: + accumulate_strided_impl, + ScanOpT, include_initial>; + return fn; + } + } + else { + return nullptr; + } + } +}; + +void populate_cumsum_dispatch_tables(void) +{ + td_ns::DispatchTableBuilder + dtb1; + dtb1.populate_dispatch_table(cumsum_1d_contig_dispatch_table); + + td_ns::DispatchTableBuilder + dtb2; + dtb2.populate_dispatch_table(cumsum_strided_dispatch_table); + + td_ns::DispatchTableBuilder + dtb3; + dtb3.populate_dispatch_table( + cumsum_1d_include_initial_contig_dispatch_table); + + td_ns::DispatchTableBuilder + dtb4; + dtb4.populate_dispatch_table(cumsum_include_initial_strided_dispatch_table); + + return; +} + +} // namespace impl + +void init_cumulative_sum(py::module_ m) +{ + using arrayT = dpctl::tensor::usm_ndarray; + using event_vecT = std::vector; + + using impl::populate_cumsum_dispatch_tables; + populate_cumsum_dispatch_tables(); + + using impl::cumsum_1d_contig_dispatch_table; + using impl::cumsum_strided_dispatch_table; + auto cumsum_pyapi = [&](const arrayT &src, int trailing_dims_to_accumulate, + const arrayT &dst, sycl::queue &exec_q, + const event_vecT &depends = {}) { + return py_accumulate_over_axis( + src, trailing_dims_to_accumulate, dst, exec_q, depends, + cumsum_strided_dispatch_table, cumsum_1d_contig_dispatch_table); + }; + m.def("_cumsum_over_axis", cumsum_pyapi, "", py::arg("src"), + py::arg("trailing_dims_to_accumulate"), py::arg("dst"), + py::arg("sycl_queue"), py::arg("depends") = py::list()); + + using impl::cumsum_1d_include_initial_contig_dispatch_table; + using impl::cumsum_include_initial_strided_dispatch_table; + auto cumsum_include_initial_pyapi = + [&](const arrayT &src, const arrayT &dst, sycl::queue &exec_q, + const event_vecT &depends = {}) { + return py_accumulate_final_axis_include_initial( + src, dst, exec_q, depends, + cumsum_include_initial_strided_dispatch_table, + cumsum_1d_include_initial_contig_dispatch_table); + }; + m.def("_cumsum_final_axis_include_initial", cumsum_include_initial_pyapi, + "", py::arg("src"), py::arg("dst"), py::arg("sycl_queue"), + py::arg("depends") = py::list()); + + auto cumsum_dtype_supported = [&](const py::dtype &input_dtype, + const py::dtype &output_dtype) { + using dpctl::tensor::py_internal::py_accumulate_dtype_supported; + return py_accumulate_dtype_supported(input_dtype, output_dtype, + cumsum_strided_dispatch_table); + }; + m.def("_cumsum_dtype_supported", cumsum_dtype_supported, "", + py::arg("arg_dtype"), py::arg("out_dtype")); +} + +} // namespace dpctl::tensor::py_internal diff --git a/dpctl_ext/tensor/libtensor/source/accumulators/cumulative_sum.hpp b/dpctl_ext/tensor/libtensor/source/accumulators/cumulative_sum.hpp new file mode 100644 index 00000000000..5e06b222a3b --- /dev/null +++ b/dpctl_ext/tensor/libtensor/source/accumulators/cumulative_sum.hpp @@ -0,0 +1,46 @@ +//***************************************************************************** +// Copyright (c) 2026, Intel Corporation +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// - Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// - Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +//***************************************************************************** +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines functions of dpctl.tensor._tensor_accumulation_impl +// extensions +//===----------------------------------------------------------------------===// + +#pragma once +#include + +namespace py = pybind11; + +namespace dpctl::tensor::py_internal +{ + +extern void init_cumulative_sum(py::module_ m); + +} // namespace dpctl::tensor::py_internal diff --git a/dpctl_ext/tensor/libtensor/source/tensor_accumulation.cpp b/dpctl_ext/tensor/libtensor/source/tensor_accumulation.cpp new file mode 100644 index 00000000000..faa3fc8b52c --- /dev/null +++ b/dpctl_ext/tensor/libtensor/source/tensor_accumulation.cpp @@ -0,0 +1,43 @@ +//***************************************************************************** +// Copyright (c) 2026, Intel Corporation +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// - Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// - Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +//***************************************************************************** +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines functions of dpctl.tensor._tensor_accumulation_impl +// extensions +//===----------------------------------------------------------------------===// + +#include + +#include "accumulators/accumulators_common.hpp" + +PYBIND11_MODULE(_tensor_accumulation_impl, m) +{ + dpctl::tensor::py_internal::init_accumulator_functions(m); +} diff --git a/dpnp/dpnp_iface_mathematical.py b/dpnp/dpnp_iface_mathematical.py index 06f4fe93625..000c343abdb 100644 --- a/dpnp/dpnp_iface_mathematical.py +++ b/dpnp/dpnp_iface_mathematical.py @@ -1126,7 +1126,7 @@ def cumprod(a, axis=None, dtype=None, out=None): return dpnp_wrap_reduction_call( usm_a, out, - dpt.cumulative_prod, + dpt_ext.cumulative_prod, _get_reduction_res_dt(a, dtype), axis=axis, dtype=dtype, @@ -1218,7 +1218,7 @@ def cumsum(a, axis=None, dtype=None, out=None): return dpnp_wrap_reduction_call( usm_a, out, - dpt.cumulative_sum, + dpt_ext.cumulative_sum, _get_reduction_res_dt(a, dtype), axis=axis, dtype=dtype, @@ -1307,7 +1307,7 @@ def cumulative_prod( return dpnp_wrap_reduction_call( dpnp.get_usm_ndarray(x), out, - dpt.cumulative_prod, + dpt_ext.cumulative_prod, _get_reduction_res_dt(x, dtype), axis=axis, dtype=dtype, @@ -1403,7 +1403,7 @@ def cumulative_sum( return dpnp_wrap_reduction_call( dpnp.get_usm_ndarray(x), out, - dpt.cumulative_sum, + dpt_ext.cumulative_sum, _get_reduction_res_dt(x, dtype), axis=axis, dtype=dtype, diff --git a/dpnp/dpnp_iface_trigonometric.py b/dpnp/dpnp_iface_trigonometric.py index 9894bd30470..460a0dc80f0 100644 --- a/dpnp/dpnp_iface_trigonometric.py +++ b/dpnp/dpnp_iface_trigonometric.py @@ -48,6 +48,7 @@ # TODO: revert to `import dpctl.tensor...` # when dpnp fully migrates dpctl/tensor +import dpctl_ext.tensor as dpt_ext import dpctl_ext.tensor._type_utils as dtu import dpnp import dpnp.backend.extensions.ufunc._ufunc_impl as ufi @@ -934,7 +935,7 @@ def cumlogsumexp( return dpnp_wrap_reduction_call( usm_x, out, - dpt.cumulative_logsumexp, + dpt_ext.cumulative_logsumexp, _get_accumulation_res_dt(x, dtype), axis=axis, dtype=dtype,