from enum import Enum
from numbers import Integral
from typing import Any, Callable, List, NewType, Optional, Self, Tuple, Union
import numpy as np
from mpi4py import MPI
from pylops.utils import DTypeLike, NDArray
from pylops.utils import (
deps as pylops_deps, # avoid namespace crashes with pylops_mpi.utils
)
from pylops.utils._internal import _value_or_sized_to_tuple
from pylops.utils.backend import get_array_module, get_module, get_module_name
from pylops_mpi.Distributed import DistributedMixIn
from pylops_mpi.utils import deps
cupy_message = pylops_deps.cupy_import("the DistributedArray module")
nccl_message = deps.nccl_import("the DistributedArray module")
if nccl_message is None and cupy_message is None:
from cupy.cuda.nccl import NcclCommunicator
from pylops_mpi.utils._nccl import nccl_asarray, nccl_split
else:
NcclCommunicator = Any
NcclCommunicatorType = NewType("NcclCommunicator", NcclCommunicator)
[docs]
class Partition(Enum):
r"""Enum class
Distributing data among different processes.
- ``BROADCAST``: Distributes data to all processes
(ensuring that data is kept consistent across processes)
- ``UNSAFE_BROADCAST``: Distributes data to all processes
(without ensuring that data is kept consistent across processes)
- ``SCATTER``: Distributes unique portions to each process.
"""
BROADCAST = "Broadcast"
UNSAFE_BROADCAST = "UnsafeBroadcast"
SCATTER = "Scatter"
[docs]
def local_split(
global_shape: Tuple, base_comm: MPI.Comm, partition: Partition, axis: int
):
"""To get the local shape from the global shape
Parameters
----------
global_shape : :obj:`tuple`
Shape of the global array.
base_comm : :obj:`MPI.Comm`
Base MPI Communicator.
partition : :obj:`Partition`
Type of partition.
axis : :obj:`int`
Axis of distribution
Returns
-------
local_shape : :obj:`tuple`
Shape of the local array.
"""
if partition in [Partition.BROADCAST, Partition.UNSAFE_BROADCAST]:
local_shape = global_shape
else:
# Split the array
local_shape = list(global_shape)
if base_comm.Get_rank() < (global_shape[axis] % base_comm.Get_size()):
local_shape[axis] = global_shape[axis] // base_comm.Get_size() + 1
else:
local_shape[axis] = global_shape[axis] // base_comm.Get_size()
return tuple(local_shape)
def subcomm_split(
mask, comm: Optional[Union[MPI.Comm, NcclCommunicatorType]] = MPI.COMM_WORLD
):
"""Create new communicators based on mask
This method creates new communicators based on ``mask``.
Parameters
----------
mask : :obj:`list`
Mask defining subsets of ranks to consider when performing 'global'
operations on the distributed array such as dot product or norm.
comm : :obj:`mpi4py.MPI.Comm` or `cupy.cuda.nccl.NcclCommunicator`, optional
A Communicator over which array is distributed
Defaults to ``mpi4py.MPI.COMM_WORLD``.
Returns:
-------
sub_comm : :obj:`mpi4py.MPI.Comm` or :obj:`cupy.cuda.nccl.NcclCommunicator`
Subcommunicator according to mask
"""
# NcclCommunicatorType cannot be used with isinstance() so check the negate of MPI.Comm
if deps.nccl_enabled and not isinstance(comm, MPI.Comm):
sub_comm = nccl_split(mask)
else:
rank = comm.Get_rank()
sub_comm = comm.Split(color=mask[rank], key=rank)
return sub_comm
def to_scalar(x) -> Union[bool, int, float, complex]:
"""Convert a size-1 array (or a scalar) into a Python scalar
Reductions over an entire distributed array (such as dot products and
norms) return a size-1 buffer for both the numpy and cupy backends. Such
buffers are converted into Python scalars to be consistent with the
return type of their numpy counterparts (e.g., :func:`numpy.linalg.norm`)
and to avoid deprecated implicit array-to-scalar conversions in the
callers. Note that the type of the scalar is preserved (e.g., a complex
dot product returns a :obj:`complex`).
Parameters
----------
x : :obj:`numpy.ndarray` or :obj:`cupy.ndarray` or :obj:`float`
Size-1 array (or scalar) to convert.
Returns
-------
scalar : :obj:`bool` or :obj:`int` or :obj:`float` or :obj:`complex`
Converted scalar.
"""
return x.item() if hasattr(x, "item") else x
[docs]
class DistributedArray(DistributedMixIn):
r"""Distributed Numpy Arrays
Multidimensional NumPy-like distributed arrays.
It brings NumPy arrays to high-performance computing.
.. warning:: When setting the partition of the DistributedArray to
:obj:`pylops_mpi.Partition.BROADCAST`, it is crucial to be aware
that any attempts to make arrays different from rank to rank will be
overwritten by the actions of rank 0. This is accomplished internally
by broadcasting the content of rank 0 every time a modification of
the array is attempted. Such a behaviour does however incur a cost
as communication may be not needed if the user ensures not to modify
the content of the array in different ranks in a different way. To
avoid broadcasting, one can use :obj:`pylops_mpi.Partition.UNSAFE_BROADCAST`
instead.
Parameters
----------
global_shape : :obj:`tuple` or :obj:`int`
Shape of the global array.
base_comm : :obj:`mpi4py.MPI.Comm`, optional
MPI Communicator over which array is distributed.
Defaults to ``mpi4py.MPI.COMM_WORLD``.
base_comm_nccl : :obj:`cupy.cuda.nccl.NcclCommunicator`, optional
NCCL Communicator over which array is distributed. Whenever NCCL
Communicator is provided, the base_comm will be set to MPI.COMM_WORLD.
partition : :obj:`Partition`, optional
Broadcast, UnsafeBroadcast, or Scatter the array. Defaults to ``Partition.SCATTER``.
axis : :obj:`int`, optional
Axis along which distribution occurs. Defaults to ``0``.
local_array : :obj:`numpy.ndarray` or :obj:`cupy.ndarray`, optional
Existing local array to use as the underlying storage instead of
allocating a new one. Its shape must match the expected local shape
for the current rank. When provided, the local array's dtype and engine
(numpy or cupy) are used, and the ``dtype`` and ``engine`` arguments
are ignored.
local_shapes : :obj:`list`, optional
List of tuples or integers representing local shapes at each rank.
mask : :obj:`list`, optional
Mask defining subsets of ranks to consider when performing 'global'
operations on the distributed array such as dot product or norm.
engine : :obj:`str`, optional
Engine used to store array (``numpy`` or ``cupy``)
dtype : :obj:`str`, optional
Type of elements in input array. Defaults to ``numpy.float64``.
Raises
------
IndexError
If ``axis`` is out of bounds for ``global_shape``.
ValueError
If ``partition`` is not one of :obj:`Partition.BROADCAST`, :obj:`Partition.UNSAFE_BROADCAST`, or
:obj:`Partition.SCATTER`.
ValueError
If ``base_comm_nccl`` is provided while ``engine`` is not ``"cupy"``.
ValueError
If ``local_array`` is provided and its shape does not match the
expected local shape for the current rank.
"""
def __init__(
self,
global_shape: Union[Tuple, Integral],
base_comm: Optional[MPI.Comm] = MPI.COMM_WORLD,
base_comm_nccl: Optional[NcclCommunicatorType] = None,
partition: Partition = Partition.SCATTER,
axis: int = 0,
local_array: Optional[NDArray] = None,
local_shapes: Optional[List[Union[Tuple, Integral]]] = None,
mask: Optional[List[Integral]] = None,
engine: Optional[str] = "numpy",
dtype: Optional[DTypeLike] = np.float64,
):
if isinstance(global_shape, Integral):
global_shape = (global_shape,)
if len(global_shape) <= axis:
raise IndexError(
f"Axis {axis} out of range for DistributedArray "
f"of shape {global_shape}"
)
if partition not in Partition:
raise ValueError(
f"Should be either {Partition.BROADCAST}, "
f"{Partition.UNSAFE_BROADCAST} or {Partition.SCATTER}"
)
self._engine = (
engine
if local_array is None
else get_module_name(get_array_module(local_array))
)
if base_comm_nccl and self._engine != "cupy":
raise ValueError("NCCL Communicator only works with engine `cupy`")
self.dtype = dtype if local_array is None else local_array.dtype
self._global_shape = _value_or_sized_to_tuple(global_shape)
self._base_comm_nccl = base_comm_nccl
if base_comm_nccl is None:
self._base_comm = base_comm
else:
self._base_comm = MPI.COMM_WORLD
self._partition = partition
self._axis = axis
self._mask = mask
self._sub_comm = (
(base_comm if base_comm_nccl is None else base_comm_nccl)
if mask is None
else subcomm_split(
mask, (base_comm if base_comm_nccl is None else base_comm_nccl)
)
)
local_shapes = (
local_shapes
if local_shapes is None
else [_value_or_sized_to_tuple(local_shape) for local_shape in local_shapes]
)
self._check_local_shapes(local_shapes)
self._local_shapes = local_shapes if local_shapes else None
self._local_shape = (
local_shapes[self.rank]
if local_shapes
else local_split(global_shape, base_comm, partition, axis)
)
if local_array is None:
self._local_array = get_module(self._engine).empty(
shape=self.local_shape,
dtype=self.dtype,
)
else:
if local_array.shape != self.local_shape:
raise ValueError(
f"local_array has shape {local_array.shape}, "
f"expected {self.local_shape}"
)
self._local_array = local_array
def __getitem__(self, index):
return self.local_array[index]
def __setitem__(self, index, value):
"""Setter Method
`Partition.SCATTER` - Local Arrays are assigned their
unique values.
`Partition.UNSAFE_BROADCAST` - Local arrays are updated on each
rank without communication across ranks.
`Partition.BROADCAST` - The value at rank-0 is broadcasted
and is assigned to all the ranks.
Parameters
----------
index : :obj:`int` or :obj:`slice`
Represents the index positions where a value needs to be assigned.
value : :obj:`int` or :obj:`numpy.ndarray`
Represents the value that will be assigned to the local array at
the specified index positions.
"""
if self.partition is Partition.BROADCAST:
ncp = get_module(self.engine)
view = self.local_array[index]
buf = ncp.empty(view.shape, dtype=self.dtype)
if self.rank == 0:
buf[...] = value
buf = self._bcast(
self.base_comm, self.base_comm_nccl, buf, root=0, engine=self.engine
)
self.local_array[index] = buf
else:
self.local_array[index] = value
@property
def global_shape(self):
"""Global Shape of the array
Returns
-------
global_shape : :obj:`tuple`
"""
return self._global_shape
@property
def base_comm(self):
"""Base MPI Communicator
Returns
-------
base_comm : :obj:`MPI.Comm`
"""
return self._base_comm
@property
def base_comm_nccl(self):
"""Base NCCL Communicator
Returns
-------
base_comm : :obj:`cupy.cuda.nccl.NcclCommunicator`
"""
return self._base_comm_nccl
@property
def local_shape(self):
"""Local Shape of the Distributed array
Returns
-------
local_shape : :obj:`tuple`
"""
return self._local_shape
@property
def mask(self):
"""Mask of the Distributed array
Returns
-------
mask : :obj:`list`
"""
return self._mask
@property
def engine(self):
"""Engine of the Distributed Array
Returns
-------
engine : :obj:`str`
Engine
"""
return self._engine
@property
def local_array(self):
"""View of the Local Array
Returns
-------
local_array : :obj:`numpy.ndarray`
"""
return self._local_array
@property
def rank(self):
"""Rank of the current process
Returns
-------
rank : :obj:`int`
"""
return self.base_comm.Get_rank()
@property
def size(self):
"""Total number of processes
Size of parallel environment
Returns
-------
size : :obj:`int`
"""
return self.base_comm.Get_size()
@property
def axis(self):
"""Axis along which distribution occurs
Returns
-------
axis : :obj:`int`
"""
return self._axis
@property
def ndim(self):
"""Number of dimensions of the global array
Returns
-------
ndim : :obj:`int`
"""
return len(self.global_shape)
@property
def partition(self):
"""Type of Distribution
Returns
-------
partition_type : :obj:`str`
"""
return self._partition
@property
def local_shapes(self):
"""Gather Local shapes from all ranks
The shapes are gathered (collective) the first time this property is
accessed, and cached afterwards.
Returns
-------
local_shapes : :obj:`list`
"""
if self._local_shapes is None:
if deps.nccl_enabled and getattr(self, "base_comm_nccl"):
self._local_shapes = self._nccl_local_shapes(False)
else:
self._local_shapes = self._allgather(
self.base_comm, self.base_comm_nccl, self.local_shape
)
return self._local_shapes
@property
def sub_comm(self):
"""MPI Sub-Communicator
Returns
-------
sub_comm : :obj:`MPI.Comm` or `cupy.cuda.nccl.NcclCommunicator`
"""
return self._sub_comm
def asarray(self, masked: bool = False):
"""Global view of the array
Gather all the local arrays from base communicator or subcommunicator
Parameters
----------
masked : :obj:`bool`
Return local arrays of the subcommunicator (`True`) or base communicator (`False`).
Returns
-------
final_array : :obj:`numpy.ndarray`
Global Array gathered at all ranks
"""
# Since the global array was replicated at all ranks
if self.partition in [Partition.BROADCAST, Partition.UNSAFE_BROADCAST]:
# Get only self.local_array.
return self.local_array
if deps.nccl_enabled and getattr(self, "base_comm_nccl"):
return nccl_asarray(
self.sub_comm if masked else self.base_comm_nccl,
self.local_array,
self._nccl_local_shapes(masked),
self.axis,
)
else:
# Gather all the local arrays and apply concatenation.
if masked:
final_array = self._allgather_subcomm(
self.sub_comm,
self.base_comm_nccl,
self.local_array,
engine=self.engine,
)
else:
final_array = self._allgather(
self.base_comm,
self.base_comm_nccl,
self.local_array,
engine=self.engine,
)
return np.concatenate(final_array, axis=self.axis)
@classmethod
def to_dist(
cls,
x: NDArray,
base_comm: MPI.Comm = MPI.COMM_WORLD,
base_comm_nccl: NcclCommunicatorType = None,
partition: Partition = Partition.SCATTER,
axis: int = 0,
local_shapes: Optional[List[Tuple]] = None,
mask: Optional[List[Integral]] = None,
):
"""Convert A Global Array to a Distributed Array
Parameters
----------
x : :obj:`numpy.ndarray`
Global array.
base_comm : :obj:`MPI.Comm`, optional
MPI base communicator
base_comm_nccl : :obj:`cupy.cuda.nccl.NcclCommunicator`, optional
NCCL base communicator
partition : :obj:`Partition`, optional
Distributes the array, Defaults to ``Partition.Scatter``.
axis : :obj:`int`, optional
Axis of Distribution
local_shapes : :obj:`list`, optional
Local Shapes at each rank.
mask : :obj:`list`, optional
Mask defining subsets of ranks to consider when performing 'global'
operations on the distributed array such as dot product or norm.
Returns
-------
dist_array : :obj:`DistributedArray`
Distributed Array of the Global Array
"""
dist_array = DistributedArray(
global_shape=x.shape,
base_comm=base_comm,
base_comm_nccl=base_comm_nccl,
partition=partition,
axis=axis,
local_shapes=local_shapes,
mask=mask,
engine=get_module_name(get_array_module(x)),
dtype=x.dtype,
)
if partition in [Partition.BROADCAST, Partition.UNSAFE_BROADCAST]:
dist_array[:] = x
else:
slices = [slice(None)] * x.ndim
local_shapes = np.append(
[0],
dist_array._allgather(
base_comm, base_comm_nccl, dist_array.local_shape[axis]
),
)
sum_shapes = np.cumsum(local_shapes)
slices[axis] = slice(
sum_shapes[dist_array.rank], sum_shapes[dist_array.rank + 1], None
)
dist_array[:] = x[tuple(slices)]
return dist_array
def redistribute(self, axis: int):
"""Redistribute the array along the specified axis.
If the array is SCATTER-partitioned and the requested axis differs
from the current one, this method performs an all-to-all data exchange
across ranks to realign the distribution along `axis`. It slices the
local data into per-destination chunks, exchanges them using
send/receive operations, and concatenates the received buffers to
form the new local array.
If the original distributed array has unbalanced local shapes,
the redistributed array will be balanced along the new axis.
Parameters
----------
axis: :obj:`int`
Redistribute the Distributed Array along this axs.
Returns
-------
redist_array : :obj:`DistributedArray`
Distributed Array
"""
if self.axis == axis or self.partition is not Partition.SCATTER:
return self
ncp = get_module(self.engine)
redist_array = DistributedArray(
global_shape=self.global_shape,
base_comm=self.base_comm,
base_comm_nccl=self.base_comm_nccl,
mask=self.mask,
axis=axis,
engine=self.engine,
dtype=self.dtype,
)
counts_from = [x[self.axis] for x in self.local_shapes]
counts_to = [x[axis] for x in redist_array.local_shapes]
offsets_to = ncp.cumsum(ncp.array([0] + counts_to[:-1]))
send_slices = []
for r in range(self.size):
start = offsets_to[r]
end = start + counts_to[r]
slicer = [slice(None)] * self.local_array.ndim
slicer[axis] = slice(start, end)
send_slices.append(self.local_array[tuple(slicer)].copy())
recv_list = []
sender_shape = list(self.global_shape)
for r in range(self.size):
sender_shape[self.axis] = counts_from[r]
sender_shape[axis] = counts_to[self.rank]
recv_buf = ncp.empty(sender_shape, dtype=self.dtype)
if r == self.rank:
recv_buf[:] = send_slices[self.rank]
else:
recv_buf = self._sendrecv(
base_comm=self.base_comm,
base_comm_nccl=self.base_comm_nccl,
sendbuf=send_slices[r],
dest=r,
sendtag=0,
recvbuf=recv_buf,
source=r,
recvtag=0,
engine=self.engine,
)
recv_list.append(recv_buf)
redist_array[:] = ncp.concatenate(recv_list, axis=self.axis)
return redist_array
def _check_local_shapes(self, local_shapes):
"""Check if the local shapes align with the global shape"""
if local_shapes:
if len(local_shapes) != self.size:
raise ValueError(
f"Length of local shapes is not equal to number of processes; "
f"{len(local_shapes)} != {self.size}"
)
# Check if local shape == global shape
if (
self.partition in [Partition.BROADCAST, Partition.UNSAFE_BROADCAST]
and local_shapes[self.rank] != self.global_shape
):
raise ValueError(
f"Local shape is not equal to global shape at rank = {self.rank};"
f"{local_shapes[self.rank]} != {self.global_shape}"
)
elif self.partition is Partition.SCATTER:
local_shape = local_shapes[self.rank]
# Check if local shape sum up to global shape and other dimensions align with global shape
if self.base_comm.allreduce(
local_shape[self.axis]
) != self.global_shape[self.axis] or not np.array_equal(
np.delete(local_shape, self.axis),
np.delete(self.global_shape, self.axis),
):
raise ValueError(
f"Local shapes don't align with the global shape;"
f"{local_shapes} != {self.global_shape}"
)
def _check_partition_shape(self, dist_array):
"""Check Partition and Local Shape of the Array"""
if self.partition != dist_array.partition:
raise ValueError("Partition of both the arrays must be same")
if self.local_shape != dist_array.local_shape:
raise ValueError(
f"Local Array Shape Mismatch - "
f"{self.local_shape} != {dist_array.local_shape}"
)
def _check_mask(self, dist_array):
"""Check mask of the Array"""
if not np.array_equal(self.mask, dist_array.mask):
raise ValueError("Mask of both the arrays must be same")
def _nccl_local_shapes(self, masked: bool):
"""Get the the list of shapes of every GPU in the communicator"""
# gather tuple of shapes from every rank within thee communicator
# and copy from GPU to CPU
if masked:
all_tuples = self._allgather_subcomm(
self.sub_comm, self.base_comm_nccl, self.local_shape
).get()
else:
all_tuples = self._allgather(
self.base_comm, self.base_comm_nccl, self.local_shape
).get()
# NCCL returns the flat array that packs every tuple as 1-dimensional array
# unpack each tuple from each rank
tuple_len = len(self.local_shape)
local_shapes = [
tuple(all_tuples[i : i + tuple_len])
for i in range(0, len(all_tuples), tuple_len)
]
return local_shapes
def _like(self, local_array: Optional[NDArray] = None, dtype=None):
"""Create a DistributedArray with the same metadata as ``self``
The metadata comes from an array that is already valid, so no
collective communication is performed and the sub-communicator is
reused. The only check is a local one on the shape of ``local_array``.
Parameters
----------
local_array : :obj:`numpy.ndarray` or :obj:`cupy.ndarray`, optional
Local array to use as storage, whose shape must be equal to the
local shape of ``self``. Its dtype becomes the dtype of the new
array (``dtype`` is ignored). If ``None``, an uninitialized array
is allocated.
dtype : :obj:`str`, optional
Type of the elements of the allocated array. Only used when
``local_array`` is ``None``. If ``None``, it defaults to
``self.dtype``.
Raises
------
ValueError
If ``local_array`` is provided and its shape does not match the
local shape of ``self``.
"""
arr = object.__new__(type(self))
arr._global_shape = self._global_shape
arr._base_comm = self._base_comm
arr._base_comm_nccl = self._base_comm_nccl
arr._partition = self._partition
arr._axis = self._axis
arr._mask = self._mask
arr._sub_comm = self._sub_comm
arr._local_shape = self._local_shape
arr._local_shapes = self._local_shapes
arr._engine = self._engine
if local_array is None:
arr.dtype = self.dtype if dtype is None else dtype
arr._local_array = get_module(self._engine).empty(
self._local_shape, dtype=arr.dtype
)
else:
if local_array.shape != self._local_shape:
raise ValueError(
f"local_array has shape {local_array.shape}, "
f"expected {self._local_shape}"
)
arr.dtype = local_array.dtype
arr._local_array = local_array
return arr
def _ufunc(
self,
ufunc: Callable,
other: Union[float, int, NDArray, Self],
out: Optional[Self] = None,
) -> Self:
"""Apply a binary ufunc between ``self`` and ``other``
The ufunc is applied to the local arrays, optionally writing directly
into ``out``. If the cast into ``out`` is unsafe (e.g., complex into
real), the result is computed first and then assigned to ``out``,
which casts it.
Parameters
----------
ufunc : :obj:`callable`
Binary ufunc of the array module of the engine (e.g.,
``numpy.add``).
other : :obj:`pylops_mpi.DistributedArray`, scalar or array-like
Second operand. If a :obj:`pylops_mpi.DistributedArray`, it must
have the same partition, shape and mask as ``self``.
out : :obj:`pylops_mpi.DistributedArray`, optional
Array where the result is stored (it can be ``self`` or
``other``). If ``None``, a new array is created.
Returns
-------
out : :obj:`pylops_mpi.DistributedArray`
Result of the operation. This is the ``out`` array if provided,
otherwise a newly created array.
"""
if isinstance(other, DistributedArray):
self._check_partition_shape(other)
self._check_mask(other)
other = other.local_array
if out is None:
out = self._like()
if self.partition is not Partition.BROADCAST:
try:
# single pass, no temporary for safe cast
ufunc(self.local_array, other, out=out.local_array)
return out
except TypeError:
# unsafe cast into out (e.g., complex into real): fall back
# to the assignment, which casts
pass
out[:] = ufunc(self.local_array, other)
return out
def __neg__(self):
out = self._like()
if self.partition is Partition.BROADCAST:
out[:] = -self.local_array
else:
get_module(self.engine).negative(self.local_array, out=out.local_array)
return out
def __add__(self, x):
return self.add(x)
def __iadd__(self, x):
return self.iadd(x)
def __sub__(self, x):
return self.subtract(x)
def __isub__(self, x):
return self.subtract(x, out=self)
def __mul__(self, x):
return self.multiply(x)
def __imul__(self, x):
return self.multiply(x, out=self)
def __rmul__(self, x):
return self.multiply(x)
def add(
self, dist_array: Union[float, int, Self], out: Optional[Self] = None
) -> Self:
"""Distributed Addition of arrays
Parameters
----------
dist_array : :obj:`pylops_mpi.DistributedArray` or scalar
Array to add.
out : :obj:`pylops_mpi.DistributedArray`, optional
Array where the result is stored (it can be ``self`` or
``dist_array``). If ``None``, a new array is created.
Returns
-------
out : :obj:`pylops_mpi.DistributedArray`
Result of the addition. This is the ``out`` array if provided,
otherwise a newly created array.
"""
return self._ufunc(get_module(self.engine).add, dist_array, out)
def iadd(self, dist_array: Union[float, int, Self]) -> Self:
"""Distributed In-place Addition of arrays
Parameters
----------
dist_array : :obj:`pylops_mpi.DistributedArray` or scalar
Array to add.
Returns
-------
out : :obj:`pylops_mpi.DistributedArray`
This array (``self``), updated in place with the result of the addition.
"""
return self.add(dist_array, out=self)
def subtract(
self, dist_array: Union[float, int, Self], out: Optional[Self] = None
) -> Self:
"""Distributed Subtraction of arrays
Parameters
----------
dist_array : :obj:`pylops_mpi.DistributedArray` or scalar
Array to subtract.
out : :obj:`pylops_mpi.DistributedArray`, optional
Array where the result is stored (it can be ``self`` or
``dist_array``). If ``None``, a new array is created.
Returns
-------
out : :obj:`pylops_mpi.DistributedArray`
Result of the subtraction. This is the ``out`` array if provided,
otherwise a newly created array.
"""
return self._ufunc(get_module(self.engine).subtract, dist_array, out)
def multiply(
self, dist_array: Union[float, int, Self], out: Optional[Self] = None
) -> Self:
"""Distributed Element-wise multiplication
Parameters
----------
dist_array : :obj:`pylops_mpi.DistributedArray` or scalar
Array to multiply by.
out : :obj:`pylops_mpi.DistributedArray`, optional
Array where the result is stored (it can be ``self`` or
``dist_array``). If ``None``, a new array is created.
Returns
-------
out : :obj:`pylops_mpi.DistributedArray`
Result of the multiplication. This is the ``out`` array if
provided, otherwise a newly created array.
"""
return self._ufunc(get_module(self.engine).multiply, dist_array, out)
def _local_dot(self, dist_array: Self, vdot: bool = False) -> Union[float, complex]:
"""Dot product of the local arrays (no communication)
Parameters
----------
dist_array : :obj:`pylops_mpi.DistributedArray`
The distributed array whose local array is used as the second
operand of the dot product.
vdot : :obj:`bool`, optional
If ``True``, conjugate the local array of ``self`` (like
:func:`numpy.vdot`). If ``False``, compute the standard dot
product (like :func:`numpy.dot`). Defaults to ``False``.
Returns
-------
result : :obj:`float` or :obj:`complex`
Dot product of the flattened local arrays on this rank.
"""
ncp = get_module(self.engine)
dot_func = ncp.vdot if vdot else ncp.dot
return dot_func(self.local_array.ravel(), dist_array.local_array.ravel())
def dot(self, dist_array: Self, vdot: bool = False) -> Union[float, complex]:
"""Distributed dot-product
Compute the distributed dot product between this array and another
distributed array.
Parameters
----------
dist_array : :obj:`pylops_mpi.DistributedArray`
The distributed array with which to compute the dot product.
It must have a compatible shape and partitioning.
vdot : bool, optional, Defaults to `False`
If True, compute the complex conjugate dot product (like numpy.vdot),
where the first argument is conjugated. If False, compute the standard
dot product.
Returns
-------
result : :obj:`float` or :obj:`complex`
The result of the dot product across all ranks. This is reduced across all processes.
"""
self._check_partition_shape(dist_array)
self._check_mask(dist_array)
# Convert to Partition.SCATTER if Partition.BROADCAST
x = (
DistributedArray.to_dist(
x=self.local_array,
base_comm=self.base_comm,
base_comm_nccl=self.base_comm_nccl,
)
if self.partition in [Partition.BROADCAST, Partition.UNSAFE_BROADCAST]
else self
)
y = (
DistributedArray.to_dist(
x=dist_array.local_array,
base_comm=self.base_comm,
base_comm_nccl=self.base_comm_nccl,
)
if self.partition in [Partition.BROADCAST, Partition.UNSAFE_BROADCAST]
else dist_array
)
# Note that _allreduce_subcomm returns a size-1 array, which is
# converted into a scalar to be consistent with numpy.dot/numpy.vdot
return to_scalar(
self._allreduce_subcomm(
self.sub_comm,
self.base_comm_nccl,
x._local_dot(y, vdot),
engine=self.engine,
)
)
def _local_sqnorm(self, local_array: NDArray) -> float:
"""Squared 2-norm of the local array (no communication)
Parameters
----------
local_array : :obj:`numpy.ndarray`
Local array whose squared 2-norm is computed.
Returns
-------
result : :obj:`float`
Sum of the squared absolute values of the flattened array on
this rank, as ``float64``.
"""
ncp = get_module(self.engine)
local = local_array.ravel()
if ncp.iscomplexobj(local):
# real view of the same buffer: sum of squares of real and
# imaginary parts (overflows to inf rather than nan)
local = local.view(np.finfo(local.dtype).dtype)
return ncp.vdot(local, local).astype(ncp.float64)
def _compute_vector_norm(
self, local_array: NDArray, axis: int, ord: Optional[int] = None
):
"""Compute Vector norm using MPI
Parameters
----------
local_array : :obj:`numpy.ndarray`
Local Array at each rank
axis : :obj:`int`
Axis along which norm is computed
ord : :obj:`int`, optional
Order of the norm
"""
# Compute along any axis
ncp = get_module(self.engine)
ord = 2 if ord is None else ord
if local_array.ndim == 1:
recv_buf = ncp.empty(shape=1, dtype=ncp.float64)
else:
global_shape = list(self.global_shape)
global_shape[axis] = 1
recv_buf = ncp.empty(shape=global_shape, dtype=ncp.float64)
if ord in ["fro", "nuc"]:
raise ValueError(f"norm-{ord} not possible for vectors")
elif ord == 2 and local_array.ndim == 1:
# 2-norm of a vector: no float64 temporaries
recv_buf = self._allreduce_subcomm(
self.sub_comm,
self.base_comm_nccl,
self._local_sqnorm(local_array),
engine=self.engine,
)
recv_buf = ncp.sqrt(recv_buf)
elif ord == 0:
# Count non-zero then sum reduction
recv_buf = self._allreduce_subcomm(
self.sub_comm,
self.base_comm_nccl,
ncp.count_nonzero(local_array, axis=axis).astype(ncp.float64),
engine=self.engine,
)
elif ord == ncp.inf:
# Calculate max followed by max reduction
# CuPy + non-CUDA-aware MPI does not work well with buffered communication, particularly
# with MAX, MIN operator. Here we copy the array back to CPU, transfer, and copy them back to GPUs
send_buf = ncp.max(ncp.abs(local_array), axis=axis).astype(ncp.float64)
if (
self.engine == "cupy"
and self.base_comm_nccl is None
and not deps.cuda_aware_mpi_enabled
):
# CuPy + non-CUDA-aware MPI: This will call non-buffered communication
# which return a list of object - must be copied back to a GPU memory.
recv_buf = self._allreduce_subcomm(
self.sub_comm,
self.base_comm_nccl,
send_buf.get(),
recv_buf.get(),
op=MPI.MAX,
engine=self.engine,
)
recv_buf = ncp.asarray(ncp.squeeze(recv_buf, axis=axis))
else:
recv_buf = self._allreduce_subcomm(
self.sub_comm,
self.base_comm_nccl,
send_buf,
recv_buf,
op=MPI.MAX,
engine=self.engine,
)
# TODO (tharitt): In current implementation, there seems to be a semantic difference between Buffered MPI and NCCL
# the (1, size) is collapsed to (size, ) with buffered MPI while NCCL retains it.
# There may be a way to unify it - may be something to do with how we allocate the recv_buf.
if self.base_comm_nccl:
recv_buf = ncp.squeeze(recv_buf, axis=axis)
elif ord == -ncp.inf:
# Calculate min followed by min reduction
# See the comment above in +infinity norm
send_buf = ncp.min(ncp.abs(local_array), axis=axis).astype(ncp.float64)
if (
self.engine == "cupy"
and self.base_comm_nccl is None
and not deps.cuda_aware_mpi_enabled
):
recv_buf = self._allreduce_subcomm(
self.sub_comm,
self.base_comm_nccl,
send_buf.get(),
recv_buf.get(),
op=MPI.MIN,
engine=self.engine,
)
recv_buf = ncp.asarray(ncp.squeeze(recv_buf, axis=axis))
else:
recv_buf = self._allreduce_subcomm(
self.sub_comm,
self.base_comm_nccl,
send_buf,
recv_buf,
op=MPI.MIN,
engine=self.engine,
)
if self.base_comm_nccl:
recv_buf = ncp.asarray(ncp.squeeze(recv_buf, axis=axis))
else:
recv_buf = self._allreduce_subcomm(
self.sub_comm,
self.base_comm_nccl,
ncp.sum(ncp.abs(ncp.float_power(local_array, ord)), axis=axis),
engine=self.engine,
)
recv_buf = ncp.power(recv_buf, 1.0 / ord)
return recv_buf
def zeros_like(self):
"""Creates a copy of the DistributedArray filled with zeros"""
arr = self._like()
arr.local_array[...] = 0.0
return arr
def norm(
self, ord: Optional[int] = None, axis: Optional[int] = None
) -> Union[float, NDArray]:
"""Distributed numpy.linalg.norm method
Parameters
----------
ord : :obj:`int`, optional
Order of the norm.
axis : :obj:`int`, optional
Axis along which vector norm needs to be computed.
Default is `None`, meaning the entire array is treated as a flattened vector.
Returns
-------
norm : :obj:`float` or :obj:`numpy.ndarray`
The computed norm of the distributed array. A scalar is returned
when ``axis=None``, an array otherwise.
"""
# Convert to Partition.SCATTER if Partition.BROADCAST
x = (
DistributedArray.to_dist(
x=self.local_array,
base_comm=self.base_comm,
base_comm_nccl=self.base_comm_nccl,
)
if self.partition in [Partition.BROADCAST, Partition.UNSAFE_BROADCAST]
else self
)
if axis is None:
# Flatten the local arrays and calculate norm. Note that
# _compute_vector_norm returns a size-1 array, which is converted
# into a scalar to be consistent with numpy.linalg.norm
return to_scalar(
x._compute_vector_norm(x.local_array.ravel(), axis=0, ord=ord)
)
if axis >= self.ndim:
raise ValueError(
f"axis={axis} is out of range for array of dimension {self.ndim}"
)
if self.axis != axis:
ncp = get_module(self.engine)
norm_axis = self.axis - 1 if axis < self.axis else self.axis
recv_buf = self._allgather(
self.base_comm,
self.base_comm_nccl,
ncp.linalg.norm(self.local_array, axis=axis, ord=ord),
engine=self.engine,
)
return ncp.concatenate(recv_buf, axis=norm_axis)
# Calculate vector norm along the axis
return x._compute_vector_norm(x.local_array, axis=axis, ord=ord)
def conj(self):
"""Distributed conj() method"""
return self._like(local_array=self.local_array.conj())
def copy(self):
"""Creates a copy of the DistributedArray"""
return self._like(local_array=self.local_array.copy())
def ravel(self, order: Optional[str] = "C"):
"""Return a flattened DistributedArray
Parameters
----------
order : :obj:`str`, optional
Order in which array needs to be flattened.
{'C','F', 'A', 'K'}
Returns
-------
arr : :obj:`pylops_mpi.DistributedArray`
Flattened 1-D DistributedArray
"""
local_shapes = [
(np.prod(local_shape, axis=-1),) for local_shape in self.local_shapes
]
local_array = np.ravel(self.local_array, order=order)
arr = DistributedArray(
global_shape=np.prod(self.global_shape),
base_comm=self.base_comm,
base_comm_nccl=self.base_comm_nccl,
local_array=local_array,
local_shapes=local_shapes,
mask=self.mask,
partition=self.partition,
engine=self.engine,
dtype=self.dtype,
)
return arr
def reshape(self, local_shape: Tuple, axis: Optional[int] = 0):
"""Return a reshaped DistributedArray
Parameters
----------
local_shape : :obj:`tuple`
Shape of the local array on each MPI rank.
axis : :obj:`int`, optional
Distribution axis in the reshaped global array. Defaults to ``0``.
Raises
------
ValueError
If the provided local shapes are incompatible with the array partitioning.
For :class:`Partition.SCATTER`, local shapes must agree on every non-distribution axis.
For :class:`Partition.BROADCAST` and :class:`Partition.UNSAFE_BROADCAST`, all local shapes must be identical
across ranks.
"""
local_shapes = self.base_comm.allgather(local_shape)
global_shape = list(local_shapes[0])
ref_shape = local_shapes[0]
if self.partition is Partition.SCATTER:
if (
local_shape[:axis] != ref_shape[:axis]
or local_shape[axis + 1 :] != ref_shape[axis + 1 :]
):
raise ValueError(
f"All local shapes must match on every axis except axis={axis}. "
f"Got {local_shape} in rank {self.rank} and {ref_shape} in rank 0."
)
global_shape = list(ref_shape)
global_shape[axis] = sum(ls[axis] for ls in local_shapes)
else:
if local_shape != ref_shape:
raise ValueError(
"All local shapes must be identical for Partition.BROADCAST and Partition.UNSAFE_BROADCAST. "
f"Got {local_shape} in rank {self.rank} and {ref_shape} in rank 0."
)
local_array = self.local_array.reshape(local_shapes[self.rank])
arr = DistributedArray(
global_shape=tuple(global_shape),
base_comm=self.base_comm,
base_comm_nccl=self.base_comm_nccl,
local_shapes=local_shapes,
mask=self.mask,
partition=self.partition,
engine=self.engine,
dtype=self.dtype,
local_array=local_array,
)
return arr
def empty_like(self):
"""Creates an empty like DistributedArray with uninitialized values"""
return self._like()
def add_ghost_cells(
self, cells_front: Optional[int] = None, cells_back: Optional[int] = None
):
"""Add ghost cells to the DistributedArray along the axis
of partition at each rank.
Parameters
----------
cells_front : :obj:`int`, optional
Number of cells to be added from the previous process
to the start of the array at each rank. Defaults to ``None``.
cells_back : :obj:`int`, optional
Number of cells to be added from the next process
to the back of the array at each rank. Defaults to ``None``.
Returns
-------
ghosted_array : :obj:`numpy.ndarray`
Ghosted Array
"""
ghosted_array = self.local_array.copy()
ncp = get_module(self.engine)
if cells_front is not None:
# cells_front is small array of int. Explicitly use MPI
total_cells_front = self.base_comm.allgather(cells_front) + [0]
# Read cells_front which needs to be sent to rank + 1(cells_front for rank + 1)
cells_front = total_cells_front[self.rank + 1]
send_buf = ncp.take(
self.local_array, ncp.arange(-cells_front, 0), axis=self.axis
)
recv_shapes = self.local_shapes
if self.rank != 0:
# from receiver's perspective (rank), the recv buffer have the same shape as the sender's array (rank-1)
# in every dimension except the shape at axis=self.axis
recv_shape = list(recv_shapes[self.rank - 1])
recv_shape[self.axis] = total_cells_front[self.rank]
recv_buf = ncp.zeros(recv_shape, dtype=ghosted_array.dtype)
# Transfer of ghost cells can be skipped if len(recv_buf) = 0
# Additionally, NCCL will hang if the buffer size is 0 so this optimization is somewhat mandatory
if len(recv_buf) != 0:
ghosted_array = ncp.concatenate(
[
self._recv(
self.base_comm,
self.base_comm_nccl,
recv_buf,
source=self.rank - 1,
tag=1,
engine=self.engine,
),
ghosted_array,
],
axis=self.axis,
)
# The skip in sender is to match with what described in receiver
if self.rank != self.size - 1 and len(send_buf) != 0:
if cells_front > self.local_shape[self.axis]:
raise ValueError(
f"Local Shape at rank={self.rank} along axis={self.axis} "
f"should be > {cells_front}: dim({self.axis}) "
f"{self.local_shape[self.axis]} < {cells_front}; "
f"to achieve this use NUM_PROCESSES <= "
f"{max(1, self.global_shape[self.axis] // cells_front)}"
)
self._send(
self.base_comm,
self.base_comm_nccl,
send_buf,
dest=self.rank + 1,
tag=1,
engine=self.engine,
)
if cells_back is not None:
total_cells_back = self.base_comm.allgather(cells_back) + [0]
# Read cells_back which needs to be sent to rank - 1(cells_back for rank - 1)
cells_back = total_cells_back[self.rank - 1]
send_buf = ncp.take(
self.local_array, ncp.arange(cells_back), axis=self.axis
)
# Same reasoning as sending cell front applied
recv_shapes = self.local_shapes
if self.rank != 0 and len(send_buf) != 0:
if cells_back > self.local_shape[self.axis]:
raise ValueError(
f"Local Shape at rank={self.rank} along axis={self.axis} "
f"should be > {cells_back}: dim({self.axis}) "
f"{self.local_shape[self.axis]} < {cells_back}; "
f"to achieve this use NUM_PROCESSES <= "
f"{max(1, self.global_shape[self.axis] // cells_back)}"
)
self._send(
self.base_comm,
self.base_comm_nccl,
send_buf,
dest=self.rank - 1,
tag=0,
engine=self.engine,
)
if self.rank != self.size - 1:
recv_shape = list(recv_shapes[self.rank + 1])
recv_shape[self.axis] = total_cells_back[self.rank]
recv_buf = ncp.zeros(recv_shape, dtype=ghosted_array.dtype)
if len(recv_buf) != 0:
ghosted_array = ncp.append(
ghosted_array,
self._recv(
self.base_comm,
self.base_comm_nccl,
recv_buf,
source=self.rank + 1,
tag=0,
engine=self.engine,
),
axis=self.axis,
)
return ghosted_array
def __repr__(self):
return (
f"<DistributedArray with global shape={self.global_shape}, "
f"local shape={self.local_shape}"
f", dtype={self.dtype}, "
f"processes={[i for i in range(self.size)]})> "
)
[docs]
class StackedDistributedArray:
r"""Stacked DistributedArrays
Stack DistributedArray objects and power them with basic mathematical operations.
This class allows one to work with a series of distributed arrays to avoid
having to create a single distributed array with some special internal sorting.
.. note:: All :class:`pylops_mpi.DistributedArray` objects passed to
``distarrays`` must share the same engine (``numpy`` or ``cupy``).
Parameters
----------
distarrays : :obj:`list`
List of :class:`pylops_mpi.DistributedArray` objects.
base_comm : :obj:`mpi4py.MPI.Comm`, optional
Base MPI Communicator.
Defaults to ``mpi4py.MPI.COMM_WORLD``.
Raises
------
ValueError
Stacked distributed arrays have different engine
"""
def __init__(self, distarrays: List, base_comm: MPI.Comm = MPI.COMM_WORLD):
self.distarrays = distarrays
self.narrays = len(distarrays)
self.base_comm = base_comm
self.rank = base_comm.Get_rank()
self.size = base_comm.Get_size()
# Ensure all stacked arrays share the same engine
engines = [distarr.engine for distarr in distarrays]
if any(engine != engines[0] for engine in engines[1:]):
raise ValueError(f"Stacked arrays have mismatching engines: {engines}")
# Define global shape as sum as shapes
self._global_shape = distarrays[0].global_shape
for iarr in range(1, self.narrays):
self._global_shape = tuple(
[
g1 + g2
for g1, g2 in zip(self._global_shape, distarrays[iarr].global_shape)
]
)
def __repr__(self) -> str:
repr_dist = "\n".join([distarray.__repr__() for distarray in self.distarrays])
return (
f"<StackedDistributedArray with {self.narrays} distributed arrays: \n"
+ repr_dist
)
def __getitem__(self, index):
return self.distarrays[index]
def __setitem__(self, index, value):
target = self.distarrays[index]
if isinstance(target, StackedDistributedArray):
# nested StackedDistributedArray: assign each sub-array in turn
for iarr in range(target.narrays):
target[iarr] = value[iarr]
elif isinstance(value, DistributedArray):
# plain DistributedArray: assign the local array
target[:] = value[:]
else:
# plain DistributedArray assigned from an array-like / scalar
target[:] = value
@property
def global_shape(self):
"""Global Shape of the stacked array
Returns
-------
global_shape : :obj:`tuple`
"""
return self._global_shape
@property
def engine(self):
"""Engine of the Stacked Distributed Array
All stacked arrays share the same engine (enforced at creation time).
A nested :class:`pylops_mpi.StackedDistributedArray` exposes this same
property, so this resolves recursively until it reaches a
:class:`pylops_mpi.DistributedArray`.
Returns
-------
engine : :obj:`str`
Engine
"""
return self.distarrays[0].engine
def asarray(self) -> NDArray:
"""Global view of the array
Gather all the distributed arrays and return
a flattened version
Returns
-------
final_array : :obj:`numpy.ndarray`
Global Array gathered at all ranks and flattened
"""
ncp = get_module(self.engine)
return ncp.hstack([distarr.asarray().ravel() for distarr in self.distarrays])
def _check_stacked_size(self, stacked_array: Self) -> None:
"""Check that arrays have consistent size"""
if self.narrays != stacked_array.narrays:
raise ValueError(
"Stacked arrays must be composed the same number of of distributed arrays"
)
for iarr in range(self.narrays):
if self.distarrays[iarr].global_shape != stacked_array[iarr].global_shape:
raise ValueError(
f"Stacked arrays {iarr} have different global shape:"
f"{self.distarrays[iarr].global_shape} / "
f"{stacked_array[iarr].global_shape}"
)
def __neg__(self):
return self.multiply(-1)
def __add__(self, x):
return self.add(x)
def __iadd__(self, x):
return self.iadd(x)
def __sub__(self, x):
return self.subtract(x)
def __isub__(self, x):
return self.subtract(x, out=self)
def __mul__(self, x):
return self.multiply(x)
def __imul__(self, x):
return self.multiply(x, out=self)
def __rmul__(self, x):
return self.multiply(x)
def _apply(
self, name: str, other: float | int | Self, out: Optional[Self] = None
) -> Self:
"""Apply the binary method ``name`` of the children, writing into ``out``
The result is allocated once (if ``out`` is not provided) and each
child writes directly into its portion of it, recursing into nested
stacked arrays.
"""
is_stacked = isinstance(other, StackedDistributedArray)
if is_stacked:
self._check_stacked_size(other)
if out is None:
out = self.empty_like()
for iarr in range(self.narrays):
getattr(self[iarr], name)(
other[iarr] if is_stacked else other, out=out[iarr]
)
return out
def _leaves(self) -> List[DistributedArray]:
"""Flat list of the (non-stacked) DistributedArrays"""
leaves = []
for distarray in self.distarrays:
if isinstance(distarray, StackedDistributedArray):
leaves.extend(distarray._leaves())
else:
leaves.append(distarray)
return leaves
def _fusable(self, leaves: List[DistributedArray]) -> bool:
"""Whether the reductions of all leaves can share a single allreduce
The partial results of the leaves can be summed in one allreduce
(instead of one per leaf) only if every leaf:
- is partitioned as ``Partition.SCATTER``, so that each rank holds
a distinct portion of the data and the global result is the sum of
the local ones (``BROADCAST`` leaves hold a full copy on every rank
and would be counted multiple times);
- has no ``mask``, i.e. the reduction is over all ranks and not over
subgroups of ranks;
- uses the same MPI communicator (``base_comm``) and the same NCCL
communicator (``base_comm_nccl``) as the first leaf, so a single
collective call is valid for all of them.
Parameters
----------
leaves : :obj:`list` of :obj:`pylops_mpi.DistributedArray`
Flat list of the non-stacked arrays (see ``_leaves``).
Returns
-------
fusable : :obj:`bool`
``True`` if a single allreduce can be used, ``False`` if each
leaf must be reduced separately.
"""
first = leaves[0]
return all(
leaf.partition is Partition.SCATTER
and leaf.mask is None
and leaf.base_comm is first.base_comm
and leaf.base_comm_nccl is first.base_comm_nccl
for leaf in leaves
)
def _fused_allreduce(self, leaves: List[DistributedArray], local):
"""Sum-reduce the local partial result of the leaves in one allreduce
Used by :meth:`dot` and :meth:`norm` to replace one allreduce per leaf
with a single one. The leaves must satisfy :meth:`_fusable`, which
guarantees that they share the same communicator, engine and
partitioning, so the collective call is issued through the first leaf.
Parameters
----------
leaves : :obj:`list` of :obj:`pylops_mpi.DistributedArray`
Flat list of the non-stacked arrays (see ``_leaves``). Only the
first leaf is used to select the communicator and engine.
local : :obj:`float` or :obj:`complex` or array-like
Partial result computed locally on this rank, already summed over
all leaves (e.g., sum of the local dot products or of the
local squared norms).
Returns
-------
result : :obj:`float` or :obj:`complex`
Sum of ``local`` over all ranks, converted to a scalar
irrespective of the engine used (NumPy, CuPy or NCCL).
"""
first = leaves[0]
return to_scalar(
first._allreduce_subcomm(
first.sub_comm,
first.base_comm_nccl,
local,
engine=first.engine,
)
)
def add(
self, stacked_array: float | int | Self, out: Optional[Self] = None
) -> Self:
"""Stacked Distributed Addition of arrays
Parameters
----------
stacked_array : :obj:`pylops_mpi.StackedDistributedArray` or scalar
Array to add.
out : :obj:`pylops_mpi.StackedDistributedArray`, optional
Array where the result is stored (it can be ``self``).
If ``None``, a new array is created.
Returns
-------
out : :obj:`pylops_mpi.StackedDistributedArray`
Result of the addition. This is the ``out`` array if provided,
otherwise a newly created array.
"""
return self._apply("add", stacked_array, out)
def iadd(self, stacked_array: float | int | Self) -> Self:
"""Stacked Distributed In-Place Addition of arrays
Parameters
----------
stacked_array : :obj:`pylops_mpi.StackedDistributedArray` or scalar
Array to add.
Returns
-------
out : :obj:`pylops_mpi.StackedDistributedArray`
This array (``self``), updated in place with the result of the addition.
"""
return self.add(stacked_array, out=self)
def subtract(
self, stacked_array: float | int | Self, out: Optional[Self] = None
) -> Self:
"""Stacked Distributed Subtraction of arrays
Parameters
----------
stacked_array : :obj:`pylops_mpi.StackedDistributedArray` or scalar
Array to subtract.
out : :obj:`pylops_mpi.StackedDistributedArray`, optional
Array where the result is stored (it can be ``self``).
If ``None``, a new array is created.
Returns
-------
out : :obj:`pylops_mpi.StackedDistributedArray`
Result of the subtraction. This is the ``out`` array if provided,
otherwise a newly created array.
"""
return self._apply("subtract", stacked_array, out)
def multiply(
self, stacked_array: float | int | Self, out: Optional[Self] = None
) -> Self:
"""Stacked Distributed Multiplication of arrays
Parameters
----------
stacked_array : :obj:`pylops_mpi.StackedDistributedArray` or scalar
Array to multiply by.
out : :obj:`pylops_mpi.StackedDistributedArray`, optional
Array where the result is stored (it can be ``self``).
If ``None``, a new array is created.
Returns
-------
out : :obj:`pylops_mpi.StackedDistributedArray`
Result of the multiplication. This is the ``out`` array if provided,
otherwise a newly created array.
"""
return self._apply("multiply", stacked_array, out)
def dot(self, stacked_array: Self, vdot: bool = False) -> Union[float, complex]:
"""
Compute the distributed dot product between this array and another
distributed array.
Parameters
----------
stacked_array : :obj:`pylops_mpi.StackedDistributedArray`
The distributed array with which to compute the dot product.
It must have a compatible shape and partitioning.
vdot : bool, optional, Defaults to `False`
If True, compute the complex conjugate dot product (like numpy.vdot),
where the first argument is conjugated. If False, compute the standard
dot product.
Returns
-------
result : :obj:`float` or :obj:`complex`
The result of the dot product across all ranks. This is reduced across all processes.
"""
self._check_stacked_size(stacked_array)
leaves = self._leaves()
other_leaves = stacked_array._leaves()
if len(leaves) == len(other_leaves) and self._fusable(leaves):
# sum the local dot products and reduce only once
for leaf, other_leaf in zip(leaves, other_leaves):
leaf._check_partition_shape(other_leaf)
leaf._check_mask(other_leaf)
local = sum(
leaf._local_dot(other_leaf, vdot)
for leaf, other_leaf in zip(leaves, other_leaves)
)
return self._fused_allreduce(leaves, local)
dotprod = 0.0
for iarr in range(self.narrays):
dotprod += self[iarr].dot(stacked_array[iarr], vdot=vdot)
return dotprod
def norm(self, ord: Optional[int] = None) -> float:
"""numpy.linalg.norm method on stacked Distributed arrays
Parameters
----------
ord : :obj:`int`, optional
Order of the norm.
Returns
-------
norm : :obj:`float`
The computed norm of the stacked distributed array.
"""
ord = 2 if ord is None else ord
leaves = self._leaves()
# Fast-fused L2 norm
if ord == 2 and self._fusable(leaves):
# sum the local squared norms and reduce only once
local = sum(leaf._local_sqnorm(leaf.local_array) for leaf in leaves)
return to_scalar(np.sqrt(self._fused_allreduce(leaves, local)))
# Other norms - note that the norms of the individual distributed
# arrays are scalars (irrespective of the engine used), thus numpy
# is used to combine them
norms = np.array([distarray.norm(ord) for distarray in self.distarrays])
if ord in ["fro", "nuc"]:
raise ValueError(f"norm-{ord} not possible for vectors")
elif ord == 0:
# Count non-zero then sum reduction
norm = np.sum(norms)
elif ord == np.inf:
# Calculate max followed by max reduction
norm = np.max(norms)
elif ord == -np.inf:
# Calculate min followed by max reduction
norm = np.min(norms)
else:
norm = np.power(np.sum(np.power(norms, ord)), 1.0 / ord)
return to_scalar(norm)
def conj(self) -> Self:
"""Distributed conj() method"""
ConjArray = StackedDistributedArray(
[distarray.conj() for distarray in self.distarrays]
)
return ConjArray
def copy(self) -> Self:
"""Creates a copy of the DistributedArray"""
arr = StackedDistributedArray(
[distarray.copy() for distarray in self.distarrays]
)
return arr
def zeros_like(self) -> Self:
"""Creates a zero like StackedDistributedArray"""
return StackedDistributedArray(
distarrays=[distarray.zeros_like() for distarray in self.distarrays]
)
def empty_like(self) -> Self:
"""Creates an empty like StackedDistributedArray with uninitialized values"""
return StackedDistributedArray(
distarrays=[distarray.empty_like() for distarray in self.distarrays]
)