subroutine ESMF_ArrayHaloStore(array, routehandle, keywordEnforcer, &
startregion, haloLDepth, haloUDepth, pipelineDepth, rc)
!
! !ARGUMENTS:
type(ESMF_Array), intent(inout) :: array
type(ESMF_RouteHandle), intent(inout) :: routehandle
type(ESMF_KeywordEnforcer), optional:: keywordEnforcer ! must use keywords below
type(ESMF_StartRegion_Flag), intent(in), optional :: startregion
integer, intent(in), optional :: haloLDepth(:)
integer, intent(in), optional :: haloUDepth(:)
integer, intent(inout), optional :: pipelineDepth
integer, intent(out), optional :: rc
!
! !STATUS:
! \begin{itemize}
! \item\apiStatusCompatibleVersion{5.2.0r}
! \item\apiStatusModifiedSinceVersion{5.2.0r}
! \begin{description}
! \item[6.1.0] Added argument {\tt pipelineDepth}.
! The new argument provide access to the tuning parameter
! affecting the sparse matrix execution.
! \end{description}
! \end{itemize}
!
! !DESCRIPTION:
! Store an Array halo operation over the data in {\tt array}. By default,
! i.e. without specifying {\tt startregion}, {\tt haloLDepth} and
! {\tt haloUDepth}, all elements in the total Array region that lie outside
! the exclusive region will be considered potential destination elements for
! halo. However, only those elements that have a corresponding halo source
! element, i.e. an exclusive element on one of the DEs, will be updated under
! the halo operation. Elements that have no associated source remain
! unchanged under halo.
!
! Specifying {\tt startregion} allows the shape of the effective halo region
! to be changed from the inside. Setting this flag to
! {\tt ESMF\_STARTREGION\_COMPUTATIONAL} means that only elements outside
! the computational region of the Array are considered for potential
! destination elements for the halo operation. The default is
! {\tt ESMF\_STARTREGION\_EXCLUSIVE}.
!
! The {\tt haloLDepth} and {\tt haloUDepth} arguments allow to reduce
! the extent of the effective halo region. Starting at the region specified
! by {\tt startregion}, the {\tt haloLDepth} and {\tt haloUDepth}
! define a halo depth in each direction. Note that the maximum halo region is
! limited by the total Array region, independent of the actual
! {\tt haloLDepth} and {\tt haloUDepth} setting. The total Array region is
! local DE specific. The {\tt haloLDepth} and {\tt haloUDepth} are interpreted
! as the maximum desired extent, reducing the potentially larger region
! available for the halo operation.
!
! The routine returns an {\tt ESMF\_RouteHandle} that can be used to call
! {\tt ESMF\_ArrayHalo()} on any Array that matches
! {\tt array} in {\em type}, {\em kind}, and
! memory layout of the {\em distributed} dimensions. However, the size,
! number, and index order of {\em undistributed} dimensions may be different.
! See section \ref{RH:Reusability} for a more detailed discussion of
! RouteHandle reusability.
!
! This call is {\em collective} across the current VM.
!
! \begin{description}
! \item [array]
! {\tt ESMF\_Array} containing data to be haloed. The data in the halo
! region may be destroyed by this call.
! \item [routehandle]
! Handle to the precomputed Route.
! \item [{[startregion]}]
! \begin{sloppypar}
! The start of the effective halo region on every DE. The default
! setting is {\tt ESMF\_STARTREGION\_EXCLUSIVE}, rendering all non-exclusive
! elements potential halo destination elements.
! See section \ref{const:startregion} for a complete list of
! valid settings.
! \end{sloppypar}
! \item[{[haloLDepth]}]
! This vector specifies the lower corner of the effective halo
! region with respect to the lower corner of {\tt startregion}.
! The size of {\tt haloLDepth} must equal the number of distributed Array
! dimensions.
! \item[{[haloUDepth]}]
! This vector specifies the upper corner of the effective halo
! region with respect to the upper corner of {\tt startregion}.
! The size of {\tt haloUDepth} must equal the number of distributed Array
! dimensions.
! \item [{[pipelineDepth]}]
! The {\tt pipelineDepth} parameter controls how many messages a PET
! may have outstanding during a halo exchange. Larger values
! of {\tt pipelineDepth} typically lead to better performance. However,
! on some systems too large a value may lead to performance degradation,
! or runtime errors.
!
! Note that the pipeline depth has no effect on the bit-for-bit
! reproducibility of the results. However, it may affect the performance
! reproducibility of the exchange.
!
! The {\tt ESMF\_ArraySMMStore()} method implements an auto-tuning scheme
! for the {\tt pipelineDepth} parameter. The intent on the
! {\tt pipelineDepth} argument is "{\tt inout}" in order to
! support both overriding and accessing the auto-tuning parameter.
! If an argument $>= 0$ is specified, it is used for the
! {\tt pipelineDepth} parameter, and the auto-tuning phase is skipped.
! In this case the {\tt pipelineDepth} argument is not modified on
! return. If the provided argument is $< 0$, the {\tt pipelineDepth}
! parameter is determined internally using the auto-tuning scheme. In this
! case the {\tt pipelineDepth} argument is re-set to the internally
! determined value on return. Auto-tuning is also used if the optional
! {\tt pipelineDepth} argument is omitted.
!
! \item [{[rc]}]
! Return code; equals {\tt ESMF\_SUCCESS} if there are no errors.
! \end{description}
!
!EOP
!------------------------------------------------------------------------------
integer :: localrc ! local return code
type(ESMF_StartRegion_Flag) :: opt_startregion ! helper variable
type(ESMF_InterArray) :: haloLDepthArg ! helper variable
type(ESMF_InterArray) :: haloUDepthArg ! helper variable
! initialize return code; assume routine not implemented
localrc = ESMF_RC_NOT_IMPL
if (present(rc)) rc = ESMF_RC_NOT_IMPL
! Check init status of arguments
ESMF_INIT_CHECK_DEEP(ESMF_ArrayGetInit, array, rc)
! Set default flags
opt_startregion = ESMF_STARTREGION_EXCLUSIVE
if (present(startregion)) opt_startregion = startregion
! Deal with (optional) array arguments
haloLDepthArg = ESMF_InterArrayCreate(haloLDepth, rc=localrc)
if (ESMF_LogFoundError(localrc, ESMF_ERR_PASSTHRU, &
ESMF_CONTEXT, rcToReturn=rc)) return
haloUDepthArg = ESMF_InterArrayCreate(haloUDepth, rc=localrc)
if (ESMF_LogFoundError(localrc, ESMF_ERR_PASSTHRU, &
ESMF_CONTEXT, rcToReturn=rc)) return
! Call into the C++ interface, which will sort out optional arguments
call c_ESMC_ArrayHaloStore(array, routehandle, opt_startregion, &
haloLDepthArg, haloUDepthArg, pipelineDepth, localrc)
if (ESMF_LogFoundError(localrc, ESMF_ERR_PASSTHRU, &
ESMF_CONTEXT, rcToReturn=rc)) return
! garbage collection
call ESMF_InterArrayDestroy(haloLDepthArg, rc=localrc)
if (ESMF_LogFoundError(localrc, ESMF_ERR_PASSTHRU, &
ESMF_CONTEXT, rcToReturn=rc)) return
call ESMF_InterArrayDestroy(haloUDepthArg, rc=localrc)
if (ESMF_LogFoundError(localrc, ESMF_ERR_PASSTHRU, &
ESMF_CONTEXT, rcToReturn=rc)) return
! Mark routehandle object as being created
call ESMF_RouteHandleSetInitCreated(routehandle, rc=localrc)
if (ESMF_LogFoundError(localrc, ESMF_ERR_PASSTHRU, &
ESMF_CONTEXT, rcToReturn=rc)) return
! return successfully
if (present(rc)) rc = ESMF_SUCCESS
end subroutine ESMF_ArrayHaloStore