! $Id$ ! ! Earth System Modeling Framework ! Copyright (c) 2002-2023, University Corporation for Atmospheric Research, ! Massachusetts Institute of Technology, Geophysical Fluid Dynamics ! Laboratory, University of Michigan, National Centers for Environmental ! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, ! NASA Goddard Space Flight Center. ! Licensed under the University of Illinois-NCSA License. ! !============================================================================== ! program ESMF_FieldEx !------------------------------------------------------------------------------ !ESMF_MULTI_PROC_EXAMPLE String used by test script to count examples. !============================================================================== ! !PROGRAM: ESMF_FieldEx - Field Examples ! ! !DESCRIPTION: ! ! This program shows examples of Field get data pointer methods !----------------------------------------------------------------------------- #include "ESMF.h" ! ESMF Framework module use ESMF_TestMod use ESMF implicit none ! Local variables integer :: rc type(ESMF_DistGrid) :: distgrid type(ESMF_ArraySpec) :: arrayspec real(ESMF_KIND_R8), dimension(:,:,:), allocatable :: farray real(ESMF_KIND_R8), dimension(:,:,:), pointer :: farray1 real(ESMF_KIND_R8), pointer :: myFarray(:,:) type(ESMF_Field) :: field, field1, field3, field4, fieldMigrated type(ESMF_Grid) :: grid3d, grid, grid2d type(ESMF_DistGrid) :: distgrid3d type(ESMF_Array) :: array3d, array, array2d, arrayMigrated integer :: xdim, ydim, zdim, localpet, localDeCount, ssiLocalDeCount integer, allocatable:: localDeToDeMap(:) type(ESMF_Delayout) :: delayout type(ESMF_VM) :: vm logical :: ssiSharedMemoryEnabled real(ESMF_KIND_R4), dimension(:,:), pointer :: farray2dd real(ESMF_KIND_R4), dimension(:,:), pointer :: farray2d integer, dimension(2) :: compEdgeLWdith integer, dimension(2) :: compEdgeUWdith integer :: ftlb(2), ftub(2), ftc(2) integer :: compLBnd(1:3), compUBnd(1:3) integer :: exclLBnd(1:3), exclUBnd(1:3) integer :: totalLBnd(1:3), totalUBnd(1:3) integer :: comp_count(1:3) integer :: excl_count(1:3) integer :: total_count(1:3) type(ESMF_TypeKind_Flag) :: typekind integer :: dimCount type(ESMF_StaggerLoc) :: staggerloc integer :: gridToFieldMap(3) integer :: ungriddedLBound(3) integer :: ungriddedUBound(3) integer :: totalLWidth(3,1) integer :: totalUWidth(3,1) integer :: fa_shape(3) character(len=32) :: name real(ESMF_KIND_R8), dimension(:,:,:,:,:,:,:), pointer :: farray7d real(ESMF_KIND_R8), dimension(:,:,:,:,:,:,:), pointer :: farray7d2 type(ESMF_Field) :: field7d, field7d2 type(ESMF_Grid) :: grid5d type(ESMF_DistGrid) :: distgrid5d integer :: fsize(7) integer :: flbound(7), fubound(7) type(ESMF_FieldStatus_Flag) :: fstatus real(ESMF_KIND_R4) :: PI=3.14159265 integer :: finalrc, i, j, k, result character(ESMF_MAXSTR) :: testname character(ESMF_MAXSTR) :: failMsg !------------------------------------------------------------------------- !------------------------------------------------------------------------- write(failMsg, *) "Example failure" write(testname, *) "Example ESMF_FieldEx" ! ------------------------------------------------------------------------------ ! ------------------------------------------------------------------------------ ! !Set finalrc to success finalrc = ESMF_SUCCESS call ESMF_Initialize(defaultlogfilename="FieldEx.Log", & logkindflag=ESMF_LOGKIND_MULTI, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) if (.not. ESMF_TestMinPETs(4, ESMF_SRCLINE)) & call ESMF_Finalize(endflag=ESMF_END_ABORT) !>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !-------------------------------- Example ----------------------------- !>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !BOE !\subsubsection{Get Fortran data pointer, bounds, and counts information from a Field} !\label{sec:field:usage:field_get_dataptr} ! ! A user can get bounds and counts information from an {\tt ESMF\_Field} ! through the {\tt ESMF\_FieldGet()} interface. Also available through this interface ! is the intrinsic ! Fortran data pointer contained in the internal {\tt ESMF\_Array} object ! of an {\tt ESMF\_Field}. The bounds and counts information are DE specific ! for the associated Fortran data pointer. ! ! For a better discussion of the terminologies, bounds and widths in ESMF ! e.g. exclusive, computational, total bounds ! for the lower and upper corner of data region, etc.., user can refer to ! the explanation of these concepts for Grid and Array in their respective sections ! in the {\it Reference Manual}, e.g. Section \ref{Array_regions_and_default_bounds} on Array ! and Section \ref{sec:grid:usage:bounds} on Grid. ! ! In this example, we first create a 3D Field based on a 3D Grid and Array. ! Then we use the {\tt ESMF\_FieldGet()} interface to retrieve the data pointer, ! potentially updating or verifying its values. We also retrieve the bounds and counts ! information of the 3D Field to assist in data element iteration. ! !EOE !BOC xdim = 180 ydim = 90 zdim = 50 ! create a 3D data Field from a Grid and Array. ! first create a Grid grid3d = ESMF_GridCreateNoPeriDim(minIndex=(/1,1,1/), & maxIndex=(/xdim,ydim,zdim/), & regDecomp=(/2,2,1/), name="grid", rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) call ESMF_GridGet(grid=grid3d, staggerloc=ESMF_STAGGERLOC_CENTER, & distgrid=distgrid3d, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) call ESMF_GridGetFieldBounds(grid=grid3d, localDe=0, & staggerloc=ESMF_STAGGERLOC_CENTER, totalCount=fa_shape, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) allocate(farray(fa_shape(1), fa_shape(2), fa_shape(3)) ) ! create an Array array3d = ESMF_ArrayCreate(distgrid3d, farray, & indexflag=ESMF_INDEX_DELOCAL, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! create a Field field = ESMF_FieldCreate(grid=grid3d, array=array3d, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! retrieve the Fortran data pointer from the Field call ESMF_FieldGet(field=field, localDe=0, farrayPtr=farray1, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! retrieve the Fortran data pointer from the Field and bounds call ESMF_FieldGet(field=field, localDe=0, farrayPtr=farray1, & computationalLBound=compLBnd, computationalUBound=compUBnd, & exclusiveLBound=exclLBnd, exclusiveUBound=exclUBnd, & totalLBound=totalLBnd, totalUBound=totalUBnd, & computationalCount=comp_count, & exclusiveCount=excl_count, & totalCount=total_count, & rc=rc) ! iterate through the total bounds of the field data pointer do k = totalLBnd(3), totalUBnd(3) do j = totalLBnd(2), totalUBnd(2) do i = totalLBnd(1), totalUBnd(1) farray1(i, j, k) = sin(2*i/total_count(1)*PI) + & sin(4*j/total_count(2)*PI) + & sin(8*k/total_count(2)*PI) enddo enddo enddo !EOC if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) print *, "Field Get Data Pointer example returned" !>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !-------------------------------- Example ----------------------------- !>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !BOE !\subsubsection{Get Grid, Array, and other information from a Field} !\label{sec:field:usage:field_get_default} ! ! A user can get the internal {\tt ESMF\_Grid} and {\tt ESMF\_Array} ! from a {\tt ESMF\_Field}. Note that the user should not issue any destroy command ! on the retrieved grid or array object since they are referenced ! from within the {\tt ESMF\_Field}. The retrieved objects should be used ! in a read-only fashion to query additional information not directly ! available through the {\tt ESMF\_FieldGet()} interface. ! !EOE !BOC call ESMF_FieldGet(field, grid=grid, array=array, & typekind=typekind, dimCount=dimCount, staggerloc=staggerloc, & gridToFieldMap=gridToFieldMap, & ungriddedLBound=ungriddedLBound, ungriddedUBound=ungriddedUBound, & totalLWidth=totalLWidth, totalUWidth=totalUWidth, & name=name, & rc=rc) !EOC if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) print *, "Field Get Grid and Array example returned" call ESMF_FieldDestroy(field, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !-------------------------------- Example ----------------------------- !>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !BOE !\subsubsection{Create a Field with a Grid, typekind, and rank} !\label{sec:field:usage:create_grid_tkr} ! ! A user can create an {\tt ESMF\_Field} from an {\tt ESMF\_Grid} and ! typekind/rank. ! This create method associates the two objects. ! ! We first create a Grid with a regular distribution that is ! 10x20 index in 2x2 DEs. This version of Field create simply ! associates the data with the Grid. The data is referenced ! explicitly on a regular 2x2 uniform grid. ! Finally we create a Field from ! the Grid, typekind, rank, and a user specified StaggerLoc. ! ! This example also illustrates a typical use of this Field creation ! method. By creating a Field from a Grid and typekind/rank, the ! user allows the ESMF library to create a internal Array in the Field. ! Then the user can use {\tt ESMF\_FieldGet()} to retrieve the Fortran ! data array ! and necessary bounds information to assign initial values to it. !EOE !BOC ! create a grid grid = ESMF_GridCreateNoPeriDim(minIndex=(/1,1/), maxIndex=(/10,20/), & regDecomp=(/2,2/), name="atmgrid", rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! create a Field from the Grid and arrayspec field1 = ESMF_FieldCreate(grid, typekind=ESMF_TYPEKIND_R4, & indexflag=ESMF_INDEX_DELOCAL, & staggerloc=ESMF_STAGGERLOC_CENTER, name="pressure", rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) call ESMF_FieldGet(field1, localDe=0, farrayPtr=farray2dd, & totalLBound=ftlb, totalUBound=ftub, totalCount=ftc, rc=rc) do i = ftlb(1), ftub(1) do j = ftlb(2), ftub(2) farray2dd(i, j) = sin(i/ftc(1)*PI) * cos(j/ftc(2)*PI) enddo enddo if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !EOC print *, "Field creation from Grid, typekind, and rank returned" call ESMF_GridDestroy(grid, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) call ESMF_FieldDestroy(field1, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !-------------------------------- Example ----------------------------- !>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !BOE !\subsubsection{Create a Field with a Grid and Arrayspec} !\label{sec:field:usage:create_grid_arrayspec} ! ! A user can create an {\tt ESMF\_Field} from an {\tt ESMF\_Grid} and a ! {\tt ESMF\_Arrayspec} with corresponding rank and type. ! This create method associates the two objects. ! ! We first create a Grid with a regular distribution that is ! 10x20 index in 2x2 DEs. This version of Field create simply ! associates the data with the Grid. The data is referenced ! explicitly on a regular 2x2 uniform grid. ! Then we create an ArraySpec. Finally we create a Field from ! the Grid, ArraySpec, and a user specified StaggerLoc. ! ! This example also illustrates a typical use of this Field creation ! method. By creating a Field from a Grid and an ArraySpec, the ! user allows the ESMF library to create a internal Array in the Field. ! Then the user can use {\tt ESMF\_FieldGet()} to retrieve the Fortran ! data array ! and necessary bounds information to assign initial values to it. !EOE !BOC ! create a grid grid = ESMF_GridCreateNoPeriDim(minIndex=(/1,1/), maxIndex=(/10,20/), & regDecomp=(/2,2/), name="atmgrid", rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! setup arrayspec call ESMF_ArraySpecSet(arrayspec, 2, ESMF_TYPEKIND_R4, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! create a Field from the Grid and arrayspec field1 = ESMF_FieldCreate(grid, arrayspec, & indexflag=ESMF_INDEX_DELOCAL, & staggerloc=ESMF_STAGGERLOC_CENTER, name="pressure", rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) call ESMF_FieldGet(field1, localDe=0, farrayPtr=farray2dd, & totalLBound=ftlb, totalUBound=ftub, totalCount=ftc, rc=rc) do i = ftlb(1), ftub(1) do j = ftlb(2), ftub(2) farray2dd(i, j) = sin(i/ftc(1)*PI) * cos(j/ftc(2)*PI) enddo enddo if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !EOC call ESMF_FieldDestroy(field1, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !BOE ! A user can also create an ArraySpec that has a different rank ! from the Grid, For example, the following code shows creation of ! of 3D Field from a 2D Grid using a 3D ArraySpec. ! ! This example also demonstrates the technique to create a typical ! 3D data Field that has 2 gridded dimensions and 1 ungridded ! dimension. ! ! First we create a 2D grid with an index space of 180x360 equivalent to ! 180x360 Grid cells (note that for a distributed memory computer, this ! means each ! grid cell will be on a separate PE!). In the FieldCreate call, we use gridToFieldMap ! to indicate the mapping between Grid dimension and Field dimension. ! For the ungridded dimension (typically the altitude), we use ! ungriddedLBound and ungriddedUBound to describe its bounds. Internally ! the ungridded dimension has a stride of 1, so the number of elements ! of the ungridded dimension is ungriddedUBound - ungriddedLBound + 1. ! ! Note that gridToFieldMap in this specific example is (/1,2/) which ! is the default value ! so the user can neglect this argument for the FieldCreate call. !EOE !BOC grid2d = ESMF_GridCreateNoPeriDim(minIndex=(/1,1/), & maxIndex=(/180,360/), regDecomp=(/2,2/), name="atmgrid", rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) call ESMF_ArraySpecSet(arrayspec, 3, ESMF_TYPEKIND_R4, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) field1 = ESMF_FieldCreate(grid2d, arrayspec, & indexflag=ESMF_INDEX_DELOCAL, & staggerloc=ESMF_STAGGERLOC_CENTER, & gridToFieldMap=(/1,2/), & ungriddedLBound=(/1/), ungriddedUBound=(/50/), & name="pressure", rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !EOC print *, "Field creation from Grid and Arrayspec returned" !>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !-------------------------------- Example ----------------------------- !>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !BOE !\subsubsection{Create a Field with a Grid and Array} !\label{sec:field:usage:create_grid_array} ! ! A user can create an {\tt ESMF\_Field} from an {\tt ESMF\_Grid} and a ! {\tt ESMF\_Array}. The Grid was created in the previous example. ! ! This example creates a 2D {\tt ESMF\_Field} from a 2D {\tt ESMF\_Grid} ! and a 2D {\tt ESMF\_Array}. !EOE !BOC ! Get necessary information from the Grid call ESMF_GridGet(grid, staggerloc=ESMF_STAGGERLOC_CENTER, & distgrid=distgrid, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! Create a 2D ESMF_TYPEKIND_R4 arrayspec call ESMF_ArraySpecSet(arrayspec, 2, ESMF_TYPEKIND_R4, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! Create a ESMF_Array from the arrayspec and distgrid array2d = ESMF_ArrayCreate(arrayspec=arrayspec, & distgrid=distgrid, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! Create a ESMF_Field from the grid and array field4 = ESMF_FieldCreate(grid, array2d, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !EOC print *, "Field Create from a Grid and an Array returned" call ESMF_FieldDestroy(field4, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !-------------------------------- Example ----------------------------- !>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !BOE !\subsubsection{Create an empty Field and complete it ! with FieldEmptySet and FieldEmptyComplete} !\label{sec:field:usage:partial_creation} ! ! A user can create an {\tt ESMF\_Field} in three steps: first create an empty ! {\tt ESMF\_Field}; then set a {\tt ESMF\_Grid} on the empty {\tt ESMF\_Field}; ! and finally complete the {\tt ESMF\_Field} by calling {\tt ESMF\_FieldEmptyComplete}. ! !EOE !BOC ! create an empty Field field3 = ESMF_FieldEmptyCreate(name="precip", rc=rc) !EOC if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !BOC ! use FieldGet to retrieve the Field Status call ESMF_FieldGet(field3, status=fstatus, rc=rc) !EOC if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !BOE ! Once the Field is created, we can verify that the status of the Field ! is {\tt ESMF\_FIELDSTATUS\_EMPTY}. !EOE !BOC ! Test the status of the Field if (fstatus /= ESMF_FIELDSTATUS_EMPTY) then call ESMF_Finalize(endflag=ESMF_END_ABORT) endif !EOC !BOE ! Next we set a Grid on the empty Field. We use the 2D grid created in ! a previous example simply to demonstrate the method. The Field data points ! will be on east edge of the Grid cells with the specified ! {\tt ESMF\_STAGGERLOC\_EDGE1}. !EOE !BOC ! Set a grid on the Field call ESMF_FieldEmptySet(field3, grid2d, & staggerloc=ESMF_STAGGERLOC_EDGE1, rc=rc) !EOC if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !BOC ! use FieldGet to retrieve the Field Status again call ESMF_FieldGet(field3, status=fstatus, rc=rc) !EOC if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !BOC ! Test the status of the Field if (fstatus /= ESMF_FIELDSTATUS_GRIDSET) then call ESMF_Finalize(endflag=ESMF_END_ABORT) endif !EOC !BOE ! The partially created Field is completed by specifying the typekind of its ! data storage. This method is overloaded with one of the ! following parameters, arrayspec, typekind, Fortran array, or Fortran array pointer. ! Additional optional arguments can be used to specify ungridded dimensions and ! halo regions similar to the other Field creation methods. !EOE !BOC ! Complete the Field by specifying the data typekind ! to be allocated internally. call ESMF_FieldEmptyComplete(field3, typekind=ESMF_TYPEKIND_R8, & ungriddedLBound=(/1/), ungriddedUBound=(/5/), rc=rc) !EOC if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !BOC ! use FieldGet to retrieve the Field Status again call ESMF_FieldGet(field3, status=fstatus, rc=rc) !EOC if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !BOC ! Test the status of the Field if (fstatus /= ESMF_FIELDSTATUS_COMPLETE) then call ESMF_Finalize(endflag=ESMF_END_ABORT) endif !EOC print *, "Complete a Field created by ESMF_FieldEmptyCreate returned" call ESMF_FieldDestroy(field3, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !-------------------------------- Example ----------------------------- !>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !BOE !\subsubsection{Create an empty Field and complete it with FieldEmptyComplete} !\label{sec:field:usage:create_empty} ! ! A user can create an empty {\tt ESMF\_Field}. ! Then the user can finalize the empty {\tt ESMF\_Field} from a {\tt ESMF\_Grid} ! and an intrinsic ! Fortran data array. This interface is overloaded for typekind and rank ! of the Fortran data array. ! ! In this example, both the grid and the Fortran array pointer are 2 dimensional ! and each dimension of the grid is mapped to the corresponding dimension of the ! Fortran array pointer, i.e. 1st dimension of grid maps to 1st dimension of ! Fortran array pointer, 2nd dimension of grid maps to 2nd dimension of ! Fortran array pointer, so on and so forth. ! ! In order to create or complete a Field from a Grid and a Fortran array pointer, ! certain rules of the Fortran array bounds must be obeyed. We will discuss these ! rules as we progress in Field creation examples. We will make ! frequent reference to the terminologies for bounds and widths in ESMF. ! For a better discussion of ! these terminologies and concepts behind them, ! e.g. exclusive, computational, total bounds ! for the lower and upper corner of data region, etc.., users can refer to ! the explanation of these concepts for Grid and Array in their respective sections ! in the {\it Reference Manual}, e.g. Section \ref{Array_regions_and_default_bounds} on Array ! and Section \ref{sec:grid:usage:bounds} on Grid. ! The examples here are designed to help a user to get up to speed with ! creating Fields for typical use. ! ! This example introduces a helper method, the {\tt ESMF\_GridGetFieldBounds} ! interface that facilitates the computation of Fortran data array bounds ! and shape to assist {\tt ESMF\_FieldEmptyComplete} finalizing a Field from an ! intrinsic Fortran data array and a Grid. ! !EOE !BOC ! create an empty Field field3 = ESMF_FieldEmptyCreate(name="precip", rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! use FieldGet to retrieve total counts call ESMF_GridGetFieldBounds(grid2d, localDe=0, & staggerloc=ESMF_STAGGERLOC_CENTER, totalCount=ftc, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! allocate the 2d Fortran array based on retrieved total counts allocate(farray2d(ftc(1), ftc(2))) ! finalize the Field call ESMF_FieldEmptyComplete(field3, grid2d, farray2d, rc=rc) !EOC if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) print *, "Complete a Field created by ESMF_FieldEmptyCreate returned" call ESMF_FieldDestroy(field3, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) deallocate(farray2d) !>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !-------------------------------- Example ----------------------------- !>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !BOE !\subsubsection{Create a 7D Field with a 5D Grid and 2D ungridded bounds ! from a Fortran data array} !\label{sec:field:usage:create_5dgrid_7dptr_2dungridded} ! ! In this example, we will show how to create a 7D Field from a 5D {\tt ! ESMF\_Grid} and 2D ungridded bounds with arbitrary halo widths and ! gridToFieldMap. ! ! We first create a 5D DistGrid and a 5D Grid based on the DistGrid; then ! {\tt ESMF\_GridGetFieldBounds} computes the shape of a 7D array in fsize. We can then ! create a 7D Field from the 5D Grid and the 7D Fortran data array with ! other assimilating parameters. !EOE #ifndef ESMF_NO_GREATER_THAN_4D !BOC ! create a 5d distgrid distgrid5d = ESMF_DistGridCreate(minIndex=(/1,1,1,1,1/), & maxIndex=(/10,4,10,4,6/), regDecomp=(/2,1,2,1,1/), rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! Create a 5d Grid grid5d = ESMF_GridCreate(distgrid=distgrid5d, name="grid", rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! use FieldGet to retrieve total counts call ESMF_GridGetFieldBounds(grid5d, localDe=0, ungriddedLBound=(/1,2/), & ungriddedUBound=(/4,5/), & totalLWidth=(/1,1,1,2,2/), totalUWidth=(/1,2,3,4,5/), & gridToFieldMap=(/3,2,5,4,1/), & totalCount=fsize, & rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! allocate the 7d Fortran array based on retrieved total counts allocate(farray7d(fsize(1), fsize(2), fsize(3), fsize(4), fsize(5), & fsize(6), fsize(7))) ! create the Field field7d = ESMF_FieldCreate(grid5d, farray7d, ESMF_INDEX_DELOCAL, & ungriddedLBound=(/1,2/), ungriddedUBound=(/4,5/), & totalLWidth=(/1,1,1,2,2/), totalUWidth=(/1,2,3,4,5/), & gridToFieldMap=(/3,2,5,4,1/), & rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !EOC #endif !BOE ! A user can allocate the Fortran array in a different manner using the lower and ! upper bounds returned from FieldGet through the optional totalLBound and totalUBound ! arguments. In the following example, we create another 7D Field by retrieving the bounds ! and allocate the Fortran array with this approach. In this scheme, indexing the ! Fortran array is sometimes more convenient than using the shape directly. !EOE #ifndef ESMF_NO_GREATER_THAN_4D !BOC call ESMF_GridGetFieldBounds(grid5d, localDe=0, ungriddedLBound=(/1,2/), & ungriddedUBound=(/4,5/), & totalLWidth=(/1,1,1,2,2/), totalUWidth=(/1,2,3,4,5/), & gridToFieldMap=(/3,2,5,4,1/), & totalLBound=flbound, totalUBound=fubound, & rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) allocate(farray7d2(flbound(1):fubound(1), flbound(2):fubound(2), & flbound(3):fubound(3), flbound(4):fubound(4), & flbound(5):fubound(5), flbound(6):fubound(6), & flbound(7):fubound(7)) ) field7d2 = ESMF_FieldCreate(grid5d, farray7d2, ESMF_INDEX_DELOCAL, & ungriddedLBound=(/1,2/), ungriddedUBound=(/4,5/), & totalLWidth=(/1,1,1,2,2/), totalUWidth=(/1,2,3,4,5/), & gridToFieldMap=(/3,2,5,4,1/), & rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !EOC print *, "Field Create from a Grid and a Fortran data pointer returned" call ESMF_FieldDestroy(field7d) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) call ESMF_FieldDestroy(field7d2) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) call ESMF_GridDestroy(grid5d) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) call ESMF_DistGridDestroy(distgrid5d) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) deallocate(farray7d) deallocate(farray7d2) #endif ! TODO: remove this subsection !!>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !!-------------------------------- Example ----------------------------- !!>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !!BremoveOE !!\subsubsection{Destroy a Field} !!\label{sec:field:usage:destroy} !! !! When finished with an {\tt ESMF\_Field}, the destroy method !! removes it. However, the objects inside the {\tt ESMF\_Field} !! but created externally should be destroyed separately, !! since objects can be added to !! more than one {\tt ESMF\_Field}. For example, the same {\tt ESMF\_Grid} !! can be used in multiple {\tt ESMF\_Field}s. !! !!EremoveOE !!------------------------------------------------------------------------- ! !!BremoveOC call ESMF_FieldDestroy(field1, rc=rc) !!EremoveOC if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !------------------------------------------------------------------------- call ESMF_GridDestroy(grid3d, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) call ESMF_GridDestroy(grid2d, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) call ESMF_ArrayDestroy(array3d, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) deallocate(farray) ! call ESMF_FieldDestroy(field, rc=rc) ! if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! call ESMF_GridDestroy(grid, rc=rc) ! if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !=============================================================================== !!>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !!-------------------------------- Example ----------------------------- !!>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>%>% !BOE ! \subsubsection{Shared memory features: DE pinning, sharing, and migration} ! \label{Field_shared_memory_features} ! ! See \ref{Array_shared_memory_features} for a introduction of the DE pinning feature. ! Here we focus on demonstrating the use of the DE pinning feature in the context of ! ESMF Field. ! ! When an ESMF Field object is created, the specified underlying DistGrid indicates how ! many Decomposition Elements (DEs) are created. Each DE has its own memory ! allocation to hold user data. The DELayout, referenced by the DistGrid, ! determines which PET is considered the {\em owner} of each of the DEs. Queried ! for the local DEs, the Field object returns the list of DEs that are owned by ! the local PET making the query. ! ! By default DEs are {\em pinned} to the PETs under which they were created. ! The memory allocation associated with a specific DE is only defined in the ! VAS of the PET to which the DE is pinned. As a consequence, only the PET ! owning a DE has access to its memory allocation. ! ! On shared memory systems, however, ESMF allows DEs to be pinned to SSIs ! instead of PETs. In this case the PET under which a DE was created is still ! consider the owner, but now {\em all} PETs under the same SSI have access to ! the DE. For this the memory allocation associated with the DE is mapped into ! the VAS of all the PETs under the SSI. ! ! To create an Field with each DE pinned to SSI instead of PET, first query the ! VM for the available level of support. !EOE call ESMF_VMGetCurrent(vm=vm, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) call ESMF_VMGet(vm, localPet=localPet, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !BOC call ESMF_VMGet(vm, ssiSharedMemoryEnabledFlag=ssiSharedMemoryEnabled, rc=rc) !EOC if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !BOC if (ssiSharedMemoryEnabled) then !EOC !BOE ! Knowing that the SSI shared memory feature is available, it is now possible ! to create an Field object with DE to SSI pinning. !EOE !BOC grid = ESMF_GridCreateNoPeriDim(maxIndex=(/40,10/), regDecomp=(/4,1/), & coordSys = ESMF_COORDSYS_CART, & rc=rc) !EOC if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !BOC field = ESMF_FieldCreate(typekind=ESMF_TYPEKIND_R8, grid=grid, & pinflag=ESMF_PIN_DE_TO_SSI, rc=rc) !EOC if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !BOE ! Just as in the cases discussed before, where the same Grid was used, a ! default DELayout with as many DEs as PETs in the VM is constructed. Setting ! the {\tt pinflag} to {\tt ESMF\_PIN\_DE\_TO\_SSI} does not change the ! fact that each PET owns exactly one of the DEs. However, assuming that this ! code is run on a set of PETs that are all located under the same SSI, every ! PET now has {\em access} to all of the DEs. The situation can be observed by ! querying for both the {\tt localDeCount}, and the {\tt ssiLocalDeCount}. !EOE !BOC call ESMF_FieldGet(field, localDeCount=localDeCount, & ssiLocalDeCount=ssiLocalDeCount, rc=rc) !EOC if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !BOE ! \begin{sloppypar} ! Assuming execution on 4 PETs, all located on the same SSI, the values of the ! returned variable are {\tt localDeCount==1} and {\tt ssiLocalDeCount==4} on ! all of the PETs. The mapping between each PET's local DE, and the global DE ! index is provided through the {\tt localDeToDeMap} array argument. The amount ! of mapping information returned is dependent on how large {\tt localDeToDeMap} ! has been sized by the user. For {\tt size(localDeToDeMap)==localDeCount}, ! only mapping information for those DEs {\em owned} by the local PET is filled ! in. However for {\tt size(localDeToDeMap)==ssiLocalDeCount}, mapping ! information for all locally {\em accessible} DEs is returned, including ! those owned by other PETs on the same SSI. ! \end{sloppypar} !EOE !BOC allocate(localDeToDeMap(0:ssiLocalDeCount-1)) call ESMF_FieldGet(field, localDeToDeMap=localDeToDeMap, rc=rc) !EOC if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !BOE ! The first {\tt localDeCount} entries of {\tt localDeToDeMap} are always the ! global DE indices of the DEs {\em owned} by the local PET. The remaining ! {\tt ssiLocalDeCount}-{\tt localDeCount} entries are the global DE indices of ! DEs {\em shared} by other PETs. The ordering of the shared DEs is from ! smallest to greatest, excluding the locally owned DEs, which were already ! listed at the beginning of {\tt localDeToDeMap}. For the current case, again ! assuming execution on 4 PETs all located on the same SSI, we expect the ! following situation: ! ! PET 0: {\tt localDeToDeMap}=={\tt(/0,1,2,3/)}\newline ! PET 1: {\tt localDeToDeMap}=={\tt(/1,0,2,3/)}\newline ! PET 2: {\tt localDeToDeMap}=={\tt(/2,0,1,3/)}\newline ! PET 3: {\tt localDeToDeMap}=={\tt(/3,0,1,2/)}\newline ! ! Each PET can access the memory allocations associated with {\em all} of the ! DEs listed in the {\tt localDeToDeMap} returned by the Field object. Direct ! access to the Fortran array pointer of a specific memory allocation is ! available through {\tt ESMF\_FieldGet()}. Here each PET queries for the ! {\tt farrayPtr} of {\tt localDe==2}, i.e. the 2nd shared DE. !EOE !BOC call ESMF_FieldGet(field, farrayPtr=myFarray, localDe=2, rc=rc) !EOC if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !BOE ! Now variable {\tt myFarray} on PETs 0 and 1 both point to the {\em same} ! memory allocation for global DE 2. Both PETs have access to the same ! piece of shared memory! The same is true for PETs 2 and 3, pointing to the ! shared memory allocation of global DE 1. ! It is important to note that all of the typical considerations surrounding ! shared memory programming apply when accessing shared DEs! Proper ! synchronization between PETs accessing shared DEs is critical to avoid ! {\em race conditions}. Also performance issues like {\em false sharing} ! need to be considered for optimal use. ! ! For a simple demonstration, PETs 0 and 2 fill the entire memory allocation of ! DE 2 and 1, respectively, to a unique value. !EOE !BOC if (localPet==0) then myFarray = 12345.6789d0 else if (localPet==2) then myFarray = 6789.12345d0 endif !EOC !BOE ! Here synchronization is needed before any PETs that share access to the same ! DEs can safely access the data without race condition. The Field class provides ! a simple synchronization method that can be used. !EOE !BOC call ESMF_FieldSync(field, rc=rc) ! prevent race condition !EOC if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !BOE ! Now it is safe for PETs 1 and 3 to access the shared DEs. We expect to find ! the data that was set above. For simplicity of the code only the first ! array element is inspected here. !EOE !BOC if (localPet==1) then if (abs(myFarray(1,1)-12345.6789d0)>1.d10) print *, "bad data detected" else if (localPet==3) then if (abs(myFarray(1,1)-6789.12345d0)>1.d10) print *, "bad data detected" endif !EOC call ESMF_GridDestroy(grid, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) call ESMF_FieldDestroy(field, rc=rc) if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) deallocate(localDeToDeMap) !!B-OE !! Working with shared DEs requires additional bookkeeping on the user code !! level. In some situations, however, DE sharing is simply used as a mechanism !! to {\em move} DEs between PETs without requiring data copies. One practical !! application of this case is the transfer of an Field between two components, !! both of which use the same PEs, but run with different number of PETs. !! These would typically be sequential components that use OpenMP on the user !! level with varying threading levels. !! !! DEs that are pinned to SSI can be moved or {\em migrated} to any PET within !! the SSI. This is accomplished by creating a new Field object from an !! existing Field that was created with {\tt pinflag=ESMF\_PIN\_DE\_TO\_SSI}. !! The information of how the DEs are to migrate between the old and the new !! Field is provided through a DELayout object. This object must have the !! same number of DEs and describes how they map to the PETs on the current VM. !! If this is in the context of a different component, the number of PETs might !! differ from the original VM under which the existing Field was created. This !! situation is explicitly supported, still the number of DEs must match. !! !! Here a simple DELayout is created on the same 4 PETs, but with rotated !! DE ownerships: !! !! DE 0 -> PET 1 (old PET 0)\newline !! DE 1 -> PET 2 (old PET 1)\newline !! DE 2 -> PET 3 (old PET 2)\newline !! DE 3 -> PET 0 (old PET 3)\newline !!E-OE !!B-OC ! delayout = ESMF_DELayoutCreate(petMap=(/1,2,3,0/), rc=rc) ! DE->PET mapping !!E-OC ! if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! !!B-OE !! The creation of the new Field is done by reference, i.e. !! {\tt datacopyflag=ESMF\_DATACOPY\_REFERENCE}, since the new Field does !! not create its own memory allocations. Instead the new Field references the !! shared memory resources held by the incoming Field object. !!E-OE !!B-OC ! call ESMF_FieldGet(field, array=array, rc=rc) !!E-OC ! if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !!B-OC ! arrayMigrated = ESMF_ArrayCreate(array, delayout=delayout, & ! datacopyflag=ESMF_DATACOPY_REFERENCE, rc=rc) !!E-OC ! if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !!B-OC ! fieldMigrated = ESMF_FieldCreate(array=arrayMigrated, grid=grid, rc=rc) !!E-OC ! if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! !!B-OE !! Querying {\tt fieldMigrated} for the number of local DEs will return 1 on !! each PET. Sizing the {\tt localDeToDeMap} accordingly and querying for it. !!E-OE !!B-OC ! deallocate(localDeToDeMap) ! free previous allocation ! allocate(localDeToDeMap(0:1)) ! call ESMF_FieldGet(fieldMigrated, localDeToDeMap=localDeToDeMap, rc=rc) !!E-OC ! if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! !!B-OE !! This yields the following expected outcome: !! !! PET 0: {\tt localDeToDeMap}=={\tt(/1/)}\newline !! PET 1: {\tt localDeToDeMap}=={\tt(/2/)}\newline !! PET 2: {\tt localDeToDeMap}=={\tt(/3/)}\newline !! PET 3: {\tt localDeToDeMap}=={\tt(/0/)}\newline !! !! On each PET the respective Fortran array pointer is returned by the Farray. !!E-OE !!B-OC ! call ESMF_FieldGet(FieldMigrated, farrayPtr=myFarray, rc=rc) !!E-OC ! if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! !!B-OE !! The same situation could have been achieved with the original {\tt field}. !! However, it would have required first finding the correct local DE !! for the target global DE on each PET, and then querying {\tt field} !! accordingly. If needed more repeatedly, this bookkeeping would need to be !! kept in a user code data structure. The DE migration feature on the other !! hand provides a formal way to create a standard ESMF Field object that can be !! used directly in any Field level method as usual, letting ESMF handle the !! extra bookkeeping needed. !!E-OE ! !BOC endif ! ending the ssiSharedMemoryEnabled conditional !EOC ! deallocate(localDeToDeMap) !!B-OE !! Before destroying an Field whose DEs are shared between PETs, it is !! advisable to issue one more synchronization. This prevents cases where a !! PET still might be accessing a shared DE, while the owner PET is already !! destroying the Field, therefore deallocating the shared memory resource. !!E-OE !!B-OC ! call ESMF_FieldSync(field, rc=rc) ! prevent race condition !!E-OC ! if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !!B-OC ! call ESMF_FieldDestroy(field, rc=rc) !!E-OC ! if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) !!B-OE !! Remember that {\tt fieldMigrated} shares the same memory allocations that were !! held by {\tt field}. Field {\tt fieldMigrated} must therefore not be used !! beyond the life time of {\tt field}. Best to destroy it now. !!E-OE !!B-OC ! call ESMF_FieldDestroy(fieldMigrated, rc=rc) !!E-OC ! if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! call ESMF_GridDestroy(grid, rc=rc) ! if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! IMPORTANT: ESMF_STest() prints the PASS string and the # of processors in the log ! file that the scripts grep for. call ESMF_STest((finalrc.eq.ESMF_SUCCESS), testname, failMsg, result, ESMF_SRCLINE) !------------------------------------------------------------------------- call ESMF_Finalize(rc=rc) if (rc.NE.ESMF_SUCCESS) finalrc = ESMF_FAILURE !------------------------------------------------------------------------- if (finalrc.EQ.ESMF_SUCCESS) then print *, "PASS: ESMF_FieldEx.F90" else print *, "FAIL: ESMF_FieldEx.F90" end if end program ESMF_FieldEx