domain/m_domain_neighbour.f90
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | !> @brief Module that defines the TensorProdNeighbour type, which contains information needed for MPI communication | ||
| 2 | module m_domain_neighbour | ||
| 3 | #include "petsc.fi" | ||
| 4 | use m_tensorprod_indices, only: TensorProdIndices, size, Box | ||
| 5 | |||
| 6 | private | ||
| 7 | public :: TensorProdNeighbour, determine_neighbours | ||
| 8 | |||
| 9 | !> @brief The tensor product neighbour type | ||
| 10 | !> | ||
| 11 | !> This type contains information about a neighbour of the current tensor product domain, allowing for data to be exchanged | ||
| 12 | !> between the domains | ||
| 13 | type :: TensorProdNeighbour | ||
| 14 | !> The node ID of the neighbour in the communicator (the linear index corresponding to their_subinterval_ijk(3)). | ||
| 15 | integer :: their_rank | ||
| 16 | |||
| 17 | !> The node/processor ID of the neighbour in the communicator. | ||
| 18 | integer :: their_node | ||
| 19 | |||
| 20 | !> The shared memory rank of the neighbour in the communicator. | ||
| 21 | integer :: their_shmem_rank | ||
| 22 | |||
| 23 | !> The subinterval index in the x,y,z-direction for the neighbour. | ||
| 24 | integer :: their_subinterval_ijk(3) | ||
| 25 | |||
| 26 | !> The global index of the first B-spline in the neighbour's responsible B-spline space. | ||
| 27 | integer :: their_l0 | ||
| 28 | |||
| 29 | !> The global index of the last B-spline in the neighbour's responsible B-spline space. | ||
| 30 | integer :: their_l1 | ||
| 31 | |||
| 32 | !> The B-spline indices that the neighbour is responsible for. | ||
| 33 | type(TensorProdIndices) :: their_rank_resp_bspline | ||
| 34 | |||
| 35 | !> The interval indices that the neighbour is responsible for. | ||
| 36 | type(TensorProdIndices) :: their_rank_resp_intervals | ||
| 37 | |||
| 38 | !> Wether or not the neighbour is sending data to us. | ||
| 39 | logical :: send_to_us | ||
| 40 | !> The tag used for receiving data from the neighbour. | ||
| 41 | integer :: send_tag | ||
| 42 | !> The B-spline indices that the neighbour is sending to us. | ||
| 43 | type(TensorProdIndices) :: send_to_us_inds | ||
| 44 | |||
| 45 | !> Wether or not the neighbour is receiving data from us. | ||
| 46 | logical :: recv_from_us | ||
| 47 | !> The tag used for sending data to the neighbour. | ||
| 48 | integer :: recv_tag | ||
| 49 | !> The B-spline indices that the neighbour is receiving from us. | ||
| 50 | type(TensorProdIndices) :: recv_from_us_inds | ||
| 51 | contains | ||
| 52 | procedure :: init => init_tensorprod_neighbour | ||
| 53 | procedure :: empty => empty_tensorprod_neighbour | ||
| 54 | end type | ||
| 55 | |||
| 56 | contains | ||
| 57 | |||
| 58 | !> @brief Determine the neighbours for exchanging data and for the use of cartesian_to_linear | ||
| 59 | !> | ||
| 60 | !> @param[out] neighbours The neighbours of the current rank, indexed by relative subinterval indices in the x, y, and z | ||
| 61 | !> directions | ||
| 62 | !> @param[in] l0 The global index of the first B-spline in the current rank's responsible B-spline space | ||
| 63 | !> @param[in] l1 The global index of the last B-spline in the current rank's responsible B-spline space | ||
| 64 | !> @param[in] bsplines The B-spline spaces in the x, y, and z directions | ||
| 65 | !> @param[in] domain The TensorProdDomain defining the decomposition | ||
| 66 | !> @param[in] my_actv_bspline The active B-spline indices for the current rank | ||
| 67 | !> @param[in] my_resp_bspline The responsible B-spline indices for the current rank | ||
| 68 | !> @param[in] my_resp_intervals The responsible interval indices for the current rank | ||
| 69 | 59252 | subroutine determine_neighbours(neighbours, l0, l1, bsplines, domain, my_actv_bspline, my_resp_bspline, my_resp_intervals) | |
| 70 | use m_bspline, only: BSplineSpace, size | ||
| 71 | use m_domain_decomp, only: TensorProdDomain, MAX_COMM_NEIGHBOURS_X, MAX_COMM_NEIGHBOURS_Y, MAX_COMM_NEIGHBOURS_Z | ||
| 72 | |||
| 73 | implicit none | ||
| 74 | |||
| 75 | type(TensorProdNeighbour), intent(out) :: neighbours(-MAX_COMM_NEIGHBOURS_X:MAX_COMM_NEIGHBOURS_X, & | ||
| 76 | -MAX_COMM_NEIGHBOURS_Y:MAX_COMM_NEIGHBOURS_Y, & | ||
| 77 | -MAX_COMM_NEIGHBOURS_Z:MAX_COMM_NEIGHBOURS_Z) | ||
| 78 | integer, intent(in) :: l0, l1 | ||
| 79 | type(BSplineSpace), intent(in) :: bsplines(3) | ||
| 80 | type(TensorProdDomain), intent(in) :: domain | ||
| 81 | type(TensorProdIndices), intent(in) :: my_actv_bspline, my_resp_bspline, my_resp_intervals | ||
| 82 | |||
| 83 | integer :: dir, their_rank, ierr, their_l0, actual_nr_ranks | ||
| 84 | integer :: their_sub_ijk(3) | ||
| 85 | integer :: rel_sub_i, rel_sub_j, rel_sub_k | ||
| 86 | type(TensorProdIndices) :: their_resp, their_actv, their_resp_intervals | ||
| 87 | type(TensorProdDomain) :: their_domain | ||
| 88 | |||
| 89 | logical, allocatable :: neighbour_coverage(:, :, :) | ||
| 90 | integer, allocatable :: all_rank_info_linearlyindexed(:, :), all_rank_info(:, :, :, :) ! info, sub_i, sub_j, sub_k | ||
| 91 | |||
| 92 | integer, parameter :: NR_INFO = 25 | ||
| 93 | integer, parameter :: INFO_MY_RANK = 1 | ||
| 94 | integer, parameter :: INFO_MY_NODE = 2 | ||
| 95 | integer, parameter :: INFO_MY_SHMEM_RANK = 3 | ||
| 96 | integer, parameter :: INFO_SUBINTERVAL_I = 4 | ||
| 97 | integer, parameter :: INFO_SUBINTERVAL_J = 5 | ||
| 98 | integer, parameter :: INFO_SUBINTERVAL_K = 6 | ||
| 99 | integer, parameter :: INFO_RESP_I0 = 7 | ||
| 100 | integer, parameter :: INFO_RESP_I1 = 8 | ||
| 101 | integer, parameter :: INFO_RESP_J0 = 9 | ||
| 102 | integer, parameter :: INFO_RESP_J1 = 10 | ||
| 103 | integer, parameter :: INFO_RESP_K0 = 11 | ||
| 104 | integer, parameter :: INFO_RESP_K1 = 12 | ||
| 105 | integer, parameter :: INFO_ACTV_I0 = 13 | ||
| 106 | integer, parameter :: INFO_ACTV_I1 = 14 | ||
| 107 | integer, parameter :: INFO_ACTV_J0 = 15 | ||
| 108 | integer, parameter :: INFO_ACTV_J1 = 16 | ||
| 109 | integer, parameter :: INFO_ACTV_K0 = 17 | ||
| 110 | integer, parameter :: INFO_ACTV_K1 = 18 | ||
| 111 | integer, parameter :: INFO_ACTV_L0 = 19 | ||
| 112 | integer, parameter :: INFO_RESP_INTERVAL_I0 = 20 | ||
| 113 | integer, parameter :: INFO_RESP_INTERVAL_I1 = 21 | ||
| 114 | integer, parameter :: INFO_RESP_INTERVAL_J0 = 22 | ||
| 115 | integer, parameter :: INFO_RESP_INTERVAL_J1 = 23 | ||
| 116 | integer, parameter :: INFO_RESP_INTERVAL_K0 = 24 | ||
| 117 | integer, parameter :: INFO_RESP_INTERVAL_K1 = 25 | ||
| 118 | integer :: my_rank_info(1:NR_INFO) | ||
| 119 | |||
| 120 | ! These boolean arrays are used to ensure that the B-splines for which we are not responsible (i.e., my_actv_bspline setminus | ||
| 121 | ! this%resp_bspline) are present in the neighbours neighbours | ||
| 122 | allocate (neighbour_coverage(my_actv_bspline%i0:my_actv_bspline%i1, & | ||
| 123 | my_actv_bspline%j0:my_actv_bspline%j1, & | ||
| 124 |
12/18✓ Branch 4 → 3 taken 59252 times.
✗ Branch 4 → 5 not taken.
✓ Branch 5 → 6 taken 601 times.
✓ Branch 5 → 7 taken 58651 times.
✓ Branch 7 → 6 taken 58651 times.
✗ Branch 7 → 8 not taken.
✓ Branch 8 → 9 taken 4390 times.
✓ Branch 8 → 10 taken 54862 times.
✓ Branch 10 → 9 taken 54862 times.
✗ Branch 10 → 11 not taken.
✓ Branch 11 → 12 taken 59252 times.
✗ Branch 11 → 13 not taken.
✓ Branch 13 → 14 taken 54261 times.
✓ Branch 13 → 15 taken 4991 times.
✗ Branch 15 → 16 not taken.
✓ Branch 15 → 17 taken 59252 times.
✗ Branch 17 → 18 not taken.
✓ Branch 17 → 19 taken 59252 times.
|
296260 | my_actv_bspline%k0:my_actv_bspline%k1)) |
| 125 | |||
| 126 | ✗ | neighbour_coverage = .false. | |
| 127 | neighbour_coverage(my_resp_bspline%i0:my_resp_bspline%i1, & | ||
| 128 | my_resp_bspline%j0:my_resp_bspline%j1, & | ||
| 129 |
6/6✓ Branch 28 → 29 taken 2920189 times.
✓ Branch 28 → 36 taken 59252 times.
✓ Branch 30 → 31 taken 57141631 times.
✓ Branch 30 → 35 taken 2920189 times.
✓ Branch 32 → 33 taken 647744611 times.
✓ Branch 32 → 34 taken 57141631 times.
|
707865683 | my_resp_bspline%k0:my_resp_bspline%k1) = .true. |
| 130 | |||
| 131 | 59252 | their_domain = domain | |
| 132 | |||
| 133 | ! Initialize all neighbours to empty | ||
| 134 |
2/2✓ Branch 37 → 38 taken 296260 times.
✓ Branch 37 → 46 taken 59252 times.
|
355512 | do rel_sub_k = -MAX_COMM_NEIGHBOURS_Z, MAX_COMM_NEIGHBOURS_Z |
| 135 |
2/2✓ Branch 38 → 39 taken 888780 times.
✓ Branch 38 → 44 taken 296260 times.
|
1244292 | do rel_sub_j = -MAX_COMM_NEIGHBOURS_Y, MAX_COMM_NEIGHBOURS_Y |
| 136 |
2/2✓ Branch 39 → 40 taken 2666340 times.
✓ Branch 39 → 42 taken 888780 times.
|
3851380 | do rel_sub_i = -MAX_COMM_NEIGHBOURS_X, MAX_COMM_NEIGHBOURS_X |
| 137 | 3555120 | call neighbours(rel_sub_i, rel_sub_j, rel_sub_k)%empty | |
| 138 | end do | ||
| 139 | end do | ||
| 140 | end do | ||
| 141 | |||
| 142 |
1/2✗ Branch 48 → 49 not taken.
✓ Branch 48 → 50 taken 59252 times.
|
212660 | PetscCallMPI(MPI_Comm_size(domain%comm, actual_nr_ranks, ierr)) |
| 143 |
2/2✓ Branch 50 → 51 taken 8116 times.
✓ Branch 50 → 217 taken 51136 times.
|
59252 | if (actual_nr_ranks /= domain%nr_ranks) return ! We are in debug mode with a single rank: neighbours cannot be determined |
| 144 | |||
| 145 | ! Communicate rank-specific information with all ranks | ||
| 146 |
6/12✗ Branch 51 → 52 not taken.
✓ Branch 51 → 53 taken 8116 times.
✓ Branch 53 → 52 taken 8116 times.
✗ Branch 53 → 54 not taken.
✓ Branch 54 → 55 taken 8116 times.
✗ Branch 54 → 56 not taken.
✓ Branch 56 → 57 taken 8116 times.
✗ Branch 56 → 58 not taken.
✗ Branch 58 → 59 not taken.
✓ Branch 58 → 60 taken 8116 times.
✗ Branch 60 → 61 not taken.
✓ Branch 60 → 62 taken 8116 times.
|
24348 | allocate (all_rank_info_linearlyindexed(1:NR_INFO, 0:domain%nr_ranks - 1)) |
| 147 | allocate (all_rank_info(1:NR_INFO, 0:domain%nr_subintervals(1) - 1, & | ||
| 148 | 0:domain%nr_subintervals(2) - 1, & | ||
| 149 |
10/20✗ Branch 62 → 63 not taken.
✓ Branch 62 → 64 taken 8116 times.
✓ Branch 64 → 63 taken 8116 times.
✗ Branch 64 → 65 not taken.
✗ Branch 65 → 66 not taken.
✓ Branch 65 → 67 taken 8116 times.
✓ Branch 67 → 66 taken 8116 times.
✗ Branch 67 → 68 not taken.
✗ Branch 68 → 69 not taken.
✓ Branch 68 → 70 taken 8116 times.
✓ Branch 70 → 69 taken 8116 times.
✗ Branch 70 → 71 not taken.
✓ Branch 71 → 72 taken 8116 times.
✗ Branch 71 → 73 not taken.
✓ Branch 73 → 74 taken 8116 times.
✗ Branch 73 → 75 not taken.
✗ Branch 75 → 76 not taken.
✓ Branch 75 → 77 taken 8116 times.
✗ Branch 77 → 78 not taken.
✓ Branch 77 → 79 taken 8116 times.
|
40580 | 0:domain%nr_subintervals(3) - 1)) |
| 150 | |||
| 151 | 8116 | my_rank_info(INFO_MY_RANK) = domain%my_rank | |
| 152 | 8116 | my_rank_info(INFO_MY_NODE) = domain%my_node | |
| 153 | 8116 | my_rank_info(INFO_MY_SHMEM_RANK) = domain%my_shmem_rank | |
| 154 | 8116 | my_rank_info(INFO_SUBINTERVAL_I) = domain%my_subinterval_ijk(1) | |
| 155 | 8116 | my_rank_info(INFO_SUBINTERVAL_J) = domain%my_subinterval_ijk(2) | |
| 156 | 8116 | my_rank_info(INFO_SUBINTERVAL_K) = domain%my_subinterval_ijk(3) | |
| 157 | 8116 | my_rank_info(INFO_RESP_I0) = my_resp_bspline%i0 | |
| 158 | 8116 | my_rank_info(INFO_RESP_I1) = my_resp_bspline%i1 | |
| 159 | 8116 | my_rank_info(INFO_RESP_J0) = my_resp_bspline%j0 | |
| 160 | 8116 | my_rank_info(INFO_RESP_J1) = my_resp_bspline%j1 | |
| 161 | 8116 | my_rank_info(INFO_RESP_K0) = my_resp_bspline%k0 | |
| 162 | 8116 | my_rank_info(INFO_RESP_K1) = my_resp_bspline%k1 | |
| 163 | 8116 | my_rank_info(INFO_ACTV_I0) = my_actv_bspline%i0 | |
| 164 | 8116 | my_rank_info(INFO_ACTV_I1) = my_actv_bspline%i1 | |
| 165 | 8116 | my_rank_info(INFO_ACTV_J0) = my_actv_bspline%j0 | |
| 166 | 8116 | my_rank_info(INFO_ACTV_J1) = my_actv_bspline%j1 | |
| 167 | 8116 | my_rank_info(INFO_ACTV_K0) = my_actv_bspline%k0 | |
| 168 | 8116 | my_rank_info(INFO_ACTV_K1) = my_actv_bspline%k1 | |
| 169 | 8116 | my_rank_info(INFO_ACTV_L0) = l0 | |
| 170 | 8116 | my_rank_info(INFO_RESP_INTERVAL_I0) = my_resp_intervals%i0 | |
| 171 | 8116 | my_rank_info(INFO_RESP_INTERVAL_I1) = my_resp_intervals%i1 | |
| 172 | 8116 | my_rank_info(INFO_RESP_INTERVAL_J0) = my_resp_intervals%j0 | |
| 173 | 8116 | my_rank_info(INFO_RESP_INTERVAL_J1) = my_resp_intervals%j1 | |
| 174 | 8116 | my_rank_info(INFO_RESP_INTERVAL_K0) = my_resp_intervals%k0 | |
| 175 | 8116 | my_rank_info(INFO_RESP_INTERVAL_K1) = my_resp_intervals%k1 | |
| 176 |
1/2✗ Branch 80 → 81 not taken.
✓ Branch 80 → 82 taken 8116 times.
|
8116 | PetscCallMPI(MPI_Allgather(my_rank_info, NR_INFO, MPI_INTEGER, \ |
| 177 | all_rank_info_linearlyindexed, NR_INFO, MPI_INTEGER, domain%comm, ierr)) | ||
| 178 | |||
| 179 | ! Map linearly indexed rank info to subdomain-indexed rank info | ||
| 180 |
2/2✓ Branch 82 → 83 taken 8116 times.
✓ Branch 82 → 87 taken 8116 times.
|
16232 | do their_rank = 0, domain%nr_ranks - 1 |
| 181 | 8116 | their_sub_ijk(1) = all_rank_info_linearlyindexed(INFO_SUBINTERVAL_I, their_rank) | |
| 182 | 8116 | their_sub_ijk(2) = all_rank_info_linearlyindexed(INFO_SUBINTERVAL_J, their_rank) | |
| 183 | 8116 | their_sub_ijk(3) = all_rank_info_linearlyindexed(INFO_SUBINTERVAL_K, their_rank) | |
| 184 |
2/2✓ Branch 84 → 85 taken 202900 times.
✓ Branch 84 → 86 taken 8116 times.
|
219132 | all_rank_info(:, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)) = all_rank_info_linearlyindexed(:, their_rank) |
| 185 | end do | ||
| 186 | |||
| 187 | ! Initialize neighbours by looping over all possible neighbouring subdomains | ||
| 188 |
2/2✓ Branch 89 → 90 taken 8116 times.
✓ Branch 89 → 127 taken 8116 times.
|
16232 | do rel_sub_k = -domain%nr_eff_neighbours(3), domain%nr_eff_neighbours(3) |
| 189 |
2/2✓ Branch 91 → 92 taken 8116 times.
✓ Branch 91 → 125 taken 8116 times.
|
24348 | do rel_sub_j = -domain%nr_eff_neighbours(2), domain%nr_eff_neighbours(2) |
| 190 |
2/2✓ Branch 93 → 94 taken 8116 times.
✓ Branch 93 → 123 taken 8116 times.
|
24348 | do rel_sub_i = -domain%nr_eff_neighbours(1), domain%nr_eff_neighbours(1) |
| 191 | 8116 | their_sub_ijk(1) = domain%my_subinterval_ijk(1) + rel_sub_i | |
| 192 | 8116 | their_sub_ijk(2) = domain%my_subinterval_ijk(2) + rel_sub_j | |
| 193 | 8116 | their_sub_ijk(3) = domain%my_subinterval_ijk(3) + rel_sub_k | |
| 194 | |||
| 195 | ! Impose periodicity if needed | ||
| 196 |
2/2✓ Branch 95 → 96 taken 24348 times.
✓ Branch 95 → 99 taken 8116 times.
|
32464 | do dir = 1, 3 |
| 197 |
2/2✓ Branch 96 → 97 taken 10067 times.
✓ Branch 96 → 98 taken 14281 times.
|
32464 | if (bsplines(dir)%is_periodic) then |
| 198 | 10067 | their_sub_ijk(dir) = modulo(their_sub_ijk(dir), domain%nr_subintervals(dir)) | |
| 199 | end if | ||
| 200 | end do | ||
| 201 | |||
| 202 |
5/8✓ Branch 100 → 101 taken 24348 times.
✓ Branch 100 → 104 taken 8116 times.
✓ Branch 101 → 102 taken 24348 times.
✗ Branch 101 → 104 not taken.
✓ Branch 102 → 103 taken 24348 times.
✗ Branch 102 → 104 not taken.
✓ Branch 104 → 105 taken 8116 times.
✗ Branch 104 → 122 not taken.
|
32464 | if (any(their_sub_ijk < 0 .or. their_sub_ijk >= domain%nr_subintervals)) cycle ! Out of bounds and non-periodic |
| 203 | |||
| 204 | 8116 | their_domain%my_rank = all_rank_info(INFO_MY_RANK, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)) | |
| 205 | 8116 | their_domain%my_node = all_rank_info(INFO_MY_NODE, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)) | |
| 206 | 8116 | their_domain%my_shmem_rank = all_rank_info(INFO_MY_SHMEM_RANK, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)) | |
| 207 |
2/2✓ Branch 106 → 107 taken 24348 times.
✓ Branch 106 → 108 taken 8116 times.
|
32464 | their_domain%my_subinterval_ijk = their_sub_ijk |
| 208 | |||
| 209 | call their_resp%init( & | ||
| 210 | all_rank_info(INFO_RESP_I0, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)), & | ||
| 211 | all_rank_info(INFO_RESP_I1, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)), & | ||
| 212 | all_rank_info(INFO_RESP_J0, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)), & | ||
| 213 | all_rank_info(INFO_RESP_J1, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)), & | ||
| 214 | all_rank_info(INFO_RESP_K0, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)), & | ||
| 215 | all_rank_info(INFO_RESP_K1, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)), & | ||
| 216 | 8116 | bsplines) | |
| 217 | call their_actv%init( & | ||
| 218 | all_rank_info(INFO_ACTV_I0, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)), & | ||
| 219 | all_rank_info(INFO_ACTV_I1, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)), & | ||
| 220 | all_rank_info(INFO_ACTV_J0, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)), & | ||
| 221 | all_rank_info(INFO_ACTV_J1, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)), & | ||
| 222 | all_rank_info(INFO_ACTV_K0, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)), & | ||
| 223 | all_rank_info(INFO_ACTV_K1, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)), & | ||
| 224 | 8116 | bsplines) | |
| 225 | |||
| 226 | 8116 | their_l0 = all_rank_info(INFO_ACTV_L0, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)) | |
| 227 | |||
| 228 | call their_resp_intervals%init( & | ||
| 229 | all_rank_info(INFO_RESP_INTERVAL_I0, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)), & | ||
| 230 | all_rank_info(INFO_RESP_INTERVAL_I1, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)), & | ||
| 231 | all_rank_info(INFO_RESP_INTERVAL_J0, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)), & | ||
| 232 | all_rank_info(INFO_RESP_INTERVAL_J1, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)), & | ||
| 233 | all_rank_info(INFO_RESP_INTERVAL_K0, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)), & | ||
| 234 | all_rank_info(INFO_RESP_INTERVAL_K1, their_sub_ijk(1), their_sub_ijk(2), their_sub_ijk(3)), & | ||
| 235 | 8116 | bsplines, intervals=.true.) | |
| 236 | |||
| 237 | call neighbours(rel_sub_i, rel_sub_j, rel_sub_k)%init( & | ||
| 238 | domain, my_resp_bspline, my_actv_bspline, their_domain, their_resp, their_actv, their_l0, & | ||
| 239 | 8116 | size(bsplines(1)), size(bsplines(2)), size(bsplines(3))) | |
| 240 | |||
| 241 | 8116 | neighbours(rel_sub_i, rel_sub_j, rel_sub_k)%their_rank_resp_intervals = their_resp_intervals | |
| 242 | |||
| 243 |
1/2✗ Branch 112 → 113 not taken.
✓ Branch 112 → 122 taken 8116 times.
|
16232 | if (neighbours(rel_sub_i, rel_sub_j, rel_sub_k)%send_to_us) then |
| 244 | neighbour_coverage( & | ||
| 245 | & neighbours(rel_sub_i, rel_sub_j, rel_sub_k)%send_to_us_inds%i0: & | ||
| 246 | & neighbours(rel_sub_i, rel_sub_j, rel_sub_k)%send_to_us_inds%i1, & | ||
| 247 | & neighbours(rel_sub_i, rel_sub_j, rel_sub_k)%send_to_us_inds%j0: & | ||
| 248 | & neighbours(rel_sub_i, rel_sub_j, rel_sub_k)%send_to_us_inds%j1, & | ||
| 249 | & neighbours(rel_sub_i, rel_sub_j, rel_sub_k)%send_to_us_inds%k0: & | ||
| 250 | & neighbours(rel_sub_i, rel_sub_j, rel_sub_k)%send_to_us_inds%k1) & | ||
| 251 | ✗ | & = .true. | |
| 252 | end if | ||
| 253 | |||
| 254 | end do | ||
| 255 | end do | ||
| 256 | end do | ||
| 257 | |||
| 258 | ! Handle periodicity for the case of a single subinterval in a periodic direction (for the sanity check) | ||
| 259 |
4/6✓ Branch 128 → 129 taken 1133 times.
✓ Branch 128 → 130 taken 6983 times.
✗ Branch 129 → 130 not taken.
✓ Branch 129 → 131 taken 1133 times.
✗ Branch 130 → 131 not taken.
✓ Branch 130 → 149 taken 6983 times.
|
8116 | if ((bsplines(1)%is_periodic .and. domain%nr_subintervals(1) == 1) .or. domain%is_shared_memory(1)) then |
| 260 | neighbour_coverage(my_actv_bspline%i0:my_resp_bspline%i1 - 1, & | ||
| 261 | my_actv_bspline%j0:my_actv_bspline%j1, & | ||
| 262 |
6/6✓ Branch 132 → 133 taken 7948 times.
✓ Branch 132 → 140 taken 1133 times.
✓ Branch 134 → 135 taken 54657 times.
✓ Branch 134 → 139 taken 7948 times.
✓ Branch 136 → 137 taken 1280378 times.
✓ Branch 136 → 138 taken 54657 times.
|
1344116 | my_actv_bspline%k0:my_actv_bspline%k1) = .true. |
| 263 | neighbour_coverage(my_resp_bspline%i1 + 1:my_actv_bspline%i1, & | ||
| 264 | my_actv_bspline%j0:my_actv_bspline%j1, & | ||
| 265 |
6/6✓ Branch 141 → 142 taken 7948 times.
✓ Branch 141 → 150 taken 1133 times.
✓ Branch 143 → 144 taken 54657 times.
✓ Branch 143 → 148 taken 7948 times.
✓ Branch 145 → 146 taken 179246 times.
✓ Branch 145 → 147 taken 54657 times.
|
249967 | my_actv_bspline%k0:my_actv_bspline%k1) = .true. |
| 266 | end if | ||
| 267 |
4/6✓ Branch 150 → 151 taken 6595 times.
✓ Branch 150 → 152 taken 1521 times.
✗ Branch 151 → 152 not taken.
✓ Branch 151 → 153 taken 6595 times.
✗ Branch 152 → 153 not taken.
✓ Branch 152 → 171 taken 1521 times.
|
8116 | if ((bsplines(2)%is_periodic .and. domain%nr_subintervals(2) == 1) .or. domain%is_shared_memory(2)) then |
| 268 | neighbour_coverage(my_actv_bspline%i0:my_actv_bspline%i1, & | ||
| 269 | my_actv_bspline%j0:my_resp_bspline%j1 - 1, & | ||
| 270 |
6/6✓ Branch 154 → 155 taken 60509 times.
✓ Branch 154 → 162 taken 6595 times.
✓ Branch 156 → 157 taken 2316981 times.
✓ Branch 156 → 161 taken 60509 times.
✓ Branch 158 → 159 taken 136720560 times.
✓ Branch 158 → 160 taken 2316981 times.
|
139104645 | my_actv_bspline%k0:my_actv_bspline%k1) = .true. |
| 271 | neighbour_coverage(my_actv_bspline%i0:my_actv_bspline%i1, & | ||
| 272 | my_resp_bspline%j1 + 1:my_actv_bspline%j1, & | ||
| 273 |
6/6✓ Branch 163 → 164 taken 60509 times.
✓ Branch 163 → 172 taken 6595 times.
✓ Branch 165 → 166 taken 191644 times.
✓ Branch 165 → 170 taken 60509 times.
✓ Branch 167 → 168 taken 9464774 times.
✓ Branch 167 → 169 taken 191644 times.
|
9725043 | my_actv_bspline%k0:my_actv_bspline%k1) = .true. |
| 274 | end if | ||
| 275 |
4/6✓ Branch 172 → 173 taken 2339 times.
✓ Branch 172 → 174 taken 5777 times.
✗ Branch 173 → 174 not taken.
✓ Branch 173 → 175 taken 2339 times.
✗ Branch 174 → 175 not taken.
✓ Branch 174 → 193 taken 5777 times.
|
8116 | if ((bsplines(3)%is_periodic .and. domain%nr_subintervals(3) == 1) .or. domain%is_shared_memory(3)) then |
| 276 | neighbour_coverage(my_actv_bspline%i0:my_actv_bspline%i1, & | ||
| 277 | my_actv_bspline%j0:my_actv_bspline%j1, & | ||
| 278 |
6/6✓ Branch 176 → 177 taken 23949 times.
✓ Branch 176 → 184 taken 2339 times.
✓ Branch 178 → 179 taken 837678 times.
✓ Branch 178 → 183 taken 23949 times.
✓ Branch 180 → 181 taken 27710132 times.
✓ Branch 180 → 182 taken 837678 times.
|
28574098 | my_actv_bspline%k0:my_resp_bspline%k1 - 1) = .true. |
| 279 | neighbour_coverage(my_actv_bspline%i0:my_actv_bspline%i1, & | ||
| 280 | my_actv_bspline%j0:my_actv_bspline%j1, & | ||
| 281 |
6/6✓ Branch 185 → 186 taken 4494 times.
✓ Branch 185 → 194 taken 2339 times.
✓ Branch 187 → 188 taken 114799 times.
✓ Branch 187 → 192 taken 4494 times.
✓ Branch 189 → 190 taken 2307347 times.
✓ Branch 189 → 191 taken 114799 times.
|
2434756 | my_resp_bspline%k1 + 1:my_actv_bspline%k1) = .true. |
| 282 | end if | ||
| 283 | |||
| 284 | ! Sanity check: ensure that all active B-spline indices are covered by neighbours | ||
| 285 | ! TODO: this is sufficient for communication of coefficients, but it does not guarantee functionality of cartesian_to_linear | ||
| 286 |
8/10✓ Branch 195 → 196 taken 72020 times.
✓ Branch 195 → 204 taken 8116 times.
✓ Branch 197 → 198 taken 2659204 times.
✓ Branch 197 → 203 taken 72020 times.
✓ Branch 199 → 200 taken 151177734 times.
✓ Branch 199 → 202 taken 2659204 times.
✓ Branch 200 → 201 taken 151177734 times.
✗ Branch 200 → 204 not taken.
✗ Branch 204 → 205 not taken.
✓ Branch 204 → 210 taken 8116 times.
|
153917074 | if (.not. all(neighbour_coverage)) then |
| 287 | ✗ | print*,'Problem for rank ', domain%my_rank | |
| 288 | error stop 'DomainDecomp::determine_neighbours: Not all active B-spline indices are covered by '// & | ||
| 289 | & 'neighbours (EITHER reduce the number of subintervals such that the number of B-splines per subinterval is at least'// & | ||
| 290 | & 'equal to the degree (per direction), OR increase the compile-time constant MAX_COMM_NEIGHBOURS_X/Y/Z by calling cmake with'// & | ||
| 291 | & ' -DdeRham_MAX_COMM_NEIGHBOURS_<X/Y/Z>=degree_<x/y/z>, OR decrease the compile-time constant GUARD_LAYER by calling cmake'// & | ||
| 292 | ✗ | & ' with -DdeRham_GUARD_LAYER=guard_layer_size).' | |
| 293 | end if | ||
| 294 | |||
| 295 |
1/2✗ Branch 210 → 211 not taken.
✓ Branch 210 → 212 taken 8116 times.
|
8116 | deallocate (neighbour_coverage) |
| 296 |
1/2✗ Branch 212 → 213 not taken.
✓ Branch 212 → 214 taken 8116 times.
|
8116 | deallocate (all_rank_info_linearlyindexed) |
| 297 |
1/2✗ Branch 214 → 215 not taken.
✓ Branch 214 → 216 taken 8116 times.
|
8116 | deallocate (all_rank_info) |
| 298 |
4/8✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 59252 times.
✗ Branch 218 → 219 not taken.
✓ Branch 218 → 220 taken 51136 times.
✓ Branch 222 → 223 taken 51136 times.
✗ Branch 222 → 224 not taken.
✓ Branch 226 → 227 taken 51136 times.
✗ Branch 226 → 228 not taken.
|
271912 | end subroutine determine_neighbours |
| 299 | |||
| 300 | !> @brief Initialize an empty TensorProdNeighbour | ||
| 301 | !> | ||
| 302 | !> @param[inout] this The TensorProdNeighbour to initialize | ||
| 303 | 2666340 | subroutine empty_tensorprod_neighbour(this) | |
| 304 | implicit none | ||
| 305 | class(TensorProdNeighbour), intent(inout) :: this | ||
| 306 | |||
| 307 | 2666340 | this%their_rank = -1 | |
| 308 |
2/2✓ Branch 3 → 4 taken 7999020 times.
✓ Branch 3 → 5 taken 2666340 times.
|
10665360 | this%their_subinterval_ijk = -1 |
| 309 | 2666340 | this%their_l0 = -1 | |
| 310 | 2666340 | call this%their_rank_resp_bspline%empty() | |
| 311 | 2666340 | call this%their_rank_resp_intervals%empty() | |
| 312 | |||
| 313 | 2666340 | this%send_to_us = .false. | |
| 314 | 2666340 | this%send_tag = -1 | |
| 315 | 2666340 | call this%send_to_us_inds%empty() | |
| 316 | |||
| 317 | 2666340 | this%recv_from_us = .false. | |
| 318 | 2666340 | this%recv_tag = -1 | |
| 319 | 2666340 | call this%recv_from_us_inds%empty() | |
| 320 | |||
| 321 | 2666340 | end subroutine empty_tensorprod_neighbour | |
| 322 | |||
| 323 | !> @brief Initialize a TensorProdNeighbour | ||
| 324 | !> | ||
| 325 | !> @param[inout] this The TensorProdNeighbour to initialize | ||
| 326 | !> @param[in] my_domain The TensorProdDomain for the current rank | ||
| 327 | !> @param[in] my_resp_inds The responsible B-spline indices for the current rank | ||
| 328 | !> @param[in] my_actv_inds The active B-spline indices for the current rank | ||
| 329 | !> @param[in] their_domain The TensorProdDomain of the neighbour | ||
| 330 | !> @param[in] their_resp_inds The responsible B-spline indices for the neighbour | ||
| 331 | !> @param[in] their_act_inds The active B-spline indices for the neighbour | ||
| 332 | !> @param[in] their_l0 The global index of the first B-spline in the neighbour's responsible B-spline space | ||
| 333 | !> @param[in] nr_x The number of B-splines in the x-direction | ||
| 334 | !> @param[in] nr_y The number of B-splines in the y-direction | ||
| 335 | !> @param[in] nr_z The number of B-splines in the z-direction | ||
| 336 | 8116 | subroutine init_tensorprod_neighbour(this, my_domain, my_resp_inds, my_actv_inds, & | |
| 337 | their_domain, their_resp_inds, their_act_inds, their_l0, nr_x, nr_y, nr_z) | ||
| 338 | use m_tensorprod_indices, only: intersect | ||
| 339 | use m_domain_decomp, only: TensorProdDomain | ||
| 340 | implicit none | ||
| 341 | |||
| 342 | class(TensorProdNeighbour), intent(inout) :: this | ||
| 343 | type(TensorProdIndices), intent(in) :: my_resp_inds, my_actv_inds, their_resp_inds, their_act_inds | ||
| 344 | type(TensorProdDomain), intent(in) :: my_domain, their_domain | ||
| 345 | integer, intent(in) :: their_l0 | ||
| 346 | integer, intent(in) :: nr_x, nr_y, nr_z | ||
| 347 | |||
| 348 | type(TensorProdIndices) :: their_resp_inds_, their_act_inds_ | ||
| 349 | |||
| 350 | ! We compute intersections: | ||
| 351 | ! - between our responsible indices and their active indices: send to them | ||
| 352 | ! - between their responsible indices and our active indices: receive from them | ||
| 353 | |||
| 354 | ! TODO: make neighbour derive from DomainDecomp? | ||
| 355 | 8116 | this%their_rank = their_domain%my_rank | |
| 356 | 8116 | this%their_node = their_domain%my_node | |
| 357 | 8116 | this%their_shmem_rank = their_domain%my_shmem_rank | |
| 358 |
2/2✓ Branch 3 → 4 taken 24348 times.
✓ Branch 3 → 5 taken 8116 times.
|
32464 | this%their_subinterval_ijk = their_domain%my_subinterval_ijk |
| 359 | 8116 | this%their_l0 = their_l0 | |
| 360 | 8116 | this%their_l1 = their_l0 + size(their_resp_inds) - 1 | |
| 361 | 8116 | this%their_rank_resp_bspline = their_resp_inds | |
| 362 | |||
| 363 | 8116 | this%recv_tag = 0 | |
| 364 | 8116 | this%send_tag = 0 | |
| 365 |
1/2✓ Branch 5 → 6 taken 8116 times.
✗ Branch 5 → 9 not taken.
|
8116 | if (my_domain%my_node == their_domain%my_node) then |
| 366 | ! Intra-node communication is done via shared memory windows | ||
| 367 | 8116 | call this%recv_from_us_inds%empty() | |
| 368 | 8116 | call this%send_to_us_inds%empty() | |
| 369 | else | ||
| 370 | ✗ | their_resp_inds_ = their_resp_inds | |
| 371 | ✗ | their_act_inds_ = their_act_inds | |
| 372 | |||
| 373 | call match_indices_with_ours_1d(their_act_inds_%i0, their_act_inds_%i1, my_resp_inds%i0, my_resp_inds%i1, & | ||
| 374 | ✗ | & my_domain%nr_subintervals(1), nr_x) | |
| 375 | call match_indices_with_ours_1d(their_act_inds_%j0, their_act_inds_%j1, my_resp_inds%j0, my_resp_inds%j1, & | ||
| 376 | ✗ | & my_domain%nr_subintervals(2), nr_y) | |
| 377 | call match_indices_with_ours_1d(their_act_inds_%k0, their_act_inds_%k1, my_resp_inds%k0, my_resp_inds%k1, & | ||
| 378 | ✗ | & my_domain%nr_subintervals(3), nr_z) | |
| 379 | |||
| 380 | ✗ | this%recv_from_us_inds = intersect(my_resp_inds, their_act_inds_) | |
| 381 | |||
| 382 | call match_indices_with_ours_1d(their_resp_inds_%i0, their_resp_inds_%i1, my_actv_inds%i0, my_actv_inds%i1, & | ||
| 383 | ✗ | & my_domain%nr_subintervals(1), nr_x) | |
| 384 | call match_indices_with_ours_1d(their_resp_inds_%j0, their_resp_inds_%j1, my_actv_inds%j0, my_actv_inds%j1, & | ||
| 385 | ✗ | & my_domain%nr_subintervals(2), nr_y) | |
| 386 | call match_indices_with_ours_1d(their_resp_inds_%k0, their_resp_inds_%k1, my_actv_inds%k0, my_actv_inds%k1, & | ||
| 387 | ✗ | & my_domain%nr_subintervals(3), nr_z) | |
| 388 | |||
| 389 | ✗ | this%send_to_us_inds = intersect(their_resp_inds_, my_actv_inds) | |
| 390 | end if | ||
| 391 | |||
| 392 | 8116 | this%recv_from_us = size(this%recv_from_us_inds) > 0 | |
| 393 | 8116 | this%send_to_us = size(this%send_to_us_inds) > 0 | |
| 394 | 8116 | end subroutine | |
| 395 | |||
| 396 | !> @brief Match the indices of the other rank with our indices in one direction (i.e., impose periodicity on the indices when | ||
| 397 | !> needed) | ||
| 398 | !> | ||
| 399 | !> @param[inout] their_i0 The start index of the other rank's indices | ||
| 400 | !> @param[inout] their_i1 The end index of the other rank's indices | ||
| 401 | !> @param[in] my_i0 The start index of our indices | ||
| 402 | !> @param[in] my_i1 The end index of our indices | ||
| 403 | !> @param[in] nr_sub_x The number of subintervals in the x-direction | ||
| 404 | !> @param[in] nr_x The number of B-splines in the x-direction | ||
| 405 | ✗ | subroutine match_indices_with_ours_1d(their_i0, their_i1, my_i0, my_i1, nr_sub_x, nr_x) | |
| 406 | implicit none | ||
| 407 | |||
| 408 | integer, intent(inout) :: their_i0, their_i1 | ||
| 409 | integer, intent(in) :: my_i0, my_i1, nr_sub_x, nr_x | ||
| 410 | |||
| 411 | ✗ | if (nr_sub_x == 1) return | |
| 412 | |||
| 413 | ! Match the indices of the other rank with our indices | ||
| 414 | ✗ | if (max(their_i0, my_i0) > min(their_i1, my_i1)) then | |
| 415 | ✗ | if (max(their_i0 - nr_x, my_i0) <= min(their_i1 - nr_x, my_i1)) then | |
| 416 | ✗ | their_i0 = their_i0 - nr_x | |
| 417 | ✗ | their_i1 = their_i1 - nr_x | |
| 418 | ✗ | else if (max(their_i0 + nr_x, my_i0) <= min(their_i1 + nr_x, my_i1)) then | |
| 419 | ✗ | their_i0 = their_i0 + nr_x | |
| 420 | ✗ | their_i1 = their_i1 + nr_x | |
| 421 | end if | ||
| 422 | end if | ||
| 423 | end subroutine match_indices_with_ours_1d | ||
| 424 | |||
| 425 | ✗ | end module m_domain_neighbour | |
| 426 |