ESPResSo 3.2.0-11-g9950804-git Extensible Simulation Package for Soft Matter Research
ghosts.h
Go to the documentation of this file.
00001 /*
00002   Copyright (C) 2010,2011,2012,2013 The ESPResSo project
00004     Max-Planck-Institute for Polymer Research, Theory Group
00005
00006   This file is part of ESPResSo.
00007
00008   ESPResSo is free software: you can redistribute it and/or modify
00010   the Free Software Foundation, either version 3 of the License, or
00011   (at your option) any later version.
00012
00013   ESPResSo is distributed in the hope that it will be useful,
00014   but WITHOUT ANY WARRANTY; without even the implied warranty of
00015   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00016   GNU General Public License for more details.
00017
00018   You should have received a copy of the GNU General Public License
00019   along with this program.  If not, see <http://www.gnu.org/licenses/>.
00020 */
00021 #ifndef _GHOSTS_H
00022 #define _GHOSTS_H
00023 /** \file ghosts.h    Ghost particles and particle exchange.
00024
00025 In this file you find everything concerning the exchange of
00026 particle data (particles, ghosts, positions and forces) for short
00027 range interactions between the spacial domains of neighbouring
00028 nodes.
00029
00030 <h2> How does this work </h2>
00031 The ghost communication transfers data from cells on one node to cells on another node during
00032 the integration process. Note that data can only be transferred from one cell to one other, and
00033 the contents of the other cell will be overwritten. This communication is invoked by the integrator,
00034 at four different times in an integration step. These steps are reflected in \ref cell_structure structure,
00035 since they are treated differently by different cell systems:
00036 <ul>
00037 <li> ghost_cells is used to transfer the cell sizes, i. e. makes sure that for all later transfers
00038       the target cell has the same size as the source cell.
00039 <li> exchange_ghosts sets up all information on the ghosts that is necessary. Normally transfers the
00040       (shifted) position and particle properties.
00041 <li> update_ghosts is used to update the particle properties if no particles have been moved between cells.
00042       Therefore only the positions are transferred, otherwise this normally looks pretty much like the
00043       exchange_ghosts.
00044 <li> collect_ghost_forces finally is used to transfer back forces that were exerted on a ghost particle. They
00045       are simply added again to the force of the real particle. The communication process is therefore inverted
00046       with respect to exchange_ghosts and update_ghosts, i. e. sending is replaced by receiving and the other way
00047       round.
00048 </ul>
00049 The particle data that has to be transferred, and especially from where to where, heavily depends on the cell system.
00050 In Espresso, this is abstracted in form of ghost communicators (\ref GhostCommunicator) and ghost communications
00051 (\ref GhostCommunication). The ghost communicators represent the four communications above and consist of the data to
00052 transfer (which is determined by their type) and a list of ghost communications.
00053 The data types are described by the particle data classes
00054 <ul>
00055 <li> GHOSTTRANS_PROPRTS transfers the \ref ParticleProperties
00056 <li> GHOSTTRANS_POSITION transfers the \ref ParticlePosition
00057 <li> GHOSTTRANS_POSSHFTD is no transfer class, but marks that an additional vector, the shift, has to be
00058       added to the positions. Can only be used together with GHOSTTRANS_POSITION.
00059 <li> GHOSTTRANS_MOMENTUM transfers the \ref ParticleMomentum
00060 <li> GHOSTTRANS_FORCE transfers the \ref ParticleForce
00061 <li> GHOSTTRANS_PARTNUM transfers the cell sizes
00062 </ul>
00063 Each ghost communication describes a single communication of the local with another node (or all other nodes). The data
00064 transferred can be any number of cells, there are five communication types:
00065 <ul>
00066 <li> GHOST_SEND sends data to one other node
00067 <li> GHOST_RECV recvs data from one other node. In the case of forces, they are added up, everything else is overwritten
00068 <li> GHOST_BCST sends data to all nodes
00069 <li> GHOST_RDCE recvs data from all nodes.  In the case of forces, they are added up, everything else is overwritten
00070 <li> GHOST_LOCL transfer data from a local cell to another local cell. In this case, the first half of the cells are the sending cells,
00071       the other half are the corresponding receivers.
00072 </ul>
00073 Note that for the first four communications you have to make sure that when one node sends to another, that the sender has GHOST_SEND and the
00074 receiver GHOST_RECV. In the case of GHOST_BCST rsp. GHOST_RDCE, all nodes have to have the same communication type and the same master
00075 sender/receiver (just like the MPI commands).
00076
00077 A special topic are GHOST_PREFETCH and GHOST_PSTSTORE. For example, if all nodes broadcast to the other, the naive implementation will be that
00078 n_nodes times a GHOST_BCST is done with different master nodes. But this means that each time n_nodes-1 nodes wait for the master to construct
00079 its send buffer. Therefore there is the prefetch flag which can be set on a pair of recv/send operations. If the ghost communication reaches a
00080 recv operation with prefetch, the next send operation (which must have the prefetch set!!) is searched and the send buffer already created.
00081 When sending, this precreated send buffer is used. In the scenario above, all nodes create the send buffers simultaneously in the first
00082 communication step, thereby reducing the latency a little bit. The pststore is similar and postpones the write back of received data until a
00083 send operation (with a precreated send buffer) is finished.
00084
00085 The ghost communicators are created in the init routines of the cell systems, therefore have a look at \ref dd_topology_init or
00086 \ref nsq_topology_init for further details.
00087 */
00088 #include <mpi.h>
00089 #include "particle_data.h"
00090
00091 /** \name Transfer types, for \ref GhostCommunicator::type */
00092 /************************************************************/
00093 /*@{*/
00094
00095 /// send to a single node
00096 #define GHOST_SEND 0
00097 /// recv from a single node
00098 #define GHOST_RECV 1
00099 /// broadcast, the node entry gives the sender
00100 #define GHOST_BCST 2
00101 /// reduce, the node entry gives the receiver
00102 #define GHOST_RDCE 3
00103 /// transfer data from cell to cell on this node
00104 #define GHOST_LOCL 4
00105
00106 /// mask to the job area of the transfer type
00108 /// additional flag for prefetching
00109 #define GHOST_PREFETCH 16
00110 /// additional flag for poststoring
00111 #define GHOST_PSTSTORE 32
00112 /*@}*/
00113
00114
00115 /** \name Transfer data classes, for \ref GhostCommunication::type */
00116 /************************************************************/
00117 /*@{*/
00118 /// transfer \ref ParticleProperties
00119 #define GHOSTTRANS_PROPRTS  1
00120 /// transfer \ref ParticlePosition
00121 #define GHOSTTRANS_POSITION 2
00122 /** flag for \ref GHOSTTRANS_POSITION, shift the positions by \ref GhostCommunication::shift.
00123     Must be or'd together with \ref GHOSTTRANS_POSITION */
00124 #define GHOSTTRANS_POSSHFTD 4
00125 /// transfer \ref ParticleMomentum
00126 #define GHOSTTRANS_MOMENTUM 8
00127 /// transfer \ref ParticleForce
00128 #define GHOSTTRANS_FORCE    16
00129
00130 #ifdef LB
00131 /// transfer \ref ParticleLatticeCoupling
00132 #define GHOSTTRANS_COUPLING 32
00133 #endif
00134
00135 /// resize the receiver particle arrays to the size of the senders
00136 #define GHOSTTRANS_PARTNUM  64
00137 /*@}*/
00138
00139 /** \name Data Types */
00140 /************************************************************/
00141 /*@{*/
00142
00143 typedef struct {
00144
00145   /** Communication type. */
00146   int type;
00147   /** Node to communicate with (to use with all MPI operations). */
00148   int node;
00149   /** MPI communicator handle (to use with GHOST_BCST, GHOST_GATH, GHOST_RDCE). */
00150   MPI_Comm mpi_comm;
00151
00152   /** Number of particle lists to communicate. */
00153   int n_part_lists;
00154   /** Pointer array to particle lists to communicate. */
00155   ParticleList **part_lists;
00156
00157   /** if \ref GhostCommunicator::data_parts has \ref GHOSTTRANS_POSSHFTD, then this is the shift vector.
00158       Normally this a integer multiple of the box length. The shift is done on the sender side */
00159   double shift[3];
00160 } GhostCommunication;
00161
00162 /** Properties for a ghost communication. A ghost communication is defined */
00163 typedef struct {
00164
00165   /** Particle data parts to transfer */
00166   int data_parts;
00167
00168   /** number of communication steps. */
00169   int num;
00170
00171   /** List of ghost communications. */
00172   GhostCommunication *comm;
00173
00174 } GhostCommunicator;
00175
00176 /*@}*/
00177
00178 /** \name Exported Functions */
00179 /************************************************************/
00180 /*@{*/
00181
00182 /** Initialize a communicator. */
00183 void prepare_comm(GhostCommunicator *comm, int data_parts, int num);
00184
00185 /** Free a communicator. */
00186 void free_comm(GhostCommunicator *comm);
00187
00188 /** Initialize ghosts. */
00189 void ghost_init();
00190
00191 /** do a ghost communication */
00192 void ghost_communicator(GhostCommunicator *gc);
00193
00194 /** Go through \ref ghost_cells and remove the ghost entries from \ref
00195     local_particles. Part of \ref dd_exchange_and_sort_particles.*/
00196 void invalidate_ghosts();
00197
00198 /* TODO: This function is not used anywhere. To be removed?  */
00199 #ifdef GHOST_FLAG
00200 MDINLINE int ifParticleIsGhost(Particle *p){
00201    return p->l.ghost;
00202 }
00203 #endif
00204
00205 /*@}*/
00206
00207 #endif