communicator.h
Go to the documentation of this file.
1 //LIC// ====================================================================
2 //LIC// This file forms part of oomph-lib, the object-oriented,
3 //LIC// multi-physics finite-element library, available
4 //LIC// at http://www.oomph-lib.org.
5 //LIC//
6 //LIC// Version 1.0; svn revision $LastChangedRevision: 1097 $
7 //LIC//
8 //LIC// $LastChangedDate: 2015-12-17 11:53:17 +0000 (Thu, 17 Dec 2015) $
9 //LIC//
10 //LIC// Copyright (C) 2006-2016 Matthias Heil and Andrew Hazel
11 //LIC//
12 //LIC// This library is free software; you can redistribute it and/or
13 //LIC// modify it under the terms of the GNU Lesser General Public
14 //LIC// License as published by the Free Software Foundation; either
15 //LIC// version 2.1 of the License, or (at your option) any later version.
16 //LIC//
17 //LIC// This library is distributed in the hope that it will be useful,
18 //LIC// but WITHOUT ANY WARRANTY; without even the implied warranty of
19 //LIC// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 //LIC// Lesser General Public License for more details.
21 //LIC//
22 //LIC// You should have received a copy of the GNU Lesser General Public
23 //LIC// License along with this library; if not, write to the Free Software
24 //LIC// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
25 //LIC// 02110-1301 USA.
26 //LIC//
27 //LIC// The authors may be contacted at oomph-lib@maths.man.ac.uk.
28 //LIC//
29 //LIC//====================================================================
30 #ifndef OOMPH_COMMUNICATOR_CLASS_HEADER
31 #define OOMPH_COMMUNICATOR_CLASS_HEADER
32 
33 // Config header generated by autoconfig
34 #ifdef HAVE_CONFIG_H
35  #include <oomph-lib-config.h>
36 #endif
37 
38 // MPI headers
39 #ifdef OOMPH_HAS_MPI
40 #include "mpi.h"
41 #endif
42 
43 // Oomph-lib error handler
44 #include "oomph_definitions.h"
45 #include "oomph_utilities.h"
46 
47 namespace oomph{
48 
49 template <class T>
51 
52 //=============================================================================
53 /// \short An oomph-lib wrapper to the MPI_Comm communicator object. Just
54 /// contains an MPI_Comm object (which is a pointer) and wrappers to
55 /// the MPI_... methods.
56 //=============================================================================
58 {
59 
60  public:
61 
62 #ifdef OOMPH_HAS_MPI
63  /// Construct a communicator from a MPI_Comm object.
64  /// if the bool owns_mpi_comm is true then this communicator is responsible
65  /// for the destruction of the mpi_communicator. The mpi comm will be freed
66  /// when the destructor is called.
67  OomphCommunicator(const MPI_Comm mpi_communicator,
68  const bool& owns_mpi_comm = false)
69  : Serial_communicator(false)
70  {
71  // store a pointer to the communicator
72  Comm = mpi_communicator;
73 
74  // hold owns_mpi_comm
75  Owns_mpi_comm = owns_mpi_comm;
76  }
77 #endif
78 
79  /// Serial constructor
81 #ifdef OOMPH_HAS_MPI
82  : Owns_mpi_comm(false), Serial_communicator(true)
83 #endif
84  {}
85 
86  /// Copy constructor.
87  OomphCommunicator(const OomphCommunicator& communicator)
88 #ifdef OOMPH_HAS_MPI
89  : Owns_mpi_comm(false)
90  {
91  if (communicator.serial_communicator())
92  {
93  Serial_communicator = true;
94  }
95  else
96  {
97  Comm = communicator.mpi_comm();
98  Serial_communicator = false;
99  }
100  }
101 #else
102  {}
103 #endif
104 
105  /// Pointer (copy) constructor.
106  OomphCommunicator(const OomphCommunicator* communicator_pt)
107 #ifdef OOMPH_HAS_MPI
108  : Owns_mpi_comm(false)
109  {
110  if (communicator_pt->serial_communicator())
111  {
112  Serial_communicator = true;
113  }
114  else
115  {
116  Comm = communicator_pt->mpi_comm();
117  Serial_communicator = false;
118  }
119  }
120 #else
121  {}
122 #endif
123 
124  /// \short Destructor. If MPI and this preconditioner owns the MPI_Comm
125  /// object then MPI_Comm_free is called, otherwise nothing happens as
126  /// the destruction of the underlying MPI_Comm object is the responsibility
127  /// of another communicator.
129  {
130 #ifdef OOMPH_HAS_MPI
131  if (Owns_mpi_comm)
132  {
133  MPI_Comm_free(&Comm);
134  }
135 #endif
136  }
137 
138  /// assignment operator
139  void operator=(const OomphCommunicator& communicator)
140  {
141 #ifdef OOMPH_HAS_MPI
142  if (Owns_mpi_comm)
143  {
144  MPI_Comm_free(&Comm);
145  }
146  Owns_mpi_comm = false;
147  if (communicator.serial_communicator())
148  {
149  Serial_communicator = true;
150  }
151  else
152  {
153  Serial_communicator = false;
154  Comm = communicator.mpi_comm();
155  }
156 #endif
157  }
158 
159  /// number of processors
160  int nproc() const
161  {
162 #ifdef OOMPH_HAS_MPI
164  {
165  return 1;
166  }
167  else
168  {
169  int n_proc=1;
170  MPI_Comm_size(Comm,&n_proc);
171  return n_proc;
172  }
173 #else
174  return 1;
175 #endif
176  }
177 
178  /// my rank
179  int my_rank() const
180  {
181 #ifdef OOMPH_HAS_MPI
183  {
184  return 0;
185  }
186  else
187  {
188  int My_rank=0;
189  MPI_Comm_rank(Comm,&My_rank);
190  return My_rank;
191  }
192 #else
193  return 0;
194 #endif
195  }
196 
197  /// \short == operator - only returns true if communicators are MPI_IDENT,
198  /// i.e. if both group and context are the same
199  bool operator==(const OomphCommunicator& other_comm) const
200  {
201 #ifdef OOMPH_HAS_MPI
202  if (Serial_communicator != other_comm.serial_communicator())
203  {
204  return false;
205  }
206  else if (Serial_communicator)
207  {
208  return true;
209  }
210  else
211  {
212  int flag;
213  MPI_Comm_compare(Comm,other_comm.mpi_comm(),&flag);
214  if (flag == MPI_IDENT)
215  {
216  return true;
217  }
218  return false;
219  }
220 #else
221  return true;
222 #endif
223  }
224 
225  /// \short != operator returns !(==operator) (see ==operator for more details)
226  bool operator!=(const OomphCommunicator& other_comm) const
227  {
228  return !(*this == other_comm);
229  }
230 
231 #ifdef OOMPH_HAS_MPI
232  /// \short split the communicator: color is an integer label for the sub
233  /// group of processors. key is the rank in this sub group
234  OomphCommunicator* split(const int& color, const int& key)
235  {
236 #ifdef PARANOID
238  {
239  std::ostringstream error_message_stream;
240  error_message_stream
241  << "Attempted to split a serial communicator.";
242  throw OomphLibError(error_message_stream.str(),
243  OOMPH_CURRENT_FUNCTION,
244  OOMPH_EXCEPTION_LOCATION);
245  }
246 #endif
247 
248  // pointer for the split oomph-lib communicator
249  OomphCommunicator* split_comm_pt;
250 
251  // the pointer for the new mpi communicator
252  MPI_Comm* mpi_comm_pt = new MPI_Comm;
253 
254  // get the split communicator
255  MPI_Comm_split(Comm,color,key,mpi_comm_pt);
256 
257  // assemble the new oomph-lib ocmmunicator
258  // the new oomph-lib communicator owns the MPI_Comm at mpi_comm_pt
259  // and is responsible for its destruction
260  split_comm_pt = new OomphCommunicator(*mpi_comm_pt,true);
261 
262  // and return
263  return split_comm_pt;
264  }
265 #endif
266 
267 
268 #ifdef OOMPH_HAS_MPI
269  /// access function to the underlying MPI_Comm object
270  MPI_Comm mpi_comm() const
271  {
272 #ifdef PARANOID
274  {
275  std::ostringstream error_message_stream;
276  error_message_stream
277  << "Requested the MPI_Comm object for a serial communicator.";
278  throw OomphLibError(error_message_stream.str(),
279  OOMPH_CURRENT_FUNCTION,
280  OOMPH_EXCEPTION_LOCATION);
281  }
282 #endif
283  return Comm;
284  }
285 
286 
287  /// Access function (read only) to the serial communicator flag
288  bool serial_communicator() const
289  {
290  return Serial_communicator;
291  }
292 
293  /// broadcast function for Vector<int>
294  void broadcast(const int& source, Vector<int>& x)
295  {
296  int n;
297 
298  // Get number of entries on processor source (where the vector exists)
299  unsigned long n_long;
300  if (this->my_rank()==source)
301  {
302  n_long=x.size();
303 
304  // Convert to int
305  n=static_cast<int>(n_long);
306  }
307 
308  // Broadcast to everybody how many entries to expect
309  MPI_Bcast(&n,1,MPI_INT,source,this->mpi_comm());
310 
311  // Resize Vector everywhere else in preparation
312  if (this->my_rank()!=source)
313  {
314  x.resize(n);
315  }
316 
317  // Broadcast the Vector directly
318  MPI_Bcast(&x[0],n,MPI_INT,source,this->mpi_comm());
319  }
320 
321  /// broadcast function for Vector<double>
322  void broadcast(const int& source, Vector<double>& x)
323  {
324  int n;
325 
326  // Get number of entries on processor source (where the vector exists)
327  unsigned long n_long;
328  if (this->my_rank()==source)
329  {
330  n_long=x.size();
331 
332  // Convert to int
333  n=static_cast<int>(n_long);
334  }
335 
336  // Broadcast to everybody how many entries to expect
337  MPI_Bcast(&n,1,MPI_INT,source,this->mpi_comm());
338 
339  // Resize Vector everywhere else in preparation
340  if (this->my_rank()!=source)
341  {
342  x.resize(n);
343  }
344 
345  // Broadcast the Vector directly
346  MPI_Bcast(&x[0],n,MPI_DOUBLE,source,this->mpi_comm());
347  }
348 
349  /// broadcast function for DenseMatrix<double>
350  void broadcast(const int& source, DenseMatrix<double>& x);
351 
352 #endif
353 
354  private:
355 
356 #ifdef OOMPH_HAS_MPI
357  /// the MPI_Comm communicator
358  MPI_Comm Comm;
359 
360  /// \short boolean indiacting whether this communicator owns the underlying
361  /// MPI_Comm - if so the destructor will free
362  /// the communicator
364 
365  /// \short boolean to indicate if this communicator is for serial problems.
366  /// This is true when serial codes are compiled under MPI
368 
369 #endif
370 }; // end of class Communicator
371 } // end of oomph namespace
372 #endif
bool Serial_communicator
boolean to indicate if this communicator is for serial problems. This is true when serial codes are c...
Definition: communicator.h:367
bool Owns_mpi_comm
boolean indiacting whether this communicator owns the underlying MPI_Comm - if so the destructor will...
Definition: communicator.h:363
MPI_Comm Comm
the MPI_Comm communicator
Definition: communicator.h:102
OomphCommunicator(const MPI_Comm mpi_communicator, const bool &owns_mpi_comm=false)
Definition: communicator.h:67
OomphCommunicator()
Serial constructor.
Definition: communicator.h:80
void split(const DoubleVector &in_vector, Vector< DoubleVector * > &out_vector_pt)
Split a DoubleVector into the out DoubleVectors. Let vec_A be the in Vector, and let vec_B and vec_C ...
Class for dense matrices, storing all the values of the matrix as a pointer to a pointer with assorte...
Definition: communicator.h:50
An oomph-lib wrapper to the MPI_Comm communicator object. Just contains an MPI_Comm object (which is ...
Definition: communicator.h:57