linear_algebra_distribution.h
Go to the documentation of this file.
1 //LIC// ====================================================================
2 //LIC// This file forms part of oomph-lib, the object-oriented,
3 //LIC// multi-physics finite-element library, available
4 //LIC// at http://www.oomph-lib.org.
5 //LIC//
6 //LIC// Version 1.0; svn revision $LastChangedRevision: 1282 $
7 //LIC//
8 //LIC// $LastChangedDate: 2017-01-16 08:27:53 +0000 (Mon, 16 Jan 2017) $
9 //LIC//
10 //LIC// Copyright (C) 2006-2016 Matthias Heil and Andrew Hazel
11 //LIC//
12 //LIC// This library is free software; you can redistribute it and/or
13 //LIC// modify it under the terms of the GNU Lesser General Public
14 //LIC// License as published by the Free Software Foundation; either
15 //LIC// version 2.1 of the License, or (at your option) any later version.
16 //LIC//
17 //LIC// This library is distributed in the hope that it will be useful,
18 //LIC// but WITHOUT ANY WARRANTY; without even the implied warranty of
19 //LIC// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 //LIC// Lesser General Public License for more details.
21 //LIC//
22 //LIC// You should have received a copy of the GNU Lesser General Public
23 //LIC// License along with this library; if not, write to the Free Software
24 //LIC// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
25 //LIC// 02110-1301 USA.
26 //LIC//
27 //LIC// The authors may be contacted at oomph-lib@maths.man.ac.uk.
28 //LIC//
29 //LIC//====================================================================
30 #ifndef OOMPH_LINEAR_ALGEBRA_DISTRIBUTION_CLASS_HEADER
31 #define OOMPH_LINEAR_ALGEBRA_DISTRIBUTION_CLASS_HEADER
32 
33 // Config header generated by autoconfig
34 #ifdef HAVE_CONFIG_H
35  #include <oomph-lib-config.h>
36 #endif
37 
38 // MPI headers
39 #ifdef OOMPH_HAS_MPI
40 #include "mpi.h"
41 #endif
42 
43 #include <ostream>
44 
45 // General headers
46 #include "Vector.h"
47 #include "communicator.h"
48 #include "oomph_utilities.h"
49 
50 
51 
52 namespace oomph{
53 
54 //=============================================================================
55 /// \short Describes the distribution of a distributable linear algebra type
56 /// object. Typically this is a container (such as a DoubleVector) or an
57 /// operator (e.g Preconditioner or LinearSolver).
58 /// This object is used in both serial and parallel implementations. In the
59 /// serial context (no MPI) this just contains an integer indicating
60 /// the number of rows.
61 /// In parallel either each processor holds a subset of the set of global rows.
62 /// (each processor contains only a single continuous block of rows -
63 /// parametised with variables denoting the first row and the number of local
64 /// rows) or, all rows are be duplicated across all processors.
65 /// In parallel this object also contains an OomphCommunicator object which
66 /// primarily contains the MPI_Comm communicator associated with this object.
67 //=============================================================================
69 {
70 
71  public :
72 
73  /// \short Default Constructor - creates a Distribution that has not been
74  /// setup
76 
77  /// \short Constructor. Takes the first_row, nrow_local (both for this
78  /// processor) and nrow as arguments. If nrow is not provided
79  /// or equal to 0 then it will be computed automatically
81  const unsigned &first_row_,
82  const unsigned &n_row_local,
83  const unsigned &n_row = 0)
84  : Comm_pt(0)
85  {
86  this->build(&comm,first_row_,n_row_local,n_row);
87  };
88 
89  /// \short Constructor. Takes the number of global rows and uniformly
90  /// distributes them over the processors if distributed = true (default),
91  /// if distributed = false then every row is duplicated on every processor
93  const unsigned &n_row,
94  const bool &distributed_ = true)
95  : Comm_pt(0)
96  {
97  this->build(&comm,n_row,distributed_);
98  };
99 
100  /// \short Constructor. Takes the first_row, nrow_local (both for this
101  /// processor) and nrow as arguments. If nrow is not provided
102  /// or equal to 0 then it will be computed automatically
104  const unsigned &first_row_,
105  const unsigned &n_row_local,
106  const unsigned &n_row = 0)
107  : Comm_pt(0)
108  {
109  this->build(comm_pt,first_row_,n_row_local,n_row);
110  };
111 
112  /// \short Constructor. Takes the number of global rows and uniformly
113  /// distributes them over the processors if distributed = true (default),
114  /// if distributed = false then every row is duplicated on every processor
116  const unsigned &n_row,
117  const bool &distributed_ = true)
118  : Comm_pt(0)
119  {
120  this->build(comm_pt,n_row,distributed_);
121  };
122 
123  /// \short Copy Constructor.
125  : Comm_pt(0)
126  {
127  this->build(old_dist);
128  }
129 
130  /// \short pointer based copy constructor
132  : Comm_pt(0)
133  {
134  this->build(old_dist_pt);
135  }
136 
137  /// \short Destructor
139  {
140  delete Comm_pt;
141  }
142 
143  /// Assignment Operator
144  void operator=(const LinearAlgebraDistribution& old_dist)
145  {
146  this->build(old_dist);
147  }
148 
149  /// \short Sets the distribution. Takes first_row, nrow_local and
150  /// nrow as arguments. If nrow is not provided or equal to
151  /// 0 then it is computed automatically
152  void build(const OomphCommunicator* const comm_pt,
153  const unsigned& first_row,
154  const unsigned& nrow_local,
155  const unsigned& nrow = 0);
156 
157  /// \short Build the LinearAlgebraDistribution. if distributed = true
158  /// (default) then uniformly distribute nrow over all processors where
159  /// processors 0 holds approximately the first nrow/n_proc, processor
160  /// 1 holds the next nrow/n_proc and so on... or if distributed = false
161  /// then every row is held on every processor
162  void build(const OomphCommunicator* const comm_pt,
163  const unsigned& nrow,
164  const bool& distributed = true);
165 
166  /// \short Copy the argument distribution.
167  /// Also a helper method for the =assignment operator and copy constructor
168  void build(const LinearAlgebraDistribution& new_dist);
169 
170  /// \short Copy the argument distribution.
171  /// Also a helper method for the =assignment operator and copy constructor
172  void build(const LinearAlgebraDistribution* new_dist_pt)
173  {
174  this->build(*new_dist_pt);
175  }
176 
177  /// \short clears the distribution
178  void clear()
179  {
180  // delete the communicator
181  delete Comm_pt;
182  Comm_pt = 0;
183 
184  // delete first_row and nrow_local
185  First_row.clear();
186  Nrow_local.clear();
187 
188  // zero Nrow
189  Nrow = 0;
190  }
191 
192  /// \short access function to the number of global rows.
193  unsigned nrow() const
194  {
195  return Nrow;
196  }
197 
198  /// \short access function for the num of local rows on this processor. If
199  /// no MPI then Nrow is returned.
200  unsigned nrow_local() const
201  {
202  // return the nrow_local
203 #ifdef OOMPH_HAS_MPI
204  if (Distributed)
205  {
206 
207 #ifdef PARANOID
208  if (Comm_pt == 0)
209  {
210  throw OomphLibError("LinearAlgebraDistribution has not been built : Comm_pt == 0.",
211  "LinearAlgebraDistribution::nrow_local()",
212  OOMPH_EXCEPTION_LOCATION);
213  }
214 #endif
215 
216  return Nrow_local[Comm_pt->my_rank()];
217  }
218  else
219  {
220  return Nrow;
221  }
222 #else
223  return Nrow;
224 #endif
225  }
226 
227  /// \short access function for the num of local rows on this processor. If
228  /// no MPI the nrow is returned
229  unsigned nrow_local(const unsigned& p) const
230  {
231 #ifdef PARANOID
232  if (Comm_pt == 0)
233  {
234  throw OomphLibError(
235  "LinearAlgebraDistribution has not been built : Comm_pt == 0.",
236  OOMPH_CURRENT_FUNCTION,
237  OOMPH_EXCEPTION_LOCATION);
238  }
239  if (p >= unsigned(Comm_pt->nproc()))
240  {
241  std::ostringstream error_message;
242  error_message
243  << "Requested nrow_local(" << p << "), but this distribution is defined "
244  << "on " << Comm_pt->nproc() << "processors.";
245  throw OomphLibError(error_message.str(),
246  OOMPH_CURRENT_FUNCTION,
247  OOMPH_EXCEPTION_LOCATION);
248  }
249 #endif
250 
251  // return the nrow_local
252 #ifdef OOMPH_HAS_MPI
253  if (Distributed)
254  {
255  return Nrow_local[p];
256  }
257  else
258  {
259  return Nrow;
260  }
261 #else
262  return Nrow;
263 #endif
264  }
265 
266  /// \short access function for the first row on this processor. If not
267  /// distributed then this is just zero.
268  unsigned first_row() const
269  {
270  // return the first row
271 #ifdef OOMPH_HAS_MPI
272  if (Distributed)
273  {
274 
275 #ifdef PARANOID
276  if (Comm_pt == 0)
277  {
278  throw OomphLibError
279  ("LinearAlgebraDistribution has not been built : Comm_pt == 0.",
280  OOMPH_CURRENT_FUNCTION,
281  OOMPH_EXCEPTION_LOCATION);
282  }
283 #endif
284 
285  return First_row[Comm_pt->my_rank()];
286  }
287  else
288  {
289  return 0;
290  }
291 #else
292  return 0;
293 #endif
294  }
295 
296  /// \short access function for the first row on the p-th processor
297  unsigned first_row(const unsigned& p) const
298  {
299 #ifdef PARANOID
300  if (Comm_pt == 0)
301  {
302  throw OomphLibError
303  ("LinearAlgebraDistribution has not been built : Comm_pt == 0.",
304  OOMPH_CURRENT_FUNCTION,
305  OOMPH_EXCEPTION_LOCATION);
306  }
307  if (p >= unsigned(Comm_pt->nproc()))
308  {
309  std::ostringstream error_message;
310  error_message
311  << "Requested first_row(" << p << "), but this distribution is defined "
312  << "on " << Comm_pt->nproc() << "processors.";
313  throw OomphLibError(error_message.str(),
314  OOMPH_CURRENT_FUNCTION,
315  OOMPH_EXCEPTION_LOCATION);
316  }
317 
318 #endif
319 
320  // return the first row
321 #ifdef OOMPH_HAS_MPI
322  if (Distributed)
323  {
324  return First_row[p];
325  }
326  else
327  {
328  return 0;
329  }
330 #else
331  return 0;
332 #endif
333  }
334 
335  /// \short access function to the distributed - indicates whether the
336  /// distribution is serial or distributed
337  bool distributed() const
338  {
339  return Distributed;
340  }
341 
342  /// const access to the communicator pointer
344  {
345  return Comm_pt;
346  }
347 
348  /// if the communicator_pt is null then the distribution is not setup then
349  /// false is returned, otherwise return true
350  bool built() const
351  {
352  if (Comm_pt == 0)
353  {
354  return false;
355  }
356  return true;
357  }
358 
359  /// \short == Operator
360  bool operator==(const LinearAlgebraDistribution& other_dist) const;
361 
362  /// \short != operator
363  bool operator!=(const LinearAlgebraDistribution& other_dist) const
364  {
365  return !(*this == other_dist);
366  }
367 
368  /// \short << operator
369  friend std::ostream& operator<<(std::ostream& stream,
371 
372  /// \short return the local index corresponding to the global index
373  unsigned global_to_local_row_map(const unsigned& global_i) const
374  {
375 #ifdef PARANOID
376  if (global_i >= Nrow)
377  {
378  throw OomphLibError
379  ("Requested global row outside the number of global rows",
380  OOMPH_CURRENT_FUNCTION,
381  OOMPH_EXCEPTION_LOCATION);
382  }
383 #endif
384  int local_i = static_cast<int>(global_i);
385  int p = 0;
386  while ((int)(local_i - (int)nrow_local(p)) >= 0)
387  {
388  local_i -= (int)nrow_local(p);
389  p++;
390  }
391  return (unsigned)local_i;
392  }
393 
394  /// \short return the processor rank of the global row number i
395  unsigned rank_of_global_row(const unsigned i) const
396  {
397  unsigned p = 0;
398  while (i < first_row(p) || i >= first_row(p)+nrow_local(p))
399  {
400  p++;
401  }
402  return p;
403  }
404 
405  /// \short return the nrow_local Vector
407  {
408  return Nrow_local;
409  }
410 
411  /// \short return the first_row Vector
413  {
414  return First_row;
415  }
416 
417  private:
418 
419  /// the number of global rows
420  unsigned Nrow;
421 
422  /// the number of local rows on the processor
424 
425  /// the first row on this processor
427 
428  /// flag to indicate whether this distribution describes an object that is
429  /// distributed over the processors of Comm_pt (true) or duplicated over the
430  /// processors of Comm_pt (false)
432 
433  /// the pointer to the MPI communicator object in this distribution
435 }; //end of LinearAlgebraDistribution
436 
437 
438 //=============================================================================
439 /// \short Base class for any linear algebra object that is distributable.
440 /// Just contains storage for the LinearAlgebraDistribution object and
441 /// access functions
442 //=============================================================================
444  {
445 
446  public :
447 
448  /// Default constructor - create a distribution
450  {
452  }
453 
454  /// Broken copy constructor
457  {
458  BrokenCopy::broken_copy("DistributableLinearAlgebraObject");
459  }
460 
461  /// Broken assignment operator
463  {
464  BrokenCopy::broken_assign("DistributableLinearAlgebraObject");
465  }
466 
467  /// Destructor
469  {
470  delete Distribution_pt;
471  }
472 
473  /// access to the LinearAlgebraDistribution
475  {
476  return Distribution_pt;
477  }
478 
479  /// \short access function to the number of global rows.
480  unsigned nrow() const
481  {
482  return Distribution_pt->nrow();
483  }
484 
485  /// \short access function for the num of local rows on this processor.
486  unsigned nrow_local() const
487  {
488  return Distribution_pt->nrow_local();
489  }
490 
491  /// \short access function for the num of local rows on this processor.
492  unsigned nrow_local(const unsigned& p) const
493  {
494  return Distribution_pt->nrow_local(p);
495  }
496 
497  /// \short access function for the first row on this processor
498  unsigned first_row() const
499  {
500  return Distribution_pt->first_row();
501  }
502 
503  /// \short access function for the first row on this processor
504  unsigned first_row(const unsigned& p) const
505  {
506  return Distribution_pt->first_row(p);
507  }
508 
509  /// distribution is serial or distributed
510  bool distributed() const
511  {
512  return Distribution_pt->distributed();
513  }
514 
515  /// if the communicator_pt is null then the distribution is not setup then
516  /// false is returned, otherwise return true
517  bool distribution_built() const
518  {
519  return Distribution_pt->built();
520  }
521 
522  /// \short setup the distribution of this distributable linear algebra
523  /// object
525  {
526  Distribution_pt->build(dist_pt);
527  }
528 
529  /// \short setup the distribution of this distributable linear algebra
530  /// object
532  {
533  Distribution_pt->build(dist);
534  }
535 
536  protected:
537 
538  /// \short clear the distribution of this distributable linear algebra
539  /// object
541  {
543  }
544 
545  private:
546 
547  /// the LinearAlgebraDistribution object
549  }; // end of DistributableLinearAlgebraObject
550 
551 //=============================================================================
552 /// Namespace for helper functions for LinearAlgebraDistributions
553 //=============================================================================
554  namespace LinearAlgebraDistributionHelpers
555  {
556  /// \short Takes a vector of LinearAlgebraDistribution objects and
557  /// concatenates them such that the nrow_local of the out_distribution
558  /// is the sum of the nrow_local of all the in_distributions and the number
559  /// of global rows of the out_distribution is the sum of the number of global
560  /// rows of all the in_distributions.
561  /// This results in a permutation of the rows in the out_distribution.
562  /// Think of this in terms of DoubleVectors, if we have DoubleVectors with
563  /// distributions A and B, distributed across two processors (p0 and p1),
564  /// A: [a0] (on p0) B: [b0] (on p0)
565  /// [a1] (on p1) [b1] (on P1),
566  ///
567  /// then the out_distribution is
568  /// [a0 (on p0)
569  /// b0] (on p0)
570  /// [a1 (on p1)
571  /// b1] (on p1),
572  ///
573  /// as opposed to
574  /// [a0 (on p0)
575  /// a1] (on p0)
576  /// [b0 (on p1)
577  /// b1] (on p1).
578  ///
579  /// Note (1): The out_distribution may not be uniformly distributed even
580  /// if the in_distributions are uniform distributions.
581  /// Try this out with two distributions of global rows 3 and 5, uniformly
582  /// distributed across two processors. Compare this against a distribution
583  /// of global row 8 distributed across two processors.
584  ///
585  /// Note (2): There is no equivalent function which takes a Vector of
586  /// LinearAlgebraDistribution objects (as opposed to pointers), there should
587  /// not be one since we do not want to invoke the assignment operator when
588  /// creating the Vector of LinearAlgebraDistribution objects.
589  void concatenate(const Vector<LinearAlgebraDistribution*> &in_distribution_pt,
590  LinearAlgebraDistribution &out_distribution);
591  }
592 } // end of oomph namespace
593 #endif
LinearAlgebraDistribution(const OomphCommunicator *const comm_pt, const unsigned &n_row, const bool &distributed_=true)
Constructor. Takes the number of global rows and uniformly distributes them over the processors if di...
unsigned nrow_local() const
access function for the num of local rows on this processor.
void broken_copy(const std::string &class_name)
Issue error message and terminate execution.
unsigned first_row() const
access function for the first row on this processor. If not distributed then this is just zero...
friend std::ostream & operator<<(std::ostream &stream, LinearAlgebraDistribution &dist)
<< operator
void build(const LinearAlgebraDistribution *new_dist_pt)
Copy the argument distribution. Also a helper method for the =assignment operator and copy constructo...
unsigned first_row() const
access function for the first row on this processor
LinearAlgebraDistribution(const OomphCommunicator &comm, const unsigned &n_row, const bool &distributed_=true)
Constructor. Takes the number of global rows and uniformly distributes them over the processors if di...
cstr elem_len * i
Definition: cfortran.h:607
void operator=(const DistributableLinearAlgebraObject &)
Broken assignment operator.
Vector< unsigned > First_row
the first row on this processor
unsigned nrow() const
access function to the number of global rows.
unsigned nrow_local() const
access function for the num of local rows on this processor. If no MPI then Nrow is returned...
LinearAlgebraDistribution * Distribution_pt
the LinearAlgebraDistribution object
LinearAlgebraDistribution(const LinearAlgebraDistribution *old_dist_pt)
pointer based copy constructor
void build(const OomphCommunicator *const comm_pt, const unsigned &first_row, const unsigned &nrow_local, const unsigned &nrow=0)
Sets the distribution. Takes first_row, nrow_local and nrow as arguments. If nrow is not provided or ...
OomphCommunicator * Comm_pt
the pointer to the MPI communicator object in this distribution
bool distributed() const
distribution is serial or distributed
Vector< unsigned > Nrow_local
the number of local rows on the processor
Vector< unsigned > first_row_vector() const
return the first_row Vector
DistributableLinearAlgebraObject()
Default constructor - create a distribution.
bool distributed() const
access function to the distributed - indicates whether the distribution is serial or distributed ...
void concatenate(const Vector< LinearAlgebraDistribution * > &in_distribution_pt, LinearAlgebraDistribution &out_distribution)
Takes a vector of LinearAlgebraDistribution objects and concatenates them such that the nrow_local of...
Vector< unsigned > nrow_local_vector() const
return the nrow_local Vector
LinearAlgebraDistribution * distribution_pt() const
access to the LinearAlgebraDistribution
unsigned Nrow
the number of global rows
LinearAlgebraDistribution(const LinearAlgebraDistribution &old_dist)
Copy Constructor.
unsigned rank_of_global_row(const unsigned i) const
return the processor rank of the global row number i
Describes the distribution of a distributable linear algebra type object. Typically this is a contain...
Base class for any linear algebra object that is distributable. Just contains storage for the LinearA...
unsigned global_to_local_row_map(const unsigned &global_i) const
return the local index corresponding to the global index
void operator=(const LinearAlgebraDistribution &old_dist)
Assignment Operator.
unsigned first_row(const unsigned &p) const
access function for the first row on the p-th processor
unsigned nrow_local(const unsigned &p) const
access function for the num of local rows on this processor. If no MPI the nrow is returned ...
unsigned nrow() const
access function to the number of global rows.
void clear_distribution()
clear the distribution of this distributable linear algebra object
unsigned first_row(const unsigned &p) const
access function for the first row on this processor
void broken_assign(const std::string &class_name)
Issue error message and terminate execution.
LinearAlgebraDistribution(const OomphCommunicator *const comm_pt, const unsigned &first_row_, const unsigned &n_row_local, const unsigned &n_row=0)
Constructor. Takes the first_row, nrow_local (both for this processor) and nrow as arguments...
void build_distribution(const LinearAlgebraDistribution &dist)
setup the distribution of this distributable linear algebra object
void build_distribution(const LinearAlgebraDistribution *const dist_pt)
setup the distribution of this distributable linear algebra object
OomphCommunicator * communicator_pt() const
const access to the communicator pointer
void clear()
clears the distribution
bool operator!=(const LinearAlgebraDistribution &other_dist) const
!= operator
unsigned nrow_local(const unsigned &p) const
access function for the num of local rows on this processor.
LinearAlgebraDistribution(const OomphCommunicator &comm, const unsigned &first_row_, const unsigned &n_row_local, const unsigned &n_row=0)
Constructor. Takes the first_row, nrow_local (both for this processor) and nrow as arguments...
LinearAlgebraDistribution()
Default Constructor - creates a Distribution that has not been setup.
bool operator==(const LinearAlgebraDistribution &other_dist) const
== Operator
An oomph-lib wrapper to the MPI_Comm communicator object. Just contains an MPI_Comm object (which is ...
Definition: communicator.h:57