vtkPStructuredGridConnectivity.h

Go to the documentation of this file.

/*=========================================================================

 Program:   Visualization Toolkit
 Module:    vtkPStructuredGridConnectivity.h

 Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
 All rights reserved.
 See Copyright.txt or http://www.kitware.com/Copyright.htm for details.

 This software is distributed WITHOUT ANY WARRANTY; without even
 the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 PURPOSE.  See the above copyright notice for more information.

 =========================================================================*/
#ifndef VTKPSTRUCTUREDGRIDCONNECTIVITY_H_
#define VTKPSTRUCTUREDGRIDCONNECTIVITY_H_

// VTK include directives
#include "vtkFiltersParallelGeometryModule.h" // For export macro
#include "vtkStructuredGridConnectivity.h"
#include "vtkMPICommunicator.h" // Needed for vtkMPICommunicator::Request

// C++ include directives
#include <vector> // For STL vector

// Forward declarations
class vtkMultiProcessController;
class vtkMPIController;
class vtkMultiProcessStream;
//class vtkMPICommunicator::Request;

class VTKFILTERSPARALLELGEOMETRY_EXPORT vtkPStructuredGridConnectivity :
  public vtkStructuredGridConnectivity
{
public:
  static vtkPStructuredGridConnectivity* New();
  vtkTypeMacro(vtkPStructuredGridConnectivity,vtkStructuredGridConnectivity);
  void PrintSelf(ostream& os, vtkIndent indent );


  vtkSetMacro( Controller,vtkMultiProcessController* );
  vtkGetMacro( Controller,vtkMultiProcessController* );

  void SetNumberOfGrids( const unsigned int N );


  void RegisterGrid( const int gridID, int extents[6],
      vtkUnsignedCharArray* nodesGhostArray,
      vtkUnsignedCharArray* cellGhostArray,
      vtkPointData* pointData,
      vtkCellData* cellData,
      vtkPoints* gridNodes );


  int GetNumberOfLocalGrids()
    { return static_cast<int>(this->GridIds.size()); };

  int GetGridRank( const int gridID );

  bool IsGridRemote( const int gridID );

  bool IsGridLocal( const int gridID );

  void Initialize();

  void ComputeNeighbors();

  virtual void CreateGhostLayers( const int N=1 );

protected:
  vtkPStructuredGridConnectivity();
  ~vtkPStructuredGridConnectivity();

  vtkMultiProcessController *Controller;
  int Rank;
  bool Initialized;

  // BTX
  std::vector< int > GridRanks; // Corresponding rank for each grid
  std::vector< int > GridIds;   // List of GridIds, owned by this process

  // Data structures to store the remote ghost data of each grid for each one
  // of its neighbors. The first index is the global grid index. The second
  // is the neighbor index.
  std::vector< std::vector< vtkPoints* > >    RemotePoints;
  std::vector< std::vector< vtkPointData* > > RemotePointData;
  std::vector< std::vector< vtkCellData* > >  RemoteCellData;

  // Data structures to store the send/receive buffer sizes and corresponding
  // persistent buffers. The first index is the global grid index. The second
  // index is the neighbor index for the given grid.
  std::vector< std::vector< unsigned int > > SendBufferSizes;
  std::vector< std::vector< unsigned int > > RcvBufferSizes;
  std::vector< std::vector< unsigned char* > > SendBuffers;
  std::vector< std::vector< unsigned char* > > RcvBuffers;

  int TotalNumberOfSends;
  int TotalNumberOfRcvs;
  int TotalNumberOfMsgs;

  // Array of MPI requests
  vtkMPICommunicator::Request *MPIRequests;
  // ETX

  bool GridExtentsAreEqual( int rhs[6], int lhs[6] );

  bool HasPointData(const int gridIdx);

  bool HasCellData(const int gridIdx);

  bool HasPoints(const int gridIdx);

  void InitializeMessageCounters();

  void ClearRemoteData();

  void ClearRawBuffers();

  void RegisterRemoteGrid( const int gridID, int extents[6], int process );


  void TransferRemoteNeighborData(
      const int gridIdx,const int nei,const vtkStructuredNeighbor& Neighbor );

  virtual void TransferGhostDataFromNeighbors(const int gridID);

  void PackGhostData();

  void UnpackGhostData();


  void DeserializeBufferSizesForProcess(
      int *buffersizes, vtkIdType N, const int processId );

  void SerializeBufferSizes(int *&sizesbuf, vtkIdType &N);

  void ExchangeBufferSizes();

  void ExchangeGhostDataInit();

  void PostReceives();

  void PostSends();

  void CommunicateGhostData();

  void ExchangeGhostDataPost();

  void ExchangeGhostData();


  void SerializeGhostPoints(
      const int gridIdx, int ext[6], vtkMultiProcessStream& bytestream );


  void SerializeDataArray(
      vtkDataArray *dataArray, vtkMultiProcessStream& bytestream );


  void SerializeFieldData(
      int sourceExtent[6], int targetExtent[6], vtkFieldData *fieldData,
      vtkMultiProcessStream& bytestream );


  void SerializeGhostPointData(
      const int gridIdx, int ext[6], vtkMultiProcessStream& bytestream );


  void SerializeGhostCellData(
      const int gridIdx, int ext[6], vtkMultiProcessStream& bytestream );


  void DeserializeGhostPoints(
      const int gridIdx, const int nei,
      int ext[6], vtkMultiProcessStream& bytestream );


  void DeserializeDataArray(
      vtkDataArray *&dataArray,const int dataType,
      const int numberOfTuples, const int numberOfComponents,
      vtkMultiProcessStream& bytestream );


  void DeserializeFieldData(
      int ext[6], vtkFieldData *fieldData,
      vtkMultiProcessStream &bytestream );


  void DeserializeGhostPointData(
      const int gridIdx, const int nei,
      int ext[6], vtkMultiProcessStream& bytestream );


  void DeserializeGhostCellData(
      const int gridIdx, const int nei,
      int ext[6], vtkMultiProcessStream& bytestream );


  void SerializeGhostData(
      const int sndGridID, const int rcvGrid, int sndext[6],
      unsigned char*& buffer, unsigned int &size);


  void DeserializeGhostData(
      const int gridID, const int neiListID,
      const int neiGridIdx, int rcvext[6],
      unsigned char *buffer, unsigned int size );

  void ExchangeGridExtents();

  void SerializeGridExtents( int *&sndbuffer, vtkIdType &N );


  void DeserializeGridExtentForProcess(
      int *rcvbuffer, vtkIdType &N, const int processId );

private:
  vtkPStructuredGridConnectivity(const vtkPStructuredGridConnectivity& ); // Not implemented
  void operator=(const vtkPStructuredGridConnectivity& ); // Not implemented
};

//=============================================================================
//  INLINE METHODS
//=============================================================================


inline bool vtkPStructuredGridConnectivity::GridExtentsAreEqual(
    int rhs[6], int lhs[6] )
{
  for( int i=0; i < 6; ++i )
    {
    if( rhs[i] != lhs[i] )
      {
      return false;
      }
    }
  return true;
}

//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::HasPointData(const int gridIdx)
{
  // Sanity check
  assert("pre: grid index is out-of-bounds!" &&
         (gridIdx >= 0) && (gridIdx < static_cast<int>(this->NumberOfGrids)));

  if( (this->GridPointData[gridIdx] != NULL) &&
      (this->GridPointData[gridIdx]->GetNumberOfArrays() > 0) )
    {
    return true;
    }
  return false;
}

//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::HasCellData(const int gridIdx)
{
  // Sanity check
  assert("pre: grid index is out-of-bounds!" &&
         (gridIdx >= 0) && (gridIdx < static_cast<int>(this->NumberOfGrids)));

  if( (this->GridCellData[gridIdx] != NULL) &&
      (this->GridCellData[gridIdx]->GetNumberOfArrays( ) > 0) )
    {
    return true;
    }
  return false;
}

//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::HasPoints(const int gridIdx)
{
  // Sanity check
  assert("pre: grid index is out-of-bounds!" &&
         (gridIdx >= 0) && (gridIdx < static_cast<int>(this->NumberOfGrids)));

  if( this->GridPoints[gridIdx] != NULL )
    {
    return true;
    }
  return false;
}

//------------------------------------------------------------------------------
inline void vtkPStructuredGridConnectivity::InitializeMessageCounters()
{
  this->TotalNumberOfMsgs=this->TotalNumberOfRcvs=this->TotalNumberOfSends=0;
}

//------------------------------------------------------------------------------
inline void vtkPStructuredGridConnectivity::ClearRawBuffers()
{
  this->SendBufferSizes.clear();
  this->RcvBufferSizes.clear();

  // STEP 0: Clear send buffers
  for( unsigned int i=0; i < this->SendBuffers.size(); ++i )
    {
    for( unsigned int j=0; j < this->SendBuffers[i].size(); ++j )
      {
      delete [] this->SendBuffers[i][j];
      } // END for all neighbors
    this->SendBuffers[i].clear();
    } // END for all grids
  this->SendBuffers.clear();

  // STEP 1: Clear rcv buffers
  for( unsigned int i=0; i < this->RcvBuffers.size(); ++i )
    {
    for( unsigned int j=0; j < this->RcvBuffers[i].size(); ++j )
      {
      delete [] this->RcvBuffers[i][j];
      } // END for all neighbors
    this->RcvBuffers[i].clear();
    } // END for all grids
  this->RcvBuffers.clear();
}

//------------------------------------------------------------------------------
inline void vtkPStructuredGridConnectivity::ClearRemoteData()
{
  // STEP 0: Clear remote points
  for( unsigned int i=0; i < this->RemotePoints.size(); ++i )
    {
    for( unsigned int j=0; j < this->RemotePoints[i].size(); ++j )
      {
      if( this->RemotePoints[ i ][ j ] != NULL )
        {
        this->RemotePoints[ i ][ j ]->Delete();
        }
      } // END for all j
    this->RemotePoints[ i ].clear();
    } // END for all i
  this->RemotePoints.clear();

  // STEP 1: Clear remote point data
  for( unsigned int i=0; i < this->RemotePointData.size(); ++i )
    {
    for( unsigned int j=0; j < this->RemotePointData[i].size(); ++j )
      {
      if( this->RemotePointData[ i ][ j ] != NULL )
        {
        this->RemotePointData[ i ][ j ]->Delete();
        }
      } // END for all j
    this->RemotePointData[ i ].clear();
    } // END for all i
  this->RemotePointData.clear();

  // STEP 2: Clear remote cell data
  for( unsigned int i=0; i < this->RemoteCellData.size(); ++i )
    {
    for( unsigned int j=0; j < this->RemoteCellData[i].size(); ++j )
      {
      if( this->RemoteCellData[ i ][ j ] != NULL )
        {
        this->RemoteCellData[ i ][ j ]->Delete();
        }
      } // END for all j
    this->RemoteCellData[ i ].clear();
    }
  this->RemoteCellData.clear();
}

//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::IsGridRemote(const int gridID )
{
  return( !this->IsGridLocal(gridID) );
}

//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::IsGridLocal(const int gridID)
{
  assert( "pre: Instance has not been intialized!" && this->Initialized );
  assert( "pre: gridID is out-of-bounds" &&
          (gridID >= 0) && (gridID < static_cast<int>(this->NumberOfGrids) ) );
  assert( "pre: GridRanks is not properly allocated" &&
          this->NumberOfGrids == this->GridRanks.size() );
  return( (this->GridRanks[ gridID ] == this->Rank) );
}

//------------------------------------------------------------------------------
inline int vtkPStructuredGridConnectivity::GetGridRank( const int gridID )
{
  assert( "pre: Instance has not been intialized!" && this->Initialized );
  assert( "pre: gridID out-of-bounds!" &&
          (gridID >= 0  && gridID < static_cast<int>(this->NumberOfGrids)));
  return( this->GridRanks[ gridID ] );
}
#endif /* VTKPSTRUCTUREDGRIDCONNECTIVITY_H_ */