vtkPStructuredGridConnectivity.h

Go to the documentation of this file.

// SPDX-FileCopyrightText: Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
// SPDX-License-Identifier: BSD-3-Clause
#ifndef vtkPStructuredGridConnectivity_h
#define vtkPStructuredGridConnectivity_h
 
// VTK include directives
#include "vtkDeprecation.h"                   // For VTK_DEPRECATED_IN_9_6_0
#include "vtkFiltersParallelGeometryModule.h" // For export macro
#include "vtkMPICommunicator.h"               // Needed for vtkMPICommunicator::Request
#include "vtkStructuredGridConnectivity.h"
 
// C++ include directives
#include <vector> // For STL vector
 
// Forward declarations
VTK_ABI_NAMESPACE_BEGIN
class vtkMultiProcessController;
class vtkMPIController;
class vtkMultiProcessStream;
// class vtkMPICommunicator::Request;
 
class VTKFILTERSPARALLELGEOMETRY_EXPORT vtkPStructuredGridConnectivity
  : public vtkStructuredGridConnectivity
{
public:
  static vtkPStructuredGridConnectivity* New();
  vtkTypeMacro(vtkPStructuredGridConnectivity, vtkStructuredGridConnectivity);
  void PrintSelf(ostream& os, vtkIndent indent) override;
 
 
  virtual void SetController(vtkMultiProcessController*);
  vtkGetObjectMacro(Controller, vtkMultiProcessController);
 
  void SetNumberOfGrids(unsigned int N) override;
 
  void RegisterGrid(int gridID, int extents[6], vtkUnsignedCharArray* nodesGhostArray,
    vtkUnsignedCharArray* cellGhostArray, vtkPointData* pointData, vtkCellData* cellData,
    vtkPoints* gridNodes) override;
 
  int GetNumberOfLocalGrids() { return static_cast<int>(this->GridIds.size()); }
 
  int GetGridRank(int gridID);
 
  bool IsGridRemote(int gridID);
 
  bool IsGridLocal(int gridID);
 
  void Initialize();
 
  void ComputeNeighbors() override;
 
  void CreateGhostLayers(int N = 1) override;
 
protected:
  vtkPStructuredGridConnectivity();
  ~vtkPStructuredGridConnectivity() override;
 
  vtkMultiProcessController* Controller;
  int Rank;
  bool Initialized;
 
  std::vector<int> GridRanks; // Corresponding rank for each grid
  std::vector<int> GridIds;   // List of GridIds, owned by this process
 
  // Data structures to store the remote ghost data of each grid for each one
  // of its neighbors. The first index is the global grid index. The second
  // is the neighbor index.
  std::vector<std::vector<vtkPoints*>> RemotePoints;
  std::vector<std::vector<vtkPointData*>> RemotePointData;
  std::vector<std::vector<vtkCellData*>> RemoteCellData;
 
  // Data structures to store the send/receive buffer sizes and corresponding
  // persistent buffers. The first index is the global grid index. The second
  // index is the neighbor index for the given grid.
  std::vector<std::vector<unsigned int>> SendBufferSizes;
  std::vector<std::vector<unsigned int>> RcvBufferSizes;
  std::vector<std::vector<unsigned char*>> SendBuffers;
  std::vector<std::vector<unsigned char*>> RcvBuffers;
 
  int TotalNumberOfSends;
  int TotalNumberOfRcvs;
  int TotalNumberOfMsgs;
 
  // Array of MPI requests
  vtkMPICommunicator::Request* MPIRequests;
 
  bool GridExtentsAreEqual(int rhs[6], int lhs[6]);
 
  bool HasPointData(int gridIdx);
 
  bool HasCellData(int gridIdx);
 
  bool HasPoints(int gridIdx);
 
  void InitializeMessageCounters();
 
  void ClearRemoteData();
 
  void ClearRawBuffers();
 
  void RegisterRemoteGrid(int gridID, int extents[6], int process);
 
  void TransferRemoteNeighborData(int gridIdx, int nei, const vtkStructuredNeighbor& Neighbor);
 
  void TransferGhostDataFromNeighbors(int gridID) override;
 
  void PackGhostData();
 
  void UnpackGhostData();
 
  void DeserializeBufferSizesForProcess(int* buffersizes, vtkIdType N, int processId);
 
  void SerializeBufferSizes(int*& sizesbuf, vtkIdType& N);
 
  void ExchangeBufferSizes();
 
  void ExchangeGhostDataInit();
 
  void PostReceives();
 
  void PostSends();
 
  void CommunicateGhostData();
 
  void ExchangeGhostDataPost();
 
  void ExchangeGhostData();
 
  void SerializeGhostPoints(int gridIdx, int ext[6], vtkMultiProcessStream& bytestream);
 
  VTK_DEPRECATED_IN_9_6_0("Use vtkMultiProcessStream::Push(vtkDataArray*) instead")
  void SerializeDataArray(vtkDataArray*, vtkMultiProcessStream&);
 
  void SerializeFieldData(int sourceExtent[6], int targetExtent[6], vtkFieldData* fieldData,
    vtkMultiProcessStream& bytestream);
 
  void SerializeGhostPointData(int gridIdx, int ext[6], vtkMultiProcessStream& bytestream);
 
  void SerializeGhostCellData(int gridIdx, int ext[6], vtkMultiProcessStream& bytestream);
 
  void DeserializeGhostPoints(int gridIdx, int nei, int ext[6], vtkMultiProcessStream& bytestream);
 
  VTK_DEPRECATED_IN_9_6_0("Use vtkMultiProcessStream::Pop(vtkDataArray*&) instead")
  void DeserializeDataArray(vtkDataArray*&, int, int, int, vtkMultiProcessStream&);
 
  VTK_DEPRECATED_IN_9_6_0("Use vtkFieldDataSerializer::Deserialize() instead.")
  void DeserializeFieldData(int ext[6], vtkFieldData* fieldData, vtkMultiProcessStream& bytestream);
 
  void DeserializeGhostPointData(
    int gridIdx, int nei, int ext[6], vtkMultiProcessStream& bytestream);
 
  void DeserializeGhostCellData(
    int gridIdx, int nei, int ext[6], vtkMultiProcessStream& bytestream);
 
  void SerializeGhostData(
    int sndGridID, int rcvGrid, int sndext[6], unsigned char*& buffer, unsigned int& size);
 
  void DeserializeGhostData(int gridID, int neiListID, int neiGridIdx, int rcvext[6],
    unsigned char* buffer, unsigned int size);
 
  void ExchangeGridExtents();
 
  void SerializeGridExtents(int*& sndbuffer, vtkIdType& N);
 
  void DeserializeGridExtentForProcess(int* rcvbuffer, vtkIdType& N, int processId);
 
private:
  vtkPStructuredGridConnectivity(const vtkPStructuredGridConnectivity&) = delete;
  void operator=(const vtkPStructuredGridConnectivity&) = delete;
};
 
//=============================================================================
//  INLINE METHODS
//=============================================================================
 
inline bool vtkPStructuredGridConnectivity::GridExtentsAreEqual(int rhs[6], int lhs[6])
{
  for (int i = 0; i < 6; ++i)
  {
    if (rhs[i] != lhs[i])
    {
      return false;
    }
  }
  return true;
}
 
//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::HasPointData(int gridIdx)
{
  // Sanity check
  assert("pre: grid index is out-of-bounds!" && (gridIdx >= 0) &&
    (gridIdx < static_cast<int>(this->NumberOfGrids)));
 
  if ((this->GridPointData[gridIdx] != nullptr) &&
    (this->GridPointData[gridIdx]->GetNumberOfArrays() > 0))
  {
    return true;
  }
  return false;
}
 
//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::HasCellData(int gridIdx)
{
  // Sanity check
  assert("pre: grid index is out-of-bounds!" && (gridIdx >= 0) &&
    (gridIdx < static_cast<int>(this->NumberOfGrids)));
 
  if ((this->GridCellData[gridIdx] != nullptr) &&
    (this->GridCellData[gridIdx]->GetNumberOfArrays() > 0))
  {
    return true;
  }
  return false;
}
 
//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::HasPoints(int gridIdx)
{
  // Sanity check
  assert("pre: grid index is out-of-bounds!" && (gridIdx >= 0) &&
    (gridIdx < static_cast<int>(this->NumberOfGrids)));
 
  if (this->GridPoints[gridIdx] != nullptr)
  {
    return true;
  }
  return false;
}
 
//------------------------------------------------------------------------------
inline void vtkPStructuredGridConnectivity::InitializeMessageCounters()
{
  this->TotalNumberOfMsgs = this->TotalNumberOfRcvs = this->TotalNumberOfSends = 0;
}
 
//------------------------------------------------------------------------------
inline void vtkPStructuredGridConnectivity::ClearRawBuffers()
{
  this->SendBufferSizes.clear();
  this->RcvBufferSizes.clear();
 
  // STEP 0: Clear send buffers
  for (unsigned int i = 0; i < this->SendBuffers.size(); ++i)
  {
    for (unsigned int j = 0; j < this->SendBuffers[i].size(); ++j)
    {
      delete[] this->SendBuffers[i][j];
    } // END for all neighbors
    this->SendBuffers[i].clear();
  } // END for all grids
  this->SendBuffers.clear();
 
  // STEP 1: Clear rcv buffers
  for (unsigned int i = 0; i < this->RcvBuffers.size(); ++i)
  {
    for (unsigned int j = 0; j < this->RcvBuffers[i].size(); ++j)
    {
      delete[] this->RcvBuffers[i][j];
    } // END for all neighbors
    this->RcvBuffers[i].clear();
  } // END for all grids
  this->RcvBuffers.clear();
}
 
//------------------------------------------------------------------------------
inline void vtkPStructuredGridConnectivity::ClearRemoteData()
{
  // STEP 0: Clear remote points
  for (unsigned int i = 0; i < this->RemotePoints.size(); ++i)
  {
    for (unsigned int j = 0; j < this->RemotePoints[i].size(); ++j)
    {
      if (this->RemotePoints[i][j] != nullptr)
      {
        this->RemotePoints[i][j]->Delete();
      }
    } // END for all j
    this->RemotePoints[i].clear();
  } // END for all i
  this->RemotePoints.clear();
 
  // STEP 1: Clear remote point data
  for (unsigned int i = 0; i < this->RemotePointData.size(); ++i)
  {
    for (unsigned int j = 0; j < this->RemotePointData[i].size(); ++j)
    {
      if (this->RemotePointData[i][j] != nullptr)
      {
        this->RemotePointData[i][j]->Delete();
      }
    } // END for all j
    this->RemotePointData[i].clear();
  } // END for all i
  this->RemotePointData.clear();
 
  // STEP 2: Clear remote cell data
  for (unsigned int i = 0; i < this->RemoteCellData.size(); ++i)
  {
    for (unsigned int j = 0; j < this->RemoteCellData[i].size(); ++j)
    {
      if (this->RemoteCellData[i][j] != nullptr)
      {
        this->RemoteCellData[i][j]->Delete();
      }
    } // END for all j
    this->RemoteCellData[i].clear();
  }
  this->RemoteCellData.clear();
}
 
//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::IsGridRemote(int gridID)
{
  return (!this->IsGridLocal(gridID));
}
 
//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::IsGridLocal(int gridID)
{
  assert("pre: Instance has not been initialized!" && this->Initialized);
  assert("pre: gridID is out-of-bounds" && (gridID >= 0) &&
    (gridID < static_cast<int>(this->NumberOfGrids)));
  assert(
    "pre: GridRanks is not properly allocated" && this->NumberOfGrids == this->GridRanks.size());
  return ((this->GridRanks[gridID] == this->Rank));
}
 
//------------------------------------------------------------------------------
inline int vtkPStructuredGridConnectivity::GetGridRank(int gridID)
{
  assert("pre: Instance has not been initialized!" && this->Initialized);
  assert("pre: gridID out-of-bounds!" &&
    (gridID >= 0 && gridID < static_cast<int>(this->NumberOfGrids)));
  return (this->GridRanks[gridID]);
}
VTK_ABI_NAMESPACE_END
#endif /* vtkPStructuredGridConnectivity_h */