vtkDIYGhostUtilities.h

Go to the documentation of this file.

// SPDX-FileCopyrightText: Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
// SPDX-License-Identifier: BSD-3-Clause
#ifndef vtkDIYGhostUtilities_h
#define vtkDIYGhostUtilities_h
 
#include "vtkBoundingBox.h"         // For ComputeLinkMap
#include "vtkDIYExplicitAssigner.h" // For DIY assigner
#include "vtkDIYUtilities.h"        // For Block
#include "vtkNew.h"                 // For vtkNew
#include "vtkObject.h"
#include "vtkParallelDIYModule.h" // For export macros
#include "vtkQuaternion.h"        // For vtkImageData
#include "vtkSmartPointer.h"      // For vtkSmartPointer
 
#include <array>         // For VectorType and ExtentType
#include <map>           // For BlockMapType
#include <set>           // For Link
#include <unordered_map> // For BlockMapType
#include <vector>        // For LinkMap
 
// clang-format off
#include "vtk_diy2.h" // Third party include
#include VTK_DIY2(diy/assigner.hpp)
#include VTK_DIY2(diy/master.hpp)
#include VTK_DIY2(diy/partners/all-reduce.hpp)
// clang-format on
 
VTK_ABI_NAMESPACE_BEGIN
class vtkAbstractPointLocator;
class vtkAlgorithm;
class vtkCellArray;
class vtkDataArray;
class vtkDataSet;
class vtkFieldData;
class vtkIdList;
class vtkIdTypeArray;
class vtkImageData;
class vtkMatrix3x3;
class vtkMultiProcessController;
class vtkPoints;
class vtkPointSet;
class vtkPolyData;
class vtkRectilinearGrid;
class vtkStructuredGrid;
class vtkUnsignedCharArray;
class vtkUnstructuredGrid;
 
class VTKPARALLELDIY_EXPORT vtkDIYGhostUtilities : public vtkObject
{
public:
  vtkTypeMacro(vtkDIYGhostUtilities, vtkObject);
  void PrintSelf(ostream& os, vtkIndent indent) override;


  using VectorType = std::array<double, 3>;
  using QuaternionType = vtkQuaternion<double>;
  using ExtentType = std::array<int, 6>;
  template <class T>
  using BlockMapType = std::map<int, T>;
  using Links = std::set<int>;
  using LinkMap = std::vector<Links>;

  template <class DataSetT>
  struct DataSetTypeToBlockTypeConverter;
 
protected:
  struct DataSetBlockStructure
  {
    vtkSmartPointer<vtkFieldData> GhostCellData = nullptr;
    vtkSmartPointer<vtkFieldData> GhostPointData = nullptr;
  };
 
  struct DataSetInformation
  {
    virtual ~DataSetInformation() = default;

    virtual bool InputNeedsGhostsPeeledOff() const = 0;
  };

  struct GridBlockStructure : public DataSetBlockStructure
  {
    GridBlockStructure(const int* extent, int dim);

    ExtentType Extent = ExtentType{ 1, -1, 1, -1, 1, -1 };

    ExtentType ShiftedExtentWithNewGhosts;

    ExtentType ShiftedExtent;
 
    ExtentType ReceivedGhostExtent = ExtentType{ 1, -1, 1, -1, 1, -1 };

    unsigned char AdjacencyMask;

    int DataDimension;
  };

  struct GridInformation : public DataSetInformation
  {
    ~GridInformation() override = default;
 
    bool InputNeedsGhostsPeeledOff() const override { return this->Extent != this->InputExtent; }

    ExtentType Extent = ExtentType{ 1, -1, 1, -1, 1, -1 };

    ExtentType InputExtent = ExtentType{ 1, -1, 1, -1, 1, -1 };
 
    ExtentType ExtentGhostThickness;
  };
 
  struct ImageDataInformation : public GridInformation
  {
    ~ImageDataInformation() override = default;
 
    vtkImageData* Input;
  };

  struct ImageDataBlockStructure : public GridBlockStructure
  {

    ImageDataBlockStructure(const int extent[6], int dim, const double origin[3],
      const double spacing[3], const double orientationQuaternion[4]);
    ImageDataBlockStructure(const int extent[6], int dim, const double origin[3],
      const double spacing[3], vtkMatrix3x3* directionMatrix);

    ImageDataBlockStructure(vtkImageData* image, const ImageDataInformation& info);

    VectorType Origin;

    VectorType Spacing;

    QuaternionType OrientationQuaternion;
  };
 
  struct RectilinearGridInformation : public GridInformation
  {
    ~RectilinearGridInformation() override = default;


    vtkSmartPointer<vtkDataArray> XCoordinates;
    vtkSmartPointer<vtkDataArray> YCoordinates;
    vtkSmartPointer<vtkDataArray> ZCoordinates;

    vtkSmartPointer<vtkDataArray> CoordinateGhosts[6];
 
    vtkRectilinearGrid* Input;
  };

  struct RectilinearGridBlockStructure : public GridBlockStructure
  {
    RectilinearGridBlockStructure(const int extent[6], int dim, vtkDataArray* xCoordinates,
      vtkDataArray* yCoordinates, vtkDataArray* zCoordinates);

    RectilinearGridBlockStructure(vtkRectilinearGrid* grid, const RectilinearGridInformation& info);


    vtkSmartPointer<vtkDataArray> XCoordinates;
    vtkSmartPointer<vtkDataArray> YCoordinates;
    vtkSmartPointer<vtkDataArray> ZCoordinates;
  };
 
  struct StructuredGridInformation : public GridInformation
  {
    ~StructuredGridInformation() override = default;

    struct OuterPointLayersType
    {
      vtkSmartPointer<vtkPoints> Points;

      ExtentType Extent;
    };

    OuterPointLayersType OuterPointLayers[6];

    vtkPoints* InputPoints;
 
    vtkStructuredGrid* Input;
  };

  struct StructuredGridBlockStructure : public GridBlockStructure
  {
    StructuredGridBlockStructure(const int extent[6], int dim, vtkDataArray* points[6]);

    StructuredGridBlockStructure(vtkStructuredGrid* grid, const StructuredGridInformation& info);

    vtkSmartPointer<vtkPoints> OuterPointLayers[6];

    struct Grid2D
    {
      int StartX = 0;

      int StartY = 0;

      int EndX = 0;

      int EndY = 0;

      int XOrientation = 0;

      int YOrientation = 0;

      int ExtentId = -1;
    };

    Grid2D GridInterface;

    vtkNew<vtkPoints> GhostPoints;
  };
 
  struct UnstructuredDataInformation
  {
    bool InputNeedsGhostsPeeledOff() const
    {
      return this->OutputToInputCellIdRedirectionMap != nullptr;
    }

    vtkBoundingBox BoundingBox;

    vtkSmartPointer<vtkIdList> OutputToInputCellIdRedirectionMap = nullptr;


    vtkSmartPointer<vtkIdList> InputToOutputPointIdRedirectionMap = nullptr;
    vtkSmartPointer<vtkIdList> OutputToInputPointIdRedirectionMap = nullptr;

    vtkSmartPointer<vtkAlgorithm> InterfaceExtractor;

    vtkIdTypeArray* InterfacePointIds;

    vtkDataArray* InterfacePoints;

    vtkSmartPointer<vtkIdTypeArray> InterfaceGlobalPointIds;

    /*
     * This is a cursor telling the amount of points / cells information,
     * that has
     * already been added to the output. This variable is used at the very end of the pipeline.
     */
    vtkIdType CurrentMaxPointId;
    vtkIdType CurrentMaxCellId;


    vtkIdType NumberOfInputPoints;
    vtkIdType NumberOfInputCells;
  };
 
  struct UnstructuredDataBlockStructure : public DataSetBlockStructure
  {
    vtkNew<vtkIdList> MatchingReceivedPointIds;

    vtkNew<vtkIdList> RemappedMatchingReceivedPointIdsSortedLikeTarget;

    vtkNew<vtkPoints> InterfacingPoints;

    vtkSmartPointer<vtkIdTypeArray> InterfacingGlobalPointIds = nullptr;

    vtkSmartPointer<vtkIdTypeArray> GhostGlobalPointIds = nullptr;

    vtkNew<vtkPoints> GhostPoints;

    vtkNew<vtkIdList> PointIdsToSend;


    vtkNew<vtkIdTypeArray> SharedPointIds;
    vtkSmartPointer<vtkIdTypeArray> ReceivedSharedPointIds;

    std::map<vtkIdType, vtkIdType> RedirectionMapForDuplicatePointIds;

    vtkNew<vtkIdList> CellIdsToSend;

    vtkSmartPointer<vtkFieldData> InterfacingPointData;
  };
 
  struct UnstructuredGridInformation : public UnstructuredDataInformation
  {
    vtkIdType CurrentFacesSize = 0;

    vtkIdType CurrentMaxFaceId = 0;

    vtkIdType CurrentConnectivitySize = 0;
 
    vtkCellArray* Faces = nullptr;
    vtkCellArray* FaceLocations = nullptr;
 
    vtkUnstructuredGrid* Input;

    vtkIdType InputConnectivitySize = 0;

    vtkIdType InputFacesSize = 0;

    vtkIdType InputNumberOfFaces = 0;
  };
 
  struct UnstructuredGridBlockStructure : public UnstructuredDataBlockStructure
  {
    struct TopologyBufferType
    {
      vtkSmartPointer<vtkUnsignedCharArray> Types = nullptr;
      vtkSmartPointer<vtkCellArray> Faces = nullptr;
      vtkSmartPointer<vtkCellArray> FaceLocations = nullptr;
      vtkNew<vtkCellArray> CellArray;
    };
 
    TopologyBufferType SendBuffer;
    TopologyBufferType ReceiveBuffer;


    vtkIdType FacesSize = 0;
    vtkIdType FacesNum = 0;
    vtkIdType ConnectivitySize = 0;
  };
 
  struct PolyDataInformation : public UnstructuredDataInformation
  {
    vtkPolyData* Input;


    vtkIdType CurrentPolyConnectivitySize;
    vtkIdType CurrentStripConnectivitySize;
    vtkIdType CurrentLineConnectivitySize;


    vtkIdType CurrentMaxPolyId = 0;
    vtkIdType CurrentMaxStripId = 0;
    vtkIdType CurrentMaxLineId = 0;


    vtkIdType NumberOfInputVerts;
    vtkIdType NumberOfInputPolys;
    vtkIdType NumberOfInputStrips;
    vtkIdType NumberOfInputLines;


    vtkIdType InputVertConnectivitySize;
    vtkIdType InputPolyConnectivitySize;
    vtkIdType InputStripConnectivitySize;
    vtkIdType InputLineConnectivitySize;


    vtkNew<vtkIdList> OutputToInputVertCellIdRedirectionMap;
    vtkNew<vtkIdList> OutputToInputLineCellIdRedirectionMap;
    vtkNew<vtkIdList> OutputToInputPolyCellIdRedirectionMap;
    vtkNew<vtkIdList> OutputToInputStripCellIdRedirectionMap;
  };
 
  struct PolyDataBlockStructure : public UnstructuredDataBlockStructure
  {

    vtkNew<vtkIdList> PolyIdsToSend;
    vtkNew<vtkIdList> StripIdsToSend;
    vtkNew<vtkIdList> LineIdsToSend;
 
    struct TopologyBufferType
    {
      vtkNew<vtkCellArray> Polys;
      vtkNew<vtkCellArray> Strips;
      vtkNew<vtkCellArray> Lines;
    };
 
    TopologyBufferType SendBuffer;
    TopologyBufferType ReceiveBuffer;


    vtkIdType NumberOfPolysToSend = 0;
    vtkIdType NumberOfStripsToSend = 0;
    vtkIdType NumberOfLinesToSend = 0;


    vtkIdType PolyConnectivitySize = 0;
    vtkIdType StripConnectivitySize = 0;
    vtkIdType LineConnectivitySize = 0;
  };
 
private:
  struct DataSetBlock
  {
    std::unordered_map<int, std::unordered_map<vtkIdType, vtkIdType>>
      GlobalToLocalIds; //  Per attribute
    std::unordered_map<int, std::unordered_map<vtkIdType, vtkNew<vtkIdTypeArray>>>
      NeededGidsForBlocks; // Ghosts of this block per process id (per attribute)
    std::unordered_map<int, std::unordered_map<vtkIdType, vtkNew<vtkIdTypeArray>>>
      GhostGidsFromBlocks; // Ghosts of other blocks per partition id (per attribute)
  };
 
public:
  template <class BlockStructureT, class InformationT>
  struct Block
  {

    typedef BlockStructureT BlockStructureType;
    typedef InformationT InformationType;

    BlockMapType<BlockStructureType> BlockStructures;

    InformationType Information;
 
    BlockMapType<vtkBoundingBox> NeighborBoundingBoxes;
 
    vtkBoundingBox BoundingBox;
 
    vtkSmartPointer<vtkUnsignedCharArray> GhostCellArray;
    vtkSmartPointer<vtkUnsignedCharArray> GhostPointArray;
  };


  using ImageDataBlock = Block<ImageDataBlockStructure, ImageDataInformation>;
  using RectilinearGridBlock = Block<RectilinearGridBlockStructure, RectilinearGridInformation>;
  using StructuredGridBlock = Block<StructuredGridBlockStructure, StructuredGridInformation>;
  using UnstructuredDataBlock = Block<UnstructuredDataBlockStructure, UnstructuredDataInformation>;
  using UnstructuredGridBlock = Block<UnstructuredGridBlockStructure, UnstructuredGridInformation>;
  using PolyDataBlock = Block<PolyDataBlockStructure, PolyDataInformation>;

  static int SynchronizeGhostData(std::vector<vtkDataSet*>& inputsDS,
    std::vector<vtkDataSet*>& outputsDS, vtkMultiProcessController* controller, bool syncCell,
    bool SyncPoint);

  template <class DataSetT>
  static int GenerateGhostCells(std::vector<DataSetT*>& inputsDS, std::vector<DataSetT*>& outputsDS,
    int outputGhostLevels, vtkMultiProcessController* controller);


  static int GenerateGhostCellsImageData(std::vector<vtkImageData*>& inputs,
    std::vector<vtkImageData*>& outputs, int outputGhostLevels,
    vtkMultiProcessController* controller);
  static int GenerateGhostCellsRectilinearGrid(std::vector<vtkRectilinearGrid*>& inputs,
    std::vector<vtkRectilinearGrid*>& outputs, int outputGhostLevels,
    vtkMultiProcessController* controller);
  static int GenerateGhostCellsStructuredGrid(std::vector<vtkStructuredGrid*>& inputs,
    std::vector<vtkStructuredGrid*>& outputs, int outputGhostLevels,
    vtkMultiProcessController* controller);
  static int GenerateGhostCellsPolyData(std::vector<vtkPolyData*>& inputs,
    std::vector<vtkPolyData*>& outputs, int outputGhostLevels,
    vtkMultiProcessController* controller);
  static int GenerateGhostCellsUnstructuredGrid(std::vector<vtkUnstructuredGrid*>& inputs,
    std::vector<vtkUnstructuredGrid*>& outputs, int outputGhostLevels,
    vtkMultiProcessController* controller);
 
protected:
  vtkDIYGhostUtilities();
  ~vtkDIYGhostUtilities() override;


  static void CloneInputData(std::vector<vtkDataSet*>& inputs, std::vector<vtkDataSet*>& outputs,
    bool syncCell, bool syncPoint);
  static void CloneInputData(vtkDataSet* input, vtkDataSet* output, int fieldType);


  static void InitializeBlocks(
    diy::Master& master, std::vector<vtkDataSet*>& inputs, bool syncCell, bool syncPoint);
  static void InitializeBlocks(
    diy::Master& master, std::vector<vtkDataSet*>& inputs, int fieldType, unsigned char ghostFlag);


  static void ExchangeNeededIds(
    diy::Master& master, const vtkDIYExplicitAssigner& assigner, bool syncCell, bool syncPoint);
  static void ExchangeNeededIds(
    diy::Master& master, const vtkDIYExplicitAssigner& assigner, int fieldType);


  static LinkMap ComputeLinkMapUsingNeededIds(
    const diy::Master& master, bool syncCell, bool syncPoint);
  static void ComputeLinksUsingNeededIds(
    Links& links, vtkDIYGhostUtilities::DataSetBlock* block, int fieldType);


  static void ExchangeFieldData(diy::Master& master, std::vector<vtkDataSet*>& inputs,
    std::vector<vtkDataSet*>& outputs, bool syncCell, bool syncPoint);
  static void ExchangeFieldData(diy::Master& master, std::vector<vtkDataSet*>& inputs,
    std::vector<vtkDataSet*>& outputs, int fieldType);

  static void ReinitializeSelectedBits(vtkUnsignedCharArray* ghosts, unsigned char mask);

  template <class DataSetT>
  static void InitializeGhostPointArray(
    typename DataSetTypeToBlockTypeConverter<DataSetT>::BlockType* block, DataSetT* output);

  template <class DataSetT>
  static void InitializeGhostCellArray(
    typename DataSetTypeToBlockTypeConverter<DataSetT>::BlockType* block, DataSetT* output);


  static void CloneGeometricStructures(
    std::vector<vtkImageData*>& inputs, std::vector<vtkImageData*>& outputs);
  static void CloneGeometricStructures(
    std::vector<vtkRectilinearGrid*>& inputs, std::vector<vtkRectilinearGrid*>& outputs);
  static void CloneGeometricStructures(
    std::vector<vtkStructuredGrid*>& inputs, std::vector<vtkStructuredGrid*>& outputs);
  static void CloneGeometricStructures(
    std::vector<vtkUnstructuredGrid*>& inputs, std::vector<vtkUnstructuredGrid*>& outputs);
  static void CloneGeometricStructures(
    std::vector<vtkPolyData*>& inputs, std::vector<vtkPolyData*>& outputs);


  static void InitializeBlocks(diy::Master& master, std::vector<vtkImageData*>& inputs);
  static void InitializeBlocks(diy::Master& master, std::vector<vtkRectilinearGrid*>& inputs);
  static void InitializeBlocks(diy::Master& master, std::vector<vtkStructuredGrid*>& inputs);
  static void InitializeBlocks(diy::Master& master, std::vector<vtkUnstructuredGrid*>& inputs);
  static void InitializeBlocks(diy::Master& master, std::vector<vtkPolyData*>& inputs);

  template <class DataSetT>
  static void ExchangeBoundingBoxes(
    diy::Master& master, const vtkDIYExplicitAssigner& assigner, std::vector<DataSetT*>& inputs);
 
  template <class BlockT>
  static LinkMap ComputeLinkMapUsingBoundingBoxes(const diy::Master& master);


  static void ExchangeBlockStructures(diy::Master& master, std::vector<vtkImageData*>& inputs);
  static void ExchangeBlockStructures(
    diy::Master& master, std::vector<vtkRectilinearGrid*>& inputs);
  static void ExchangeBlockStructures(diy::Master& master, std::vector<vtkStructuredGrid*>& inputs);
  static void ExchangeBlockStructures(
    diy::Master& master, std::vector<vtkUnstructuredGrid*>& inputs);
  static void ExchangeBlockStructures(diy::Master& master, std::vector<vtkPolyData*>& inputs);


  static LinkMap ComputeLinkMap(
    const diy::Master& master, std::vector<vtkImageData*>& inputs, int outputGhostLevels);
  static LinkMap ComputeLinkMap(
    const diy::Master& master, std::vector<vtkRectilinearGrid*>& inputs, int outputGhostLevels);
  static LinkMap ComputeLinkMap(
    const diy::Master& master, std::vector<vtkStructuredGrid*>& inputs, int outputGhostLevels);
  static LinkMap ComputeLinkMap(
    const diy::Master& master, std::vector<vtkUnstructuredGrid*>& inputs, int outputGhostLevels);
  static LinkMap ComputeLinkMap(
    const diy::Master& master, std::vector<vtkPolyData*>& inputs, int outputGhostLevels);


  static void EnqueueGhosts(const diy::Master::ProxyWithLink& cp, const diy::BlockID& blockId,
    vtkImageData* input, ImageDataBlock* block);
  static void EnqueueGhosts(const diy::Master::ProxyWithLink& cp, const diy::BlockID& blockId,
    vtkRectilinearGrid* input, RectilinearGridBlock* block);
  static void EnqueueGhosts(const diy::Master::ProxyWithLink& cp, const diy::BlockID& blockId,
    vtkStructuredGrid* input, StructuredGridBlock* block);
  static void EnqueueGhosts(const diy::Master::ProxyWithLink& cp, const diy::BlockID& blockId,
    vtkUnstructuredGrid* input, UnstructuredGridBlock* block);
  static void EnqueueGhosts(const diy::Master::ProxyWithLink& cp, const diy::BlockID& blockId,
    vtkPolyData* input, PolyDataBlock* block);


  static void DequeueGhosts(
    const diy::Master::ProxyWithLink& cp, int gid, ImageDataBlockStructure& blockStructure);
  static void DequeueGhosts(
    const diy::Master::ProxyWithLink& cp, int gid, RectilinearGridBlockStructure& blockStructure);
  static void DequeueGhosts(
    const diy::Master::ProxyWithLink& cp, int gid, StructuredGridBlockStructure& blockStructure);
  static void DequeueGhosts(
    const diy::Master::ProxyWithLink& cp, int gid, UnstructuredGridBlockStructure& blockStructure);
  static void DequeueGhosts(
    const diy::Master::ProxyWithLink& cp, int gid, PolyDataBlockStructure& blockStructure);

  template <class DataSetT>
  static void CopyInputsAndAllocateGhosts(diy::Master& master, diy::Assigner& assigner,
    diy::RegularAllReducePartners& partners, std::vector<DataSetT*>& inputs,
    std::vector<DataSetT*>& outputs, int outputGhostLevels);


  static void DeepCopyInputAndAllocateGhosts(
    ImageDataBlock* block, vtkImageData* input, vtkImageData* outputs);
  static void DeepCopyInputAndAllocateGhosts(
    RectilinearGridBlock* block, vtkRectilinearGrid* input, vtkRectilinearGrid* outputs);
  static void DeepCopyInputAndAllocateGhosts(
    StructuredGridBlock* block, vtkStructuredGrid* input, vtkStructuredGrid* outputs);
  static void DeepCopyInputAndAllocateGhosts(
    UnstructuredGridBlock* block, vtkUnstructuredGrid* input, vtkUnstructuredGrid* outputs);
  static void DeepCopyInputAndAllocateGhosts(
    PolyDataBlock* block, vtkPolyData* input, vtkPolyData* outputs);

  template <class DataSetT>
  static bool ExchangeGhosts(diy::Master& master, diy::Assigner& assigner,
    diy::RegularAllReducePartners& partners, std::vector<DataSetT*>& inputs);

  template <class DataSetT>
  static void InitializeGhostArrays(
    diy::Master& master, std::vector<DataSetT*>& outputs, int outputGhostLevels);

  template <class DataSetT>
  static void AddGhostArrays(diy::Master& master, std::vector<DataSetT*>& outputs);


  static void FillGhostArrays(
    const diy::Master& master, std::vector<vtkImageData*>& outputs, int outputGhostLevels);
  static void FillGhostArrays(
    const diy::Master& master, std::vector<vtkRectilinearGrid*>& outputs, int outputGhostLevels);
  static void FillGhostArrays(
    const diy::Master& master, std::vector<vtkStructuredGrid*>& outputs, int outputGhostLevels);
  static void FillGhostArrays(
    const diy::Master& master, std::vector<vtkUnstructuredGrid*>& outputs, int outputGhostLevels);
  static void FillGhostArrays(
    const diy::Master& master, std::vector<vtkPolyData*>& outputs, int outputGhostLevels);
 
private:
  vtkDIYGhostUtilities(const vtkDIYGhostUtilities&) = delete;
  void operator=(const vtkDIYGhostUtilities&) = delete;


  static void InflateBoundingBoxIfNecessary(
    vtkDataSet* vtkNotUsed(input), vtkBoundingBox& vtkNotUsed(bb));
  static void InflateBoundingBoxIfNecessary(vtkPointSet* input, vtkBoundingBox& bb);
};
 
VTK_ABI_NAMESPACE_END
#include "vtkDIYGhostUtilities.txx" // for template implementations
 
#endif