dox/TextAnalysis/vtkTokenizer.h

Go to the documentation of this file.

/*=========================================================================

  Program:   Visualization Toolkit
  Module:    vtkTokenizer.h

  Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
  All rights reserved.
  See Copyright.txt or http://www.kitware.com/Copyright.htm for details.

     This software is distributed WITHOUT ANY WARRANTY; without even
     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
     PURPOSE.  See the above copyright notice for more information.

=========================================================================*/
/*-------------------------------------------------------------------------
  Copyright 2008 Sandia Corporation.
  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
  the U.S. Government retains certain rights in this software.
-------------------------------------------------------------------------*/

#ifndef __vtkTokenizer_h
#define __vtkTokenizer_h

#include <vtkTableAlgorithm.h>
#include <vtkUnicodeString.h> //Needed for delimiter specification

class VTK_TEXT_ANALYSIS_EXPORT vtkTokenizer :
  public vtkTableAlgorithm
{
public:
  static vtkTokenizer* New();
  vtkTypeMacro(vtkTokenizer, vtkTableAlgorithm);
  void PrintSelf(ostream& os, vtkIndent indent);

//BTX
  typedef std::pair<vtkUnicodeString::value_type, vtkUnicodeString::value_type> DelimiterRange;
  typedef std::vector<DelimiterRange> DelimiterRanges;

  static const DelimiterRanges Punctuation();
  static const DelimiterRanges Whitespace();
  static const DelimiterRanges Logosyllabic();

  void AddDroppedDelimiters(vtkUnicodeString::value_type begin, vtkUnicodeString::value_type end);
  void AddDroppedDelimiters(const DelimiterRanges& ranges);

  void AddKeptDelimiters(vtkUnicodeString::value_type begin, vtkUnicodeString::value_type end);

  void AddKeptDelimiters(const DelimiterRanges& ranges);
//ETX


  void DropPunctuation();
  void DropWhitespace();
  void KeepPunctuation();
  void KeepWhitespace();
  void KeepLogosyllabic();

  void ClearDroppedDelimiters();
  void ClearKeptDelimiters();

//BTX
protected:
  vtkTokenizer();
  ~vtkTokenizer();

  int FillInputPortInformation(int port, vtkInformation* info);

  virtual int RequestData(
    vtkInformation* request,
    vtkInformationVector** inputVector,
    vtkInformationVector* outputVector);

private:
  vtkTokenizer(const vtkTokenizer &); // Not implemented.
  void operator=(const vtkTokenizer &); // Not implemented.

  class Internals;
  Internals* const Implementation;
//ETX
};

#endif // __vtkTokenizer_h