25 #ifndef PRESAGE_TOKENIZER
26 #define PRESAGE_TOKENIZER
110 std::streamoff offbackup =
stream.tellg();
112 std::streamoff curroff =
offbeg;
114 while (curroff <
offend) {
116 str.push_back(
stream.peek());
std::ios::iostate currstate
std::istream & guardedStream
StreamGuard(std::istream &so, std::streamoff &of)
virtual std::string nextToken()=0
std::string separatorChars() const
bool isSeparator(const int character) const
virtual int countTokens()=0
std::string streamToString() const
virtual double progress() const =0
bool lowercaseMode() const
bool isBlankspace(const int character) const
Tokenizer(std::istream &stream, const std::string blankspaces, const std::string separators)
std::string blankspaceChars() const
virtual bool hasMoreTokens() const =0