RooStreamParser.cxx

Go to the documentation of this file.

/*****************************************************************************
 * Project: RooFit                                                           *
 * Package: RooFitCore                                                       *
 * @(#)root/roofitcore:$Id$
 * Authors:                                                                  *
 *   WV, Wouter Verkerke, UC Santa Barbara, verkerke@slac.stanford.edu       *
 *   DK, David Kirkby,    UC Irvine,         dkirkby@uci.edu                 *
 *                                                                           *
 * Copyright (c) 2000-2005, Regents of the University of California          *
 *                          and Stanford University. All rights reserved.    *
 *                                                                           *
 * Redistribution and use in source and binary forms,                        *
 * with or without modification, are permitted according to the terms        *
 * listed in LICENSE (http://roofit.sourceforge.net/license.txt)             *
 *****************************************************************************/
 
//////////////////////////////////////////////////////////////////////////////
//
// RooStreamParser is a utility class to parse istreams into tokens and optionally
// convert them into basic types (double,int,string)
//
// The general tokenizing philosophy is that there are two kinds of tokens: value
// and punctuation. The former are variable length, the latter always
// one character. A token is terminated if one of the following conditions
// occur
//         - space character found (' ',tab,newline)
//         - change of token type (value -> punctuation or vv)
//         - end of fixed-length token (punctuation only)
//         - start or end of quoted string
//
// The parser is aware of floating point notation and will assign leading
// minus signs, decimal points etc to a value token when this is obvious
// from the context. The definition of what is punctuation can be redefined.
//
 
 
#include "Riostream.h"
#include <cstdlib>
 
#ifndef _WIN32
#include <strings.h>
#endif
 
#include "RooStreamParser.h"
#include "RooMsgService.h"
#include "RooNumber.h"
#include "RooFitImplHelpers.h"
 
using std::istream, std::endl;
 
 
 
////////////////////////////////////////////////////////////////////////////////
/// Construct parser on given input stream
 
RooStreamParser::RooStreamParser(istream& is) :
  _is(&is), _atEOL(false), _atEOF(false), _prefix(""), _punct("()[]<>|/\\:?.,=+-&^%$#@!`~")
{
}
 
 
////////////////////////////////////////////////////////////////////////////////
/// Construct parser on given input stream. Use given errorPrefix to
/// prefix any parsing error messages
 
RooStreamParser::RooStreamParser(istream& is, const TString& errorPrefix) :
  _is(&is), _atEOL(false), _atEOF(false), _prefix(errorPrefix), _punct("()[]<>|/\\:?.,=+-&^%$#@!`~")
{
}
 
////////////////////////////////////////////////////////////////////////////////
/// If true, parser is at end of line in stream
 
bool RooStreamParser::atEOL()
{
  Int_t nc(_is->peek()) ;
  return (nc=='\n'||nc==-1) ;
}
 
 
 
////////////////////////////////////////////////////////////////////////////////
/// Change list of characters interpreted as punctuation
 
void RooStreamParser::setPunctuation(const TString& punct)
{
  _punct = punct ;
}
 
 
 
////////////////////////////////////////////////////////////////////////////////
/// Check if given char is considered punctuation
 
bool RooStreamParser::isPunctChar(char c) const
{
  const char* punct = _punct.Data() ;
  for (int i = 0; i < _punct.Length(); i++) {
     if (punct[i] == c) {
        return true;
     }
  }
  return false ;
}
 
 
 
////////////////////////////////////////////////////////////////////////////////
/// Read one token separated by any of the know punctuation characters
/// This function recognizes and handles comment lines in the istream (those
/// starting with '#', quoted strings ("") the content of which is not tokenized
/// and '+-.' characters that are part of a floating point numbers and are exempt
/// from being interpreted as a token separator in case '+-.' are defined as
/// token separators.
 
TString RooStreamParser::readToken()
{
  // Smart tokenizer. Absorb white space and token must be either punctuation or alphanum
  bool first(true);
  bool quotedString(false);
  bool lineCont(false);
  char buffer[64000];
  char c(0);
  char cnext = '\0';
  char cprev = ' ';
  bool haveINF(false) ;
  Int_t bufptr(0) ;
 
  // Check for end of file
  if (_is->eof() || _is->fail()) {
    _atEOF = true ;
    return TString("") ;
  }
 
  //Ignore leading newline
  if (_is->peek()=='\n') {
    _is->get(c) ;
 
    // If new line starts with #, zap it
    while (_is->peek()=='#') {
      zapToEnd(false) ;
      _is->get(c) ; // absorb newline
    }
  }
 
  while(true) {
    // Buffer overflow protection
    if (bufptr >= 63999) {
      oocoutW(nullptr, InputArguments)
              << "RooStreamParser::readToken: token length exceeds buffer capacity, terminating token early" << std::endl;
      break;
    }
 
    // Read next char
    _is->get(c) ;
 
 
 
    // Terminate at EOF, EOL or trouble
    if (_is->eof() || _is->fail() || c=='\n') break ;
 
    // Terminate as SPACE, unless we haven't seen any non-SPACE yet
    if (isspace(c)) {
      if (first) {
         continue;
      } else if (!quotedString) {
         break;
      }
    }
 
    // If '-' or '/' see what the next character is
    if (c == '.' || c=='-' || c=='+' || c=='/' || c=='\\') {
      _is->get(cnext) ;
 
 
      if (cnext=='I' || cnext=='i') {
          char tmp1;
          char tmp2;
          _is->get(tmp1);
          _is->get(tmp2);
          _is->putback(tmp2);
          _is->putback(tmp1);
          haveINF = ((cnext == 'I' && tmp1 == 'N' && tmp2 == 'F') || (cnext == 'i' && tmp1 == 'n' && tmp2 == 'f'));
      } else {
        haveINF = false ;
      }
 
      _is->putback(cnext) ;
    }
 
 
    // Check for line continuation marker
    if (c=='\\' && cnext=='\\') {
      // Kill rest of line including std::endline marker
      zapToEnd(false) ;
      _is->get(c) ;
      lineCont=true ;
      break ;
    }
 
    // Stop if begin of comments is encountered
    if (c=='/' && cnext=='/') {
      zapToEnd(false) ;
      break ;
    }
 
    // Special handling of quoted strings
    if (c=='"') {
      if (first) {
        quotedString=true ;
      } else if (!quotedString) {
        // Terminate current token. Next token will be quoted string
        _is->putback('"') ;
        break ;
      }
    }
 
    if (!quotedString) {
      // Decide if next char is punctuation (exempt - and . that are part of floating point numbers, or +/- preceding INF)
      if (isPunctChar(c) && !(c=='.' && (isdigit(cnext)||isdigit(cprev)))
          && !((c=='-'||c=='+') && isdigit(cnext) && (cprev == 'e' || cprev == 'E'))
          && (!first || !((c=='-'||c=='+') && (isdigit(cnext)||cnext=='.'||haveINF)))) {
 
        if (first) {
          // Make this a one-char punctuation token
          buffer[bufptr++]=c ;
          break ;
        } else {
          // Put back punct. char and terminate current alphanum token
          _is->putback(c) ;
          break ;
        }
      }
    } else {
      // Inside quoted string conventional tokenizing rules do not apply
 
      // Terminate token on closing quote
      if (c=='"' && !first) {
        buffer[bufptr++]=c ;
        quotedString=false ;
        break ;
      }
    }
 
    // Store in buffer
    buffer[bufptr++]=c ;
    first=false ;
    cprev=c ;
  }
 
  if (_is->eof() || _is->bad()) {
    _atEOF = true ;
  }
 
  // Check if closing quote was encountered
  if (quotedString) {
    oocoutW(nullptr,InputArguments) << "RooStreamParser::readToken: closing quote (\") missing" << std::endl ;
  }
 
  // Absorb trailing white space or absorb rest of line if // is encountered
  if (c=='\n') {
    if (!lineCont) {
      _is->putback(c) ;
    }
  } else {
    c = _is->peek() ;
 
    while ((isspace(c) || c=='/') && c != '\n') {
      if (c=='/') {
        _is->get(c) ;
        if (_is->peek()=='/') {
          zapToEnd(false) ;
        } else {
          _is->putback('/') ;
        }
        break ;
      } else {
        _is->get(c) ;
        c = _is->peek() ;
      }
    }
  }
 
  // If no token was read line is continued, return first token on next line
  if (bufptr==0 && lineCont) {
    return readToken() ;
  }
 
  // Zero terminate buffer and convert to TString
  buffer[bufptr]=0 ;
  return TString(buffer) ;
}
 
 
 
////////////////////////////////////////////////////////////////////////////////
/// Read an entire line from the stream and return as TString
/// This method recognizes the use of '\\' in the istream
/// as line continuation token.
 
TString RooStreamParser::readLine()
{
   char c;
   char buffer[64000];
   Int_t nfree(63999);
 
   if (_is->peek() == '\n')
      _is->get(c);
 
   // Read till end of line
   _is->getline(buffer, nfree, '\n');
 
   // Look for eventual continuation line sequence
   char *pcontseq = strstr(buffer, "\\\\");
   if (pcontseq)
      nfree -= (pcontseq - buffer);
   while (pcontseq) {
      _is->getline(pcontseq, nfree, '\n');
 
      char *nextpcontseq = strstr(pcontseq, "\\\\");
      if (nextpcontseq)
         nfree -= (nextpcontseq - pcontseq);
      pcontseq = nextpcontseq;
   }
 
   // Chop eventual comments
   char *pcomment = strstr(buffer, "//");
   if (pcomment)
      *pcomment = 0;
 
   // Chop leading and trailing space
   char *pstart = buffer;
   while (isspace(*pstart)) {
      pstart++;
   }
   char *pend = buffer + strlen(buffer) - 1;
   if (pend > pstart) {
      while (isspace(*pend)) {
         *pend-- = 0;
      }
   }
 
   if (_is->eof() || _is->fail()) {
      _atEOF = true;
   }
 
   // Convert to TString
   return TString(pstart);
}
 
 
 
////////////////////////////////////////////////////////////////////////////////
/// Eat all characters up to and including then end of the
/// current line. If inclContLines is true, all continuation lines
/// marked by the '\\' token are zapped as well
 
void RooStreamParser::zapToEnd(bool inclContLines)
{
  // Skip over everything until the end of the current line
  if (_is->peek()!='\n') {
 
     char buffer[64000];
     Int_t nfree(63999);
 
     // Read till end of line
     _is->getline(buffer, nfree, '\n');
 
     if (inclContLines) {
        // Look for eventual continuation line sequence
        char *pcontseq = strstr(buffer, "\\\\");
        if (pcontseq)
           nfree -= (pcontseq - buffer);
        while (pcontseq) {
           _is->getline(pcontseq, nfree, '\n');
 
           char *nextpcontseq = strstr(pcontseq, "\\\\");
           if (nextpcontseq)
              nfree -= (nextpcontseq - pcontseq);
           pcontseq = nextpcontseq;
        }
    }
 
    // Put back newline character in stream buffer
   _is->putback('\n') ;
  }
}
 
 
 
////////////////////////////////////////////////////////////////////////////////
/// Read the next token and return true if it is identical to the given 'expected' token.
 
bool RooStreamParser::expectToken(const TString& expected, bool zapOnError)
{
  TString token(readToken()) ;
 
  bool error=token.CompareTo(expected) ;
  if (error && !_prefix.IsNull()) {
    oocoutW(nullptr,InputArguments) << _prefix << ": parse error, expected '"
               << expected << "'" << ", got '" << token << "'" << std::endl ;
    if (zapOnError) zapToEnd(true) ;
  }
  return error ;
}
 
 
 
////////////////////////////////////////////////////////////////////////////////
/// Read the next token and convert it to a double. Returns true
/// if an error occurred in reading or conversion
 
bool RooStreamParser::readDouble(double& value, bool /*zapOnError*/)
{
  TString token(readToken()) ;
  if (token.IsNull()) return true ;
  return convertToDouble(token,value) ;
 
}
 
////////////////////////////////////////////////////////////////////////////////
/// Convert given string to a double. Throws exceptions if the conversion fails.
 
bool RooStreamParser::convertToDouble(const TString &token, double &value)
{
   value = toDouble(token.Data());
   return false;
}
 
 
 
////////////////////////////////////////////////////////////////////////////////
/// Read a token and convert it to an Int_t. Returns true
/// if an error occurred in reading or conversion
 
bool RooStreamParser::readInteger(Int_t& value, bool /*zapOnError*/)
{
  TString token(readToken()) ;
  if (token.IsNull()) return true ;
  return convertToInteger(token,value) ;
}
 
 
 
////////////////////////////////////////////////////////////////////////////////
/// Convert given string to an Int_t. Returns true if an error
/// occurred in conversion
 
bool RooStreamParser::convertToInteger(const TString& token, Int_t& value)
{
  char* endptr = nullptr;
  const char* data=token.Data() ;
  value = strtol(data,&endptr,10) ;
  bool error = (endptr-data!=token.Length()) ;
 
  if (error && !_prefix.IsNull()) {
    oocoutE(nullptr,InputArguments)<< _prefix << ": parse error, cannot convert '"
                   << token << "'" << " to integer" <<  std::endl ;
  }
  return error ;
}
 
 
 
////////////////////////////////////////////////////////////////////////////////
/// Read a string token. Returns true if an error occurred in reading
/// or conversion.  If a the read token is enclosed in quotation
/// marks those are stripped in the returned value
 
bool RooStreamParser::readString(TString& value, bool /*zapOnError*/)
{
  TString token(readToken()) ;
  if (token.IsNull()) return true ;
  return convertToString(token,value) ;
}
 
 
 
////////////////////////////////////////////////////////////////////////////////
/// Convert given token to a string (i.e. remove eventual quotation marks)
 
bool RooStreamParser::convertToString(const TString& token, TString& string)
{
   // Transport to buffer
  char buffer[64000];
  char *ptr;
  strncpy(buffer, token.Data(), 63999);
  if (token.Length() >= 63999) {
    oocoutW(nullptr, InputArguments) << "RooStreamParser::convertToString: token length exceeds 63999, truncated"
                                     << std::endl;
    buffer[63999] = 0;
  }
  int len = strlen(buffer) ;
 
  // Remove trailing quote if any
  if ((len) && (buffer[len-1]=='"'))
    buffer[len-1]=0 ;
 
  // Skip leading quote, if present
  ptr=(buffer[0]=='"') ? buffer+1 : buffer ;
 
  string = ptr ;
  return false ;
}