Tokenize supplied chain into tokens expected to form a boolean expression

Declared and defined in file: hzTokens.cpp

Function Logic:

Function body:

hzEcode TokenizeBool (hzVect<hzToken>& toks)hzChain& C, 
{
   //  Category: Text Processing
   //  
   //  Tokenize supplied chain into tokens expected to form a boolean expression
   //  
   //  Arguments: 1) toks The vector of tokens found in the input
   //     2) C  The input chain
   //  
   //  Returns: E_NODATA If the supplied chain is empty
   //     E_OK  If the supplied chain is tokenized
   _hzfunc("hzTokenlist::TokenizeBool") ;
   hzChain     W ;             //  For building tokens
   chIter      ci ;            //  For iteration of input
   hzToken     T ;             //  Token
   hzString    S ;             //  For assembling the token value
   uint32_t    nLine ;         //  For assigning line numbers to tokens
   char        tmp [4];        //  For operator
   toks.Clear() ;
   if (!C.Size())
       return E_NODATA ;
   ci = C ;
   nLine = 1;
   for (; !ci.eof() ;)
   {
       //  Increment line number when newlines are encountered
       if (*ci == CHAR_NL)
           { nLine++ ; ci++ ; continue ; }
       //  Strip whitespace and other non-printable chars
       if (IsBinary(*ci) || IsWhite(*ci))
           { ci++ ; continue ; }
       //  Eliminate comments
       if (ci == "/*")
       {
           for (ci += 2; !ci.eof() && ci != "*/" ; ci++) ;
           ci += 2;
           continue ;
       }
       if (ci == "//")
       {
           for (ci += 2; !ci.eof() && *ci != CHAR_NL ; ci++) ;
           ci++ ;
           continue ;
       }
       //  Assume we are at the start of a token - Check for quoted string
       if (*ci == CHAR_DQUOTE)
       {
           for (ci++ ; !ci.eof() && *ci != CHAR_DQUOTE ; ci++)
               W.AddByte(*ci) ;
           S = W ;
           W.Clear() ;
           T.Init(S, ci.Line(), TOKEN_STRING) ;
           toks.Add(T) ;
           ci++ ;
           continue ;
       }
       //  Check for valid hexadecimal value
       if (_testHexnum(S, ci))
       {
           T.Init(S, ci.Line(), TOKEN_STRING) ;
           toks.Add(T) ;
           continue ;
       }
       //  Check for integer
       if (_testInteger(S, ci))
       {
           T.Init(S, ci.Line(), TOKEN_INTEGER) ;
           toks.Add(T) ;
           continue ;
       }
       //  Check for number (std form)
       if (_testNumber(S, ci))
       {
           T.Init(S, ci.Line(), TOKEN_NUMBER) ;
           toks.Add(T) ;
           continue ;
       }
       //  Check for seperator
       if (IsPunct(*ci))
       {
           tmp[0]= *ci ;
           tmp[1]= 0;
           ci++ ;
           T.Init(tmp, nLine, TOKEN_SEPARATOR) ;
           toks.Add(T) ;
           continue ;
       }
       //  Check for operator
       if (IsSymb(*ci))
       {
           for (; !ci.eof() && IsSymb(*ci) ; ci++)
               W.AddByte(*ci) ;
           S = W ;
           W.Clear() ;
           T.Init(S, nLine, TOKEN_OPERATOR) ;
           toks.Add(T) ;
           continue ;
       }
       //  Not an operator or separator - must be general entitiy
       //  We have a rule that productions must be written out
       //  in full. E.g. 2X must be written as 2*X. The system
       //  will interpret 2X as an alpha numeric quantity
       for (; !ci.eof() && IsAlphanum(*ci) ; ci++)
           W.AddByte(*ci) ;
       S = W ;
       W.Clear() ;
       T.Init(S, nLine, TOKEN_OPERATOR) ;
       toks.Add(T) ;
   }
   return E_OK ;
}