Populate supplied vector of tokens (arg 1) by tokenizing the supplied char string (arg 2) according to the modus operandi specified by arg 3

Declared in file: hzTokens.h
Defined in file : hzTokens.cpp

Function Logic:

Function body:

hzEcode TokenizeString (hzVect<hzToken>& toks)const char* pBuf, hzTokMode eMode, 
{
   //  Category: Text Processing
   //  
   //  Populate supplied vector of tokens (arg 1) by tokenizing the supplied char string (arg 2) according to the modus operandi specified by
   //  arg 3
   //  
   //  Arguments: 1) toks The vector of tokens found in the input
   //     2) C  The input chain
   //     3) eMode The tokenization regime (either WHITE, FTEXT or BOOL)
   //  
   //  Returns: E_NODATA If the supplied chain is empty
   //     E_RANGE  If the supplied mode is invalid
   //     E_OK  If the supplied chain is tokenized
   _hzfunc("TokenizeString") ;
   hzChain C ;     //  Working chain
   C = pBuf ;
   switch  (eMode)
   {
   case TOK_MO_WHITE:  return TokenizeWords(toks, C) ;
   case TOK_MO_FTEXT:  return TokenizeFreetext(toks, C) ;
   case TOK_MO_BOOL:   return TokenizeBool(toks, C) ;
   }
   return E_RANGE ;
}