Populate supplied vector of tokens (arg 1) by tokenizing the supplied chain (arg 2) according to the modus operandi specified by arg 3

Declared in file: hzTokens.h
Defined in file : hzTokens.cpp

Function Logic:

Function body:

hzEcode TokenizeChain (hzVect<hzToken>& toks)hzChain& C, hzTokMode eMode, 
{
   //  Category: Text Processing
   //  
   //  Populate supplied vector of tokens (arg 1) by tokenizing the supplied chain (arg 2) according to the modus operandi specified by arg 3
   //  
   //  Arguments: 1) toks The vector of tokens found in the input
   //     2) C  The input chain
   //     3) eMode The tokenization regime (either WHITE, FTEXT or BOOL)
   //  
   //  Returns: E_NODATA If the supplied chain is empty
   //     E_RANGE  If the supplied mode is invalid
   //     E_OK  If the supplied chain is tokenized
   switch  (eMode)
   {
   case TOK_MO_WHITE:  return TokenizeWords(toks, C) ;
   case TOK_MO_FTEXT:  return TokenizeFreetext(toks, C) ;
   case TOK_MO_BOOL:   return TokenizeBool(toks, C) ;
   }
   return E_RANGE ;
}