//*****************************************************************************
/*
** FILE:   xe_TokenizerAuto.java
**
** (c) 1998 Steve Withall.
**
** HISTORY:
**    17Apr98  stevew  Created, split from xe_ParseManager.
**    06May98  stevew  Renamed from xe_ParseManagerAuto to xe_TokenizerAuto.
*/
package xe;

import xm.xm_ParseException;

import eh.eh_Debug;

import xa.xa_CharChecker;

import java.io.IOException;

//*****************************************************************************
/** <p>An xe_Tokenizer Perform which uses an approach to parsing the next token
 *  different to xe_Tokenizer. This approach uses the definition of each
 *  keyword in xe_TokenType to identify when the token has been parsed.</p>
 *
 *  <p>Auto-parsing in this way has turned out to be very slow - which is why
 *  this functionality has been relegated to a separate class. However, it can
 *   be reinstated simply by instantiating this class instead of
 *  xe_Tokenizer for low-level parsing.</p>
 */
public class xe_TokenizerAuto extends xe_Tokenizer
{
    //*****************************************************************************
    /** Default constructor.
     */
    public xe_TokenizerAuto()
    {
        super();
    }

    //*****************************************************************************
    /** Parse the next token. Overrides ParseNextToken() from xe_Tokenizer.
     *
     *  @param  InputConsumeWhitespaceFlag  If true, all preceding whitespace is
     *                                       separated; otherwise, it is treated
     *                                       as part of the token itself
     *  @return                             Token representing the value parsed
     *  @exception  xm_ParseException  Something was parsed which didn't fit the
     *                                  parse rules
     *  @exception  IOException        Error reading from source reader
     */
    public xe_Token parseNextToken(boolean  InputConsumeWhitespaceFlag)
                                     throws xm_ParseException, IOException
    {
        eh_Debug.add(8, "xe_TokenizerAuto.parseNextToken:");
        LastSignificantPosition = TotalCharCount;

        StringBuffer  TokenValueBuffer = new StringBuffer(DEFAULT_NAME_LENGTH);

        return autoParseNextToken(TokenValueBuffer, InputConsumeWhitespaceFlag);
    }

    //*****************************************************************************
    /** <p>Parse the next token. Parsing here is completely controlled by the
     *  definitions of keywords held in the xe_TokenType class - which it makes it
     *  extremely powerful and extensible. However, it is also relatively
     *  inefficient (having to access a hashtable for every character read). For
     *  this reason, it is intended to be called by other parse routines which are
     *  hard-wired to perform the bulk of the parsing fast.</p>
     *
     *  <p>This method can be called at any point in the parsing of a token (at the
     *  start, or half-way through), and will continue from the point which has been
     *  reached (as recorded in the InputTokenValueBuffer).</p>
     *
     *  <p>Note that when called by another parse routine, InputConsumeWhitespaceFlag
     *  should be passed as 'false' if that routine has already dealt with precding
     *  whitespace.</p>
     *
     *  @param  InputTokenValueBuffer       The characters parsed already in this
     *                                       token
     *  @param  InputConsumeWhitespaceFlag  If true, all preceding whitespace is
     *                                       separated; otherwise, it is treated
     *                                       as part of the token itself
     *  @return                             Token representing the value parsed
     *  @exception  xm_ParseException  Something was parsed which didn't fit the
     *                                  parse rules
     *  @exception  IOException        Error reading from source reader
     */
    private xe_Token autoParseNextToken(StringBuffer  InputTokenValueBuffer,
                                        boolean       InputConsumeWhitespaceFlag)
                                           throws xm_ParseException, IOException
    {
        eh_Debug.add(8, "xe_TokenizerAuto.autoParseNextToken:");

        // Get preceding whitespace if required.
        String  PrecedingWhitespace = parseWhitespace(InputConsumeWhitespaceFlag);

        xe_Token          NextToken          = null;
        String            TokenValueString   = null;
        xe_TokenTypeDefn  CurrentKeywordDefn = null;
        xe_TokenTypeDefn  BestKeywordDefn    = null;
        int               CurrentMoreControl;
        boolean           TokenEndFlag       = false;

        if (atEnd())
            NextToken = new xe_Token(xe_TokenType.END_OF_SOURCE);
        else if (    xa_CharChecker.haveNameChar(CurrentChar)
                  || xa_CharChecker.isWhitespace(CurrentChar) )
            NextToken =  parseNameToken();  // Treat this as a name
        else
        {
            // Keep adding characters until we no longer have a valid keyword
            // or no bigger keyword is possible.
            InputTokenValueBuffer.append(CurrentChar);
            while (!atEnd() && !TokenEndFlag)
            {
                TokenValueString   = InputTokenValueBuffer.toString();
                CurrentKeywordDefn = xe_TokenType.getDefn(InputTokenValueBuffer.toString() );
                eh_Debug.add(7, "xe_TokenizerAuto.autoParseNextToken: " + InputTokenValueBuffer.toString());

                if (CurrentKeywordDefn == null)
                    TokenEndFlag = true;
                else
                {
                    readNextChar();

                    BestKeywordDefn    = CurrentKeywordDefn;
                    CurrentMoreControl = CurrentKeywordDefn.getMoreControl();
                    switch (CurrentMoreControl)
                    {
                        case xe_TokenType.NO_MORE:
                            TokenEndFlag = true;
                            break;

                        case xe_TokenType.MORE_SPECIAL:
                            // Just keep going!
                            InputTokenValueBuffer.append(CurrentChar);
                            break;

                        case xe_TokenType.MORE_NAME:
                            if (xa_CharChecker.haveNameStartChar(CurrentChar))
                                parseName(InputTokenValueBuffer);
                            else
                                throwParseException("Expected name character (but got " + CurrentChar + ")");
                            break;

                        case xe_TokenType.MORE_NUMBER:
                            if (xa_CharChecker.haveNumberChar(CurrentChar))
                                InputTokenValueBuffer.append(parseNumber() );
                            else
                                throwParseException("Expected number character (but got " + CurrentChar + ")");
                            break;

                        case xe_TokenType.MORE_ANY:
                        default:
                            if (xa_CharChecker.haveNameStartChar(CurrentChar))
                                parseName(InputTokenValueBuffer);
                            else
                                InputTokenValueBuffer.append(CurrentChar);
                    }
                }
            }

//          if (InputTokenValueBuffer.length() > 0)
//              TokenValueString = InputTokenValueBuffer.toString();

            //TBD Note that the reversion to the BestKeywordDefn after failing to find
            //TBD a better one is incomplete: anything parsed in the meantime needs to
            //TBD be 'put back'. Alternatively, failing to find a better token could
            //TBD raise an exception.

            if (BestKeywordDefn == null)
                NextToken = new xe_Token(xe_TokenType.UNKNOWN);
            else
                NextToken = new xe_Token(BestKeywordDefn.getType(),
                                         InputTokenValueBuffer.toString());
        }

        // Store the preceding whitespace as part of the token.
        if (NextToken != null && PrecedingWhitespace != null)
            NextToken.setPrecedingWhitespace(PrecedingWhitespace);

        eh_Debug.add(6, "xe_TokenizerAuto.autoParseNextToken: Return (" + NextToken + ")");
        return NextToken;
    }
}

//*****************************************************************************
