/** \file
*
* Base implementation of an antlr 3 lexer.
*
* An ANTLR3 lexer implements a base recongizer, a token source and
* a lexer interface. It constructs a base recognizer with default
* functions, then overrides any of these that are parser specific (usual
* default implementation of base recognizer.
*/
// [The "BSD licence"]
// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
// http://www.temporal-wave.com
// http://www.linkedin.com/in/jimidle
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <antlr3lexer.h>
static void mTokens (pANTLR3_LEXER lexer);
static void setCharStream (pANTLR3_LEXER lexer, pANTLR3_INPUT_STREAM input);
static void pushCharStream (pANTLR3_LEXER lexer, pANTLR3_INPUT_STREAM input);
static void popCharStream (pANTLR3_LEXER lexer);
static void emitNew (pANTLR3_LEXER lexer, pANTLR3_COMMON_TOKEN token);
static pANTLR3_COMMON_TOKEN emit (pANTLR3_LEXER lexer);
static ANTLR3_BOOLEAN matchs (pANTLR3_LEXER lexer, ANTLR3_UCHAR * string);
static ANTLR3_BOOLEAN matchc (pANTLR3_LEXER lexer, ANTLR3_UCHAR c);
static ANTLR3_BOOLEAN matchRange (pANTLR3_LEXER lexer, ANTLR3_UCHAR low, ANTLR3_UCHAR high);
static void matchAny (pANTLR3_LEXER lexer);
static void recover (pANTLR3_LEXER lexer);
static ANTLR3_UINT32 getLine (pANTLR3_LEXER lexer);
static ANTLR3_MARKER getCharIndex (pANTLR3_LEXER lexer);
static ANTLR3_UINT32 getCharPositionInLine (pANTLR3_LEXER lexer);
static pANTLR3_STRING getText (pANTLR3_LEXER lexer);
static pANTLR3_COMMON_TOKEN nextToken (pANTLR3_TOKEN_SOURCE toksource);
static void displayRecognitionError (pANTLR3_BASE_RECOGNIZER rec, pANTLR3_UINT8 * tokenNames);
static void reportError (pANTLR3_BASE_RECOGNIZER rec);
static void * getCurrentInputSymbol (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM istream);
static void * getMissingSymbol (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM istream, pANTLR3_EXCEPTION e,
ANTLR3_UINT32 expectedTokenType, pANTLR3_BITSET_LIST follow);
static void reset (pANTLR3_BASE_RECOGNIZER rec);
static void freeLexer (pANTLR3_LEXER lexer);
ANTLR3_API pANTLR3_LEXER
antlr3LexerNew(ANTLR3_UINT32 sizeHint, pANTLR3_RECOGNIZER_SHARED_STATE state)
{
pANTLR3_LEXER lexer;
pANTLR3_COMMON_TOKEN specialT;
/* Allocate memory
*/
lexer = (pANTLR3_LEXER) ANTLR3_MALLOC(sizeof(ANTLR3_LEXER));
if (lexer == NULL)
{
return NULL;
}
/* Now we need to create the base recognizer
*/
lexer->rec = antlr3BaseRecognizerNew(ANTLR3_TYPE_LEXER, sizeHint, state);
if (lexer->rec == NULL)
{
lexer->free(lexer);
return NULL;
}
lexer->rec->super = lexer;
lexer->rec->displayRecognitionError = displayRecognitionError;
lexer->rec->reportError = reportError;
lexer->rec->reset = reset;
lexer->rec->getCurrentInputSymbol = getCurrentInputSymbol;
lexer->rec->getMissingSymbol = getMissingSymbol;
/* Now install the token source interface
*/
if (lexer->rec->state->tokSource == NULL)
{
lexer->rec->state->tokSource = (pANTLR3_TOKEN_SOURCE)ANTLR3_CALLOC(1, sizeof(ANTLR3_TOKEN_SOURCE));
if (lexer->rec->state->tokSource == NULL)
{
lexer->rec->free(lexer->rec);
lexer->free(lexer);
return NULL;
}
lexer->rec->state->tokSource->super = lexer;
/* Install the default nextToken() method, which may be overridden
* by generated code, or by anything else in fact.
*/
lexer->rec->state->tokSource->nextToken = nextToken;
lexer->rec->state->tokSource->strFactory = NULL;
lexer->rec->state->tokFactory = NULL;
}
/* Install the lexer API
*/
lexer->setCharStream = setCharStream;
lexer->mTokens = (void (*)(void *))(mTokens);
lexer->setCharStream = setCharStream;
lexer->pushCharStream = pushCharStream;
lexer->popCharStream = popCharStream;
lexer->emit = emit;
lexer->emitNew = emitNew;
lexer->matchs = matchs;
lexer->matchc = matchc;
lexer->matchRange = matchRange;
lexer->matchAny = matchAny;
lexer->recover = recover;
lexer->getLine = getLine;
lexer->getCharIndex = getCharIndex;
lexer->getCharPositionInLine = getCharPositionInLine;
lexer->getText = getText;
lexer->free = freeLexer;
/* Initialise the eof token
*/
specialT = &(lexer->rec->state->tokSource->eofToken);
antlr3SetTokenAPI (specialT);
specialT->setType (specialT, ANTLR3_TOKEN_EOF);
specialT->factoryMade = ANTLR3_TRUE; // Prevent things trying to free() it
specialT->strFactory = NULL;
specialT->textState = ANTLR3_TEXT_NONE;
specialT->custom = NULL;
specialT->user1 = 0;
specialT->user2 = 0;
specialT->user3 = 0;
// Initialize the skip token.
//
specialT = &(lexer->rec->state->tokSource->skipToken);
antlr3SetTokenAPI (specialT);
specialT->setType (specialT, ANTLR3_TOKEN_INVALID);
specialT->factoryMade = ANTLR3_TRUE; // Prevent things trying to free() it
specialT->strFactory = NULL;
specialT->custom = NULL;
specialT->user1 = 0;
specialT->user2 = 0;
specialT->user3 = 0;
return lexer;
}
static void
reset (pANTLR3_BASE_RECOGNIZER rec)
{
pANTLR3_LEXER lexer;
lexer = rec->super;
lexer->rec->state->token = NULL;
lexer->rec->state->type = ANTLR3_TOKEN_INVALID;
lexer->rec->state->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
lexer->rec->state->tokenStartCharIndex = -1;
lexer->rec->state->tokenStartCharPositionInLine = -1;
lexer->rec->state->tokenStartLine = -1;
lexer->rec->state->text = NULL;
// OK - that's all hunky dory, but we may well have had
// a token factory that needs a reset. Do that here
//
if (lexer->rec->state->tokFactory != NULL)
{
lexer->rec->state->tokFactory->reset(lexer->rec->state->tokFactory);
}
}
///
/// \brief
/// Returns the next available token from the current input stream.
///
/// \param toksource
/// Points to the implementation of a token source. The lexer is
/// addressed by the super structure pointer.
///
/// \returns
/// The next token in the current input stream or the EOF token
/// if there are no more tokens.
///
/// \remarks
/// Write remarks for nextToken here.
///
/// \see nextToken
///
ANTLR3_INLINE static pANTLR3_COMMON_TOKEN
nextTokenStr (pANTLR3_TOKEN_SOURCE toksource)
{
pANTLR3_LEXER lexer;
pANTLR3_RECOGNIZER_SHARED_STATE state;
pANTLR3_INPUT_STREAM input;
pANTLR3_INT_STREAM istream;
lexer = (pANTLR3_LEXER)(toksource->super);
state = lexer->rec->state;
input = lexer->input;
istream = input->istream;
/// Loop until we get a non skipped token or EOF
///
for (;;)
{
// Get rid of any previous token (token factory takes care of
// any de-allocation when this token is finally used up.
//
state->token = NULL;
state->error = ANTLR3_FALSE; // Start out without an exception
state->failed = ANTLR3_FALSE;
// Now call the matching rules and see if we can generate a new token
//
for (;;)
{
// Record the start of the token in our input stream.
//
state->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
state->tokenStartCharIndex = (ANTLR3_MARKER)(((pANTLR3_UINT8)input->nextChar));
state->tokenStartCharPositionInLine = input->charPositionInLine;
state->tokenStartLine = input->line;
state->text = NULL;
state->custom = NULL;
state->user1 = 0;
state->user2 = 0;
state->user3 = 0;
if (istream->_LA(istream, 1) == ANTLR3_CHARSTREAM_EOF)
{
// Reached the end of the current stream, nothing more to do if this is
// the last in the stack.
//
pANTLR3_COMMON_TOKEN teof = &(toksource->eofToken);
teof->setStartIndex (teof, lexer->getCharIndex(lexer));
teof->setStopIndex (teof, lexer->getCharIndex(lexer));
teof->setLine (teof, lexer->getLine(lexer));
teof->factoryMade = ANTLR3_TRUE; // This isn't really manufactured but it stops things from trying to free it
return teof;
}
state->token = NULL;
state->error = ANTLR3_FALSE; // Start out without an exception
state->failed = ANTLR3_FALSE;
// Call the generated lexer, see if it can get a new token together.
//
lexer->mTokens(lexer->ctx);
if (state->error == ANTLR3_TRUE)
{
// Recognition exception, report it and try to recover.
//
state->failed = ANTLR3_TRUE;
lexer->rec->reportError(lexer->rec);
lexer->recover(lexer);
}
else
{
if (state->token == NULL)
{
// Emit the real token, which adds it in to the token stream basically
//
emit(lexer);
}
else if (state->token == &(toksource->skipToken))
{
// A real token could have been generated, but "Computer say's naaaaah" and it
// it is just something we need to skip altogether.
//
continue;
}
// Good token, not skipped, not EOF token
//
return state->token;
}
}
}
}
/**
* \brief
* Default implementation of the nextToken() call for a lexer.
*
* \param toksource
* Points to the implementation of a token source. The lexer is
* addressed by the super structure pointer.
*
* \returns
* The next token in the current input stream or the EOF token
* if there are no more tokens in any input stream in the stack.
*
* Write detailed description for nextToken here.
*
* \remarks
* Write remarks for nextToken here.
*
* \see nextTokenStr
*/
static pANTLR3_COMMON_TOKEN
nextToken (pANTLR3_TOKEN_SOURCE toksource)
{
pANTLR3_COMMON_TOKEN tok;
// Find the next token in the current stream
//
tok = nextTokenStr(toksource);
// If we got to the EOF token then switch to the previous
// input stream if there were any and just return the
// EOF if there are none. We must check the next token
// in any outstanding input stream we pop into the active
// role to see if it was sitting at EOF after PUSHing the
// stream we just consumed, otherwise we will return EOF
// on the reinstalled input stream, when in actual fact
// there might be more input streams to POP before the
// real EOF of the whole logical inptu stream. Hence we
// use a while loop here until we find somethign in the stream
// that isn't EOF or we reach the actual end of the last input
// stream on the stack.
//
while (tok->type == ANTLR3_TOKEN_EOF)
{
pANTLR3_LEXER lexer;
lexer = (pANTLR3_LEXER)(toksource->super);
if (lexer->rec->state->streams != NULL && lexer->rec->state->streams->size(lexer->rec->state->streams) > 0)
{
// We have another input stream in the stack so we
// need to revert to it, then resume the loop to check
// it wasn't sitting at EOF itself.
//
lexer->popCharStream(lexer);
tok = nextTokenStr(toksource);
}
else
{
// There were no more streams on the input stack
// so this EOF is the 'real' logical EOF for
// the input stream. So we just exit the loop and
// return the EOF we have found.
//
break;
}
}
// return whatever token we have, which may be EOF
//
return tok;
}
ANTLR3_API pANTLR3_LEXER
antlr3LexerNewStream(ANTLR3_UINT32 sizeHint, pANTLR3_INPUT_STREAM input, pANTLR3_RECOGNIZER_SHARED_STATE state)
{
pANTLR3_LEXER lexer;
// Create a basic lexer first
//
lexer = antlr3LexerNew(sizeHint, state);
if (lexer != NULL)
{
// Install the input stream and reset the lexer
//
setCharStream(lexer, input);
}
return lexer;
}
static void mTokens (pANTLR3_LEXER lexer)
{
if (lexer) // Fool compiler, avoid pragmas
{
ANTLR3_FPRINTF(stderr, "lexer->mTokens(): Error: No lexer rules were added to the lexer yet!\n");
}
}
static void
reportError (pANTLR3_BASE_RECOGNIZER rec)
{
// Indicate this recognizer had an error while processing.
//
rec->state->errorCount++;
rec->displayRecognitionError(rec, rec->state->tokenNames);
}
#ifdef ANTLR3_WINDOWS
#pragma warning( disable : 4100 )
#endif
/** Default lexer error handler (works for 8 bit streams only!!!)
*/
static void
displayRecognitionError (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_UINT8 * tokenNames)
{
pANTLR3_LEXER lexer;
pANTLR3_EXCEPTION ex;
pANTLR3_STRING ftext;
lexer = (pANTLR3_LEXER)(recognizer->super);
ex = lexer->rec->state->exception;
// See if there is a 'filename' we can use
//
if (ex->name == NULL)
{
ANTLR3_FPRINTF(stderr, "-unknown source-(");
}
else
{
ftext = ex->streamName->to8(ex->streamName);
ANTLR3_FPRINTF(stderr, "%s(", ftext->chars);
}
ANTLR3_FPRINTF(stderr, "%d) ", recognizer->state->exception->line);
ANTLR3_FPRINTF(stderr, ": lexer error %d :\n\t%s at offset %d, ",
ex->type,
(pANTLR3_UINT8) (ex->message),
ex->charPositionInLine+1
);
{
ANTLR3_INT32 width;
width = ANTLR3_UINT32_CAST(( (pANTLR3_UINT8)(lexer->input->data) + (lexer->input->size(lexer->input) )) - (pANTLR3_UINT8)(ex->index));
if (width >= 1)
{
if (isprint(ex->c))
{
ANTLR3_FPRINTF(stderr, "near '%c' :\n", ex->c);
}
else
{
ANTLR3_FPRINTF(stderr, "near char(%#02X) :\n", (ANTLR3_UINT8)(ex->c));
}
ANTLR3_FPRINTF(stderr, "\t%.*s\n", width > 20 ? 20 : width ,((pANTLR3_UINT8)ex->index));
}
else
{
ANTLR3_FPRINTF(stderr, "(end of input).\n\t This indicates a poorly specified lexer RULE\n\t or unterminated input element such as: \"STRING[\"]\n");
ANTLR3_FPRINTF(stderr, "\t The lexer was matching from line %d, offset %d, which\n\t ",
(ANTLR3_UINT32)(lexer->rec->state->tokenStartLine),
(ANTLR3_UINT32)(lexer->rec->state->tokenStartCharPositionInLine)
);
width = ANTLR3_UINT32_CAST(((pANTLR3_UINT8)(lexer->input->data)+(lexer->input->size(lexer->input))) - (pANTLR3_UINT8)(lexer->rec->state->tokenStartCharIndex));
if (width >= 1)
{
ANTLR3_FPRINTF(stderr, "looks like this:\n\t\t%.*s\n", width > 20 ? 20 : width ,(pANTLR3_UINT8)(lexer->rec->state->tokenStartCharIndex));
}
else
{
ANTLR3_FPRINTF(stderr, "is also the end of the line, so you must check your lexer rules\n");
}
}
}
}
static void setCharStream (pANTLR3_LEXER lexer, pANTLR3_INPUT_STREAM input)
{
/* Install the input interface
*/
lexer->input = input;
/* We may need a token factory for the lexer; we don't destroy any existing factory
* until the lexer is destroyed, as people may still be using the tokens it produced.
* TODO: Later I will provide a dup() method for a token so that it can extract itself
* out of the factory.
*/
if (lexer->rec->state->tokFactory == NULL)
{
lexer->rec->state->tokFactory = antlr3TokenFactoryNew(input);
}
else
{
/* When the input stream is being changed on the fly, rather than
* at the start of a new lexer, then we must tell the tokenFactory
* which input stream to adorn the tokens with so that when they
* are asked to provide their original input strings they can
* do so from the correct text stream.
*/
lexer->rec->state->tokFactory->setInputStream(lexer->rec->state->tokFactory, input);
}
/* Propagate the string factory so that we preserve the encoding form from
* the input stream.
*/
if (lexer->rec->state->tokSource->strFactory == NULL)
{
lexer->rec->state->tokSource->strFactory = input->strFactory;
// Set the newly acquired string factory up for our pre-made tokens
// for EOF.
//
if (lexer->rec->state->tokSource->eofToken.strFactory == NULL)
{
lexer->rec->state->tokSource->eofToken.strFactory = input->strFactory;
}
}
/* This is a lexer, install the appropriate exception creator
*/
lexer->rec->exConstruct = antlr3RecognitionExceptionNew;
/* Set the current token to nothing
*/
lexer->rec->state->token = NULL;
lexer->rec->state->text = NULL;
lexer->rec->state->tokenStartCharIndex = -1;
/* Copy the name of the char stream to the token source
*/
lexer->rec->state->tokSource->fileName = input->fileName;
}
/*!
* \brief
* Change to a new input stream, remembering the old one.
*
* \param lexer
* Pointer to the lexer instance to switch input streams for.
*
* \param input
* New input stream to install as the current one.
*
* Switches the current character input stream to
* a new one, saving the old one, which we will revert to at the end of this
* new one.
*/
static void
pushCharStream (pANTLR3_LEXER lexer, pANTLR3_INPUT_STREAM input)
{
// Do we need a new input stream stack?
//
if (lexer->rec->state->streams == NULL)
{
// This is the first call to stack a new
// stream and so we must create the stack first.
//
lexer->rec->state->streams = antlr3StackNew(0);
if (lexer->rec->state->streams == NULL)
{
// Could not do this, we just fail to push it.
// TODO: Consider if this is what we want to do, but then
// any programmer can override this method to do something else.
return;
}
}
// We have a stack, so we can save the current input stream
// into it.
//
lexer->input->istream->mark(lexer->input->istream);
lexer->rec->state->streams->push(lexer->rec->state->streams, lexer->input, NULL);
// And now we can install this new one
//
lexer->setCharStream(lexer, input);
}
/*!
* \brief
* Stops using the current input stream and reverts to any prior
* input stream on the stack.
*
* \param lexer
* Description of parameter lexer.
*
* Pointer to a function that abandons the current input stream, whether it
* is empty or not and reverts to the previous stacked input stream.
*
* \remark
* The function fails silently if there are no prior input streams.
*/
static void
popCharStream (pANTLR3_LEXER lexer)
{
pANTLR3_INPUT_STREAM input;
// If we do not have a stream stack or we are already at the
// stack bottom, then do nothing.
//
if (lexer->rec->state->streams != NULL && lexer->rec->state->streams->size(lexer->rec->state->streams) > 0)
{
// We just leave the current stream to its fate, we do not close
// it or anything as we do not know what the programmer intended
// for it. This method can always be overridden of course.
// So just find out what was currently saved on the stack and use
// that now, then pop it from the stack.
//
input = (pANTLR3_INPUT_STREAM)(lexer->rec->state->streams->top);
lexer->rec->state->streams->pop(lexer->rec->state->streams);
// Now install the stream as the current one.
//
lexer->setCharStream(lexer, input);
lexer->input->istream->rewindLast(lexer->input->istream);
}
return;
}
static void emitNew (pANTLR3_LEXER lexer, pANTLR3_COMMON_TOKEN token)
{
lexer->rec->state->token = token; /* Voila! */
}
static pANTLR3_COMMON_TOKEN
emit (pANTLR3_LEXER lexer)
{
pANTLR3_COMMON_TOKEN token;
/* We could check pointers to token factories and so on, but
* we are in code that we want to run as fast as possible
* so we are not checking any errors. So make sure you have installed an input stream before
* trying to emit a new token.
*/
token = lexer->rec->state->tokFactory->newToken(lexer->rec->state->tokFactory);
/* Install the supplied information, and some other bits we already know
* get added automatically, such as the input stream it is associated with
* (though it can all be overridden of course)
*/
token->type = lexer->rec->state->type;
token->channel = lexer->rec->state->channel;
token->start = lexer->rec->state->tokenStartCharIndex;
token->stop = lexer->getCharIndex(lexer) - 1;
token->line = lexer->rec->state->tokenStartLine;
token->charPosition = lexer->rec->state->tokenStartCharPositionInLine;
if (lexer->rec->state->text != NULL)
{
token->textState = ANTLR3_TEXT_STRING;
token->tokText.text = lexer->rec->state->text;
}
else
{
token->textState = ANTLR3_TEXT_NONE;
}
token->lineStart = lexer->input->currentLine;
token->user1 = lexer->rec->state->user1;
token->user2 = lexer->rec->state->user2;
token->user3 = lexer->rec->state->user3;
token->custom = lexer->rec->state->custom;
lexer->rec->state->token = token;
return token;
}
/**
* Free the resources allocated by a lexer
*/
static void
freeLexer (pANTLR3_LEXER lexer)
{
// This may have ben a delegate or delegator lexer, in which case the
// state may already have been freed (and set to NULL therefore)
// so we ignore the state if we don't have it.
//
if (lexer->rec->state != NULL)
{
if (lexer->rec->state->streams != NULL)
{
lexer->rec->state->streams->free(lexer->rec->state->streams);
}
if (lexer->rec->state->tokFactory != NULL)
{
lexer->rec->state->tokFactory->close(lexer->rec->state->tokFactory);
lexer->rec->state->tokFactory = NULL;
}
if (lexer->rec->state->tokSource != NULL)
{
ANTLR3_FREE(lexer->rec->state->tokSource);
lexer->rec->state->tokSource = NULL;
}
}
if (lexer->rec != NULL)
{
lexer->rec->free(lexer->rec);
lexer->rec = NULL;
}
ANTLR3_FREE(lexer);
}
/** Implementation of matchs for the lexer, overrides any
* base implementation in the base recognizer.
*
* \remark
* Note that the generated code lays down arrays of ints for constant
* strings so that they are int UTF32 form!
*/
static ANTLR3_BOOLEAN
matchs(pANTLR3_LEXER lexer, ANTLR3_UCHAR * string)
{
while (*string != ANTLR3_STRING_TERMINATOR)
{
if (lexer->input->istream->_LA(lexer->input->istream, 1) != (*string))
{
if (lexer->rec->state->backtracking > 0)
{
lexer->rec->state->failed = ANTLR3_TRUE;
return ANTLR3_FALSE;
}
lexer->rec->exConstruct(lexer->rec);
lexer->rec->state->failed = ANTLR3_TRUE;
/* TODO: Implement exception creation more fully perhaps
*/
lexer->recover(lexer);
return ANTLR3_FALSE;
}
/* Matched correctly, do consume it
*/
lexer->input->istream->consume(lexer->input->istream);
string++;
/* Reset any failed indicator
*/
lexer->rec->state->failed = ANTLR3_FALSE;
}
return ANTLR3_TRUE;
}
/** Implementation of matchc for the lexer, overrides any
* base implementation in the base recognizer.
*
* \remark
* Note that the generated code lays down arrays of ints for constant
* strings so that they are int UTF32 form!
*/
static ANTLR3_BOOLEAN
matchc(pANTLR3_LEXER lexer, ANTLR3_UCHAR c)
{
if (lexer->input->istream->_LA(lexer->input->istream, 1) == c)
{
/* Matched correctly, do consume it
*/
lexer->input->istream->consume(lexer->input->istream);
/* Reset any failed indicator
*/
lexer->rec->state->failed = ANTLR3_FALSE;
return ANTLR3_TRUE;
}
/* Failed to match, exception and recovery time.
*/
if (lexer->rec->state->backtracking > 0)
{
lexer->rec->state->failed = ANTLR3_TRUE;
return ANTLR3_FALSE;
}
lexer->rec->exConstruct(lexer->rec);
/* TODO: Implement exception creation more fully perhaps
*/
lexer->recover(lexer);
return ANTLR3_FALSE;
}
/** Implementation of match range for the lexer, overrides any
* base implementation in the base recognizer.
*
* \remark
* Note that the generated code lays down arrays of ints for constant
* strings so that they are int UTF32 form!
*/
static ANTLR3_BOOLEAN
matchRange(pANTLR3_LEXER lexer, ANTLR3_UCHAR low, ANTLR3_UCHAR high)
{
ANTLR3_UCHAR c;
/* What is in the stream at the moment?
*/
c = lexer->input->istream->_LA(lexer->input->istream, 1);
if ( c >= low && c <= high)
{
/* Matched correctly, consume it
*/
lexer->input->istream->consume(lexer->input->istream);
/* Reset any failed indicator
*/
lexer->rec->state->failed = ANTLR3_FALSE;
return ANTLR3_TRUE;
}
/* Failed to match, execption and recovery time.
*/
if (lexer->rec->state->backtracking > 0)
{
lexer->rec->state->failed = ANTLR3_TRUE;
return ANTLR3_FALSE;
}
lexer->rec->exConstruct(lexer->rec);
/* TODO: Implement exception creation more fully
*/
lexer->recover(lexer);
return ANTLR3_FALSE;
}
static void
matchAny (pANTLR3_LEXER lexer)
{
lexer->input->istream->consume(lexer->input->istream);
}
static void
recover (pANTLR3_LEXER lexer)
{
lexer->input->istream->consume(lexer->input->istream);
}
static ANTLR3_UINT32
getLine (pANTLR3_LEXER lexer)
{
return lexer->input->getLine(lexer->input);
}
static ANTLR3_UINT32
getCharPositionInLine (pANTLR3_LEXER lexer)
{
return lexer->input->charPositionInLine;
}
static ANTLR3_MARKER getCharIndex (pANTLR3_LEXER lexer)
{
return lexer->input->istream->index(lexer->input->istream);
}
static pANTLR3_STRING
getText (pANTLR3_LEXER lexer)
{
if (lexer->rec->state->text)
{
return lexer->rec->state->text;
}
return lexer->input->substr(
lexer->input,
lexer->rec->state->tokenStartCharIndex,
lexer->getCharIndex(lexer) - lexer->input->charByteSize
);
}
static void *
getCurrentInputSymbol (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM istream)
{
return NULL;
}
static void *
getMissingSymbol (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM istream, pANTLR3_EXCEPTION e,
ANTLR3_UINT32 expectedTokenType, pANTLR3_BITSET_LIST follow)
{
return NULL;
}