/// \file
/// Default implementation of CommonTokenStream
///
// [The "BSD licence"]
// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
// http://www.temporal-wave.com
// http://www.linkedin.com/in/jimidle
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <antlr3tokenstream.h>
#ifdef ANTLR3_WINDOWS
#pragma warning( disable : 4100 )
#endif
// COMMON_TOKEN_STREAM API
//
static void setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel);
static void discardTokenType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype);
static void discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard);
static pANTLR3_VECTOR getTokens (pANTLR3_COMMON_TOKEN_STREAM cts);
static pANTLR3_LIST getTokenRange (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
static pANTLR3_LIST getTokensSet (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types);
static pANTLR3_LIST getTokensList (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list);
static pANTLR3_LIST getTokensType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type);
static void reset (pANTLR3_COMMON_TOKEN_STREAM cts);
// TOKEN_STREAM API
//
static pANTLR3_COMMON_TOKEN tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
static pANTLR3_COMMON_TOKEN dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
static pANTLR3_COMMON_TOKEN get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i);
static pANTLR3_TOKEN_SOURCE getTokenSource (pANTLR3_TOKEN_STREAM ts);
static void setTokenSource (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource);
static pANTLR3_STRING toString (pANTLR3_TOKEN_STREAM ts);
static pANTLR3_STRING toStringSS (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
static pANTLR3_STRING toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop);
static void setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger);
// INT STREAM API
//
static void consume (pANTLR3_INT_STREAM is);
static void dbgConsume (pANTLR3_INT_STREAM is);
static ANTLR3_UINT32 _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
static ANTLR3_UINT32 dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
static ANTLR3_MARKER mark (pANTLR3_INT_STREAM is);
static ANTLR3_MARKER dbgMark (pANTLR3_INT_STREAM is);
static void release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark);
static ANTLR3_UINT32 size (pANTLR3_INT_STREAM is);
static ANTLR3_MARKER tindex (pANTLR3_INT_STREAM is);
static void rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
static void dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
static void rewindLast (pANTLR3_INT_STREAM is);
static void dbgRewindLast (pANTLR3_INT_STREAM is);
static void seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
static void dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
static pANTLR3_STRING getSourceName (pANTLR3_INT_STREAM is);
static void antlr3TokenStreamFree (pANTLR3_TOKEN_STREAM stream);
static void antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream);
// Helpers
//
static void fillBuffer (pANTLR3_COMMON_TOKEN_STREAM tokenStream);
static ANTLR3_UINT32 skipOffTokenChannels (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
static ANTLR3_UINT32 skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
static pANTLR3_COMMON_TOKEN LB (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
ANTLR3_API pANTLR3_TOKEN_STREAM
antlr3TokenStreamNew()
{
pANTLR3_TOKEN_STREAM stream;
// Memory for the interface structure
//
stream = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM));
if (stream == NULL)
{
return NULL;
}
// Install basic API
//
stream->free = antlr3TokenStreamFree;
return stream;
}
static void
antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream)
{
ANTLR3_FREE(stream);
}
static void
antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream)
{
// We only free up our subordinate interfaces if they belong
// to us, otherwise we let whoever owns them deal with them.
//
if (stream->tstream->super == stream)
{
if (stream->tstream->istream->super == stream->tstream)
{
stream->tstream->istream->free(stream->tstream->istream);
stream->tstream->istream = NULL;
}
stream->tstream->free(stream->tstream);
}
// Now we free our own resources
//
if (stream->tokens != NULL)
{
stream->tokens->free(stream->tokens);
stream->tokens = NULL;
}
if (stream->discardSet != NULL)
{
stream->discardSet->free(stream->discardSet);
stream->discardSet = NULL;
}
if (stream->channelOverrides != NULL)
{
stream->channelOverrides->free(stream->channelOverrides);
stream->channelOverrides = NULL;
}
// Free our memory now
//
ANTLR3_FREE(stream);
}
// Reset a token stream so it can be used again and can reuse it's
// resources.
//
static void
reset (pANTLR3_COMMON_TOKEN_STREAM cts)
{
// Free any resources that ar most like specifc to the
// run we just did.
//
if (cts->discardSet != NULL)
{
cts->discardSet->free(cts->discardSet);
cts->discardSet = NULL;
}
if (cts->channelOverrides != NULL)
{
cts->channelOverrides->free(cts->channelOverrides);
cts->channelOverrides = NULL;
}
// Now, if there were any existing tokens in the stream,
// then we just reset the vector count so that it starts
// again. We must traverse the entries unfortunately as
// there may be free pointers for custom token types and
// so on. However that is just a quick NULL check on the
// vector entries.
//
if (cts->tokens != NULL)
{
cts->tokens->clear(cts->tokens);
}
else
{
/* Install the token tracking tables
*/
cts->tokens = antlr3VectorNew(0);
}
// Reset to defaults
//
cts->discardOffChannel = ANTLR3_FALSE;
cts->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
cts->p = -1;
}
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger)
{
pANTLR3_COMMON_TOKEN_STREAM stream;
// Create a standard token stream
//
stream = antlr3CommonTokenStreamSourceNew(hint, source);
// Install the debugger object
//
stream->tstream->debugger = debugger;
// Override standard token stream methods with debugging versions
//
stream->tstream->initialStreamState = ANTLR3_FALSE;
stream->tstream->_LT = dbgTokLT;
stream->tstream->istream->consume = dbgConsume;
stream->tstream->istream->_LA = dbgLA;
stream->tstream->istream->mark = dbgMark;
stream->tstream->istream->rewind = dbgRewindStream;
stream->tstream->istream->rewindLast = dbgRewindLast;
stream->tstream->istream->seek = dbgSeek;
return stream;
}
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source)
{
pANTLR3_COMMON_TOKEN_STREAM stream;
stream = antlr3CommonTokenStreamNew(hint);
stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
stream->channelOverrides = NULL;
stream->discardSet = NULL;
stream->discardOffChannel = ANTLR3_FALSE;
stream->tstream->setTokenSource(stream->tstream, source);
stream->free = antlr3CTSFree;
return stream;
}
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint)
{
pANTLR3_COMMON_TOKEN_STREAM stream;
/* Memory for the interface structure
*/
stream = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM));
if (stream == NULL)
{
return NULL;
}
/* Create space for the token stream interface
*/
stream->tstream = antlr3TokenStreamNew();
stream->tstream->super = stream;
/* Create space for the INT_STREAM interfacce
*/
stream->tstream->istream = antlr3IntStreamNew();
stream->tstream->istream->super = (stream->tstream);
stream->tstream->istream->type = ANTLR3_TOKENSTREAM;
/* Install the token tracking tables
*/
stream->tokens = antlr3VectorNew(0);
/* Defaults
*/
stream->p = -1;
/* Install the common token stream API
*/
stream->setTokenTypeChannel = setTokenTypeChannel;
stream->discardTokenType = discardTokenType;
stream->discardOffChannelToks = discardOffChannel;
stream->getTokens = getTokens;
stream->getTokenRange = getTokenRange;
stream->getTokensSet = getTokensSet;
stream->getTokensList = getTokensList;
stream->getTokensType = getTokensType;
stream->reset = reset;
/* Install the token stream API
*/
stream->tstream->_LT = tokLT;
stream->tstream->get = get;
stream->tstream->getTokenSource = getTokenSource;
stream->tstream->setTokenSource = setTokenSource;
stream->tstream->toString = toString;
stream->tstream->toStringSS = toStringSS;
stream->tstream->toStringTT = toStringTT;
stream->tstream->setDebugListener = setDebugListener;
/* Install INT_STREAM interface
*/
stream->tstream->istream->_LA = _LA;
stream->tstream->istream->mark = mark;
stream->tstream->istream->release = release;
stream->tstream->istream->size = size;
stream->tstream->istream->index = tindex;
stream->tstream->istream->rewind = rewindStream;
stream->tstream->istream->rewindLast= rewindLast;
stream->tstream->istream->seek = seek;
stream->tstream->istream->consume = consume;
stream->tstream->istream->getSourceName = getSourceName;
return stream;
}
// Install a debug listener adn switch to debug mode methods
//
static void
setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger)
{
// Install the debugger object
//
ts->debugger = debugger;
// Override standard token stream methods with debugging versions
//
ts->initialStreamState = ANTLR3_FALSE;
ts->_LT = dbgTokLT;
ts->istream->consume = dbgConsume;
ts->istream->_LA = dbgLA;
ts->istream->mark = dbgMark;
ts->istream->rewind = dbgRewindStream;
ts->istream->rewindLast = dbgRewindLast;
ts->istream->seek = dbgSeek;
}
/** Get the ith token from the current position 1..n where k=1 is the
* first symbol of lookahead.
*/
static pANTLR3_COMMON_TOKEN
tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
{
ANTLR3_INT32 i;
ANTLR3_INT32 n;
pANTLR3_COMMON_TOKEN_STREAM cts;
cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
if (k < 0)
{
return LB(cts, -k);
}
if (cts->p == -1)
{
fillBuffer(cts);
}
// Here we used to check for k == 0 and return 0, but this seems
// a superfluous check to me. LT(k=0) is therefore just undefined
// and we won't waste the clock cycles on the check
//
if ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize)
{
pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken);
teof->setStartIndex (teof, ts->istream->index (ts->istream));
teof->setStopIndex (teof, ts->istream->index (ts->istream));
return teof;
}
i = cts->p;
n = 1;
/* Need to find k good tokens, skipping ones that are off channel
*/
while ( n < k)
{
/* Skip off-channel tokens */
i = skipOffTokenChannels(cts, i+1); /* leave p on valid token */
n++;
}
if ( (ANTLR3_UINT32) i >= ts->istream->cachedSize)
{
pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken);
teof->setStartIndex (teof, ts->istream->index(ts->istream));
teof->setStopIndex (teof, ts->istream->index(ts->istream));
return teof;
}
// Here the token must be in the input vector. Rather then incur
// function call penalty, we just return the pointer directly
// from the vector
//
return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
//return (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i);
}
/// Debug only method to flag consumption of initial off-channel
/// tokens in the input stream
///
static void
consumeInitialHiddenTokens(pANTLR3_INT_STREAM is)
{
ANTLR3_MARKER first;
ANTLR3_INT32 i;
pANTLR3_TOKEN_STREAM ts;
ts = (pANTLR3_TOKEN_STREAM) is->super;
first = is->index(is);
for (i=0; i<first; i++)
{
ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i));
}
ts->initialStreamState = ANTLR3_FALSE;
}
/// As per the normal tokLT but sends information to the debugger
///
static pANTLR3_COMMON_TOKEN
dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
{
if (ts->initialStreamState == ANTLR3_TRUE)
{
consumeInitialHiddenTokens(ts->istream);
}
return tokLT(ts, k);
}
#ifdef ANTLR3_WINDOWS
/* When fully optimized VC7 complains about non reachable code.
* Not yet sure if this is an optimizer bug, or a bug in the flow analysis
*/
#pragma warning( disable : 4702 )
#endif
static pANTLR3_COMMON_TOKEN
LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k)
{
ANTLR3_INT32 i;
ANTLR3_INT32 n;
if (cts->p == -1)
{
fillBuffer(cts);
}
if (k == 0)
{
return NULL;
}
if ((cts->p - k) < 0)
{
return NULL;
}
i = cts->p;
n = 1;
/* Need to find k good tokens, going backwards, skipping ones that are off channel
*/
while (n <= (ANTLR3_INT32) k)
{
/* Skip off-channel tokens
*/
i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token */
n++;
}
if (i < 0)
{
return NULL;
}
// Here the token must be in the input vector. Rather then incut
// function call penalty, we jsut return the pointer directly
// from the vector
//
return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
}
static pANTLR3_COMMON_TOKEN
get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i)
{
pANTLR3_COMMON_TOKEN_STREAM cts;
cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
return (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i)); /* Token index is zero based but vectors are 1 based */
}
static pANTLR3_TOKEN_SOURCE
getTokenSource (pANTLR3_TOKEN_STREAM ts)
{
return ts->tokenSource;
}
static void
setTokenSource ( pANTLR3_TOKEN_STREAM ts,
pANTLR3_TOKEN_SOURCE tokenSource)
{
ts->tokenSource = tokenSource;
}
static pANTLR3_STRING
toString (pANTLR3_TOKEN_STREAM ts)
{
pANTLR3_COMMON_TOKEN_STREAM cts;
cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
if (cts->p == -1)
{
fillBuffer(cts);
}
return ts->toStringSS(ts, 0, ts->istream->size(ts->istream));
}
static pANTLR3_STRING
toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
{
pANTLR3_STRING string;
pANTLR3_TOKEN_SOURCE tsource;
pANTLR3_COMMON_TOKEN tok;
ANTLR3_UINT32 i;
pANTLR3_COMMON_TOKEN_STREAM cts;
cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
if (cts->p == -1)
{
fillBuffer(cts);
}
if (stop >= ts->istream->size(ts->istream))
{
stop = ts->istream->size(ts->istream) - 1;
}
/* Who is giving us these tokens?
*/
tsource = ts->getTokenSource(ts);
if (tsource != NULL && cts->tokens != NULL)
{
/* Finally, let's get a string
*/
string = tsource->strFactory->newRaw(tsource->strFactory);
for (i = start; i <= stop; i++)
{
tok = ts->get(ts, i);
if (tok != NULL)
{
string->appendS(string, tok->getText(tok));
}
}
return string;
}
return NULL;
}
static pANTLR3_STRING
toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop)
{
if (start != NULL && stop != NULL)
{
return ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop));
}
else
{
return NULL;
}
}
/** Move the input pointer to the next incoming token. The stream
* must become active with LT(1) available. consume() simply
* moves the input pointer so that LT(1) points at the next
* input symbol. Consume at least one token.
*
* Walk past any token not on the channel the parser is listening to.
*/
static void
consume (pANTLR3_INT_STREAM is)
{
pANTLR3_COMMON_TOKEN_STREAM cts;
pANTLR3_TOKEN_STREAM ts;
ts = (pANTLR3_TOKEN_STREAM) is->super;
cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
if ((ANTLR3_UINT32)cts->p < cts->tokens->count)
{
cts->p++;
cts->p = skipOffTokenChannels(cts, cts->p);
}
}
/// As per ordinary consume but notifies the debugger about hidden
/// tokens and so on.
///
static void
dbgConsume (pANTLR3_INT_STREAM is)
{
pANTLR3_TOKEN_STREAM ts;
ANTLR3_MARKER a;
ANTLR3_MARKER b;
pANTLR3_COMMON_TOKEN t;
ts = (pANTLR3_TOKEN_STREAM) is->super;
if (ts->initialStreamState == ANTLR3_TRUE)
{
consumeInitialHiddenTokens(is);
}
a = is->index(is); // Where are we right now?
t = ts->_LT(ts, 1); // Current token from stream
consume(is); // Standard consumer
b = is->index(is); // Where are we after consuming 1 on channel token?
ts->debugger->consumeToken(ts->debugger, t); // Tell the debugger that we consumed the first token
if (b>a+1)
{
// The standard consume caused the index to advance by more than 1,
// which can only happen if it skipped some off-channel tokens.
// we need to tell the debugger about those tokens.
//
ANTLR3_MARKER i;
for (i = a+1; i<b; i++)
{
ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i));
}
}
}
/** A simple filter mechanism whereby you can tell this token stream
* to force all tokens of type ttype to be on channel. For example,
* when interpreting, we cannot execute actions so we need to tell
* the stream to force all WS and NEWLINE to be a different, ignored,
* channel.
*/
static void
setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel)
{
if (tokenStream->channelOverrides == NULL)
{
tokenStream->channelOverrides = antlr3ListNew(10);
}
/* We add one to the channel so we can distinguish NULL as being no entry in the
* table for a particular token type.
*/
tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL);
}
static void
discardTokenType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype)
{
if (tokenStream->discardSet == NULL)
{
tokenStream->discardSet = antlr3ListNew(31);
}
/* We add one to the channel so we can distinguish NULL as being no entry in the
* table for a particular token type. We could use bitsets for this I suppose too.
*/
tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL);
}
static void
discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard)
{
tokenStream->discardOffChannel = discard;
}
static pANTLR3_VECTOR
getTokens (pANTLR3_COMMON_TOKEN_STREAM tokenStream)
{
if (tokenStream->p == -1)
{
fillBuffer(tokenStream);
}
return tokenStream->tokens;
}
static pANTLR3_LIST
getTokenRange (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
{
return tokenStream->getTokensSet(tokenStream, start, stop, NULL);
}
/** Given a start and stop index, return a List of all tokens in
* the token type BitSet. Return null if no tokens were found. This
* method looks at both on and off channel tokens.
*/
static pANTLR3_LIST
getTokensSet (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types)
{
pANTLR3_LIST filteredList;
ANTLR3_UINT32 i;
ANTLR3_UINT32 n;
pANTLR3_COMMON_TOKEN tok;
if (tokenStream->p == -1)
{
fillBuffer(tokenStream);
}
if (stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream))
{
stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream);
}
if (start > stop)
{
return NULL;
}
/* We have the range set, now we need to iterate through the
* installed tokens and create a new list with just the ones we want
* in it. We are just moving pointers about really.
*/
filteredList = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream));
for (i = start, n = 0; i<= stop; i++)
{
tok = tokenStream->tstream->get(tokenStream->tstream, i);
if ( types == NULL
|| types->isMember(types, tok->getType(tok) == ANTLR3_TRUE)
)
{
filteredList->put(filteredList, n++, (void *)tok, NULL);
}
}
/* Did we get any then?
*/
if (filteredList->size(filteredList) == 0)
{
filteredList->free(filteredList);
filteredList = NULL;
}
return filteredList;
}
static pANTLR3_LIST
getTokensList (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list)
{
pANTLR3_BITSET bitSet;
pANTLR3_LIST newlist;
bitSet = antlr3BitsetList(list->table);
newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);
bitSet->free(bitSet);
return newlist;
}
static pANTLR3_LIST
getTokensType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type)
{
pANTLR3_BITSET bitSet;
pANTLR3_LIST newlist;
bitSet = antlr3BitsetOf(type, -1);
newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);
bitSet->free(bitSet);
return newlist;
}
static ANTLR3_UINT32
_LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
{
pANTLR3_TOKEN_STREAM ts;
pANTLR3_COMMON_TOKEN tok;
ts = (pANTLR3_TOKEN_STREAM) is->super;
tok = ts->_LT(ts, i);
if (tok != NULL)
{
return tok->getType(tok);
}
else
{
return ANTLR3_TOKEN_INVALID;
}
}
/// As per _LA() but for debug mode.
///
static ANTLR3_UINT32
dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
{
pANTLR3_TOKEN_STREAM ts;
ts = (pANTLR3_TOKEN_STREAM) is->super;
if (ts->initialStreamState == ANTLR3_TRUE)
{
consumeInitialHiddenTokens(is);
}
ts->debugger->LT(ts->debugger, i, tokLT(ts, i));
return _LA(is, i);
}
static ANTLR3_MARKER
mark (pANTLR3_INT_STREAM is)
{
is->lastMarker = is->index(is);
return is->lastMarker;
}
/// As per mark() but with a call to tell the debugger we are doing this
///
static ANTLR3_MARKER
dbgMark (pANTLR3_INT_STREAM is)
{
pANTLR3_TOKEN_STREAM ts;
ts = (pANTLR3_TOKEN_STREAM) is->super;
is->lastMarker = is->index(is);
ts->debugger->mark(ts->debugger, is->lastMarker);
return is->lastMarker;
}
static void
release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark)
{
return;
}
static ANTLR3_UINT32
size (pANTLR3_INT_STREAM is)
{
pANTLR3_COMMON_TOKEN_STREAM cts;
pANTLR3_TOKEN_STREAM ts;
if (is->cachedSize > 0)
{
return is->cachedSize;
}
ts = (pANTLR3_TOKEN_STREAM) is->super;
cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
is->cachedSize = cts->tokens->count;
return is->cachedSize;
}
static ANTLR3_MARKER
tindex (pANTLR3_INT_STREAM is)
{
pANTLR3_COMMON_TOKEN_STREAM cts;
pANTLR3_TOKEN_STREAM ts;
ts = (pANTLR3_TOKEN_STREAM) is->super;
cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
return cts->p;
}
static void
dbgRewindLast (pANTLR3_INT_STREAM is)
{
pANTLR3_TOKEN_STREAM ts;
ts = (pANTLR3_TOKEN_STREAM) is->super;
ts->debugger->rewindLast(ts->debugger);
is->rewind(is, is->lastMarker);
}
static void
rewindLast (pANTLR3_INT_STREAM is)
{
is->rewind(is, is->lastMarker);
}
static void
rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
{
is->seek(is, (ANTLR3_UINT32)(marker));
}
static void
dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
{
pANTLR3_TOKEN_STREAM ts;
ts = (pANTLR3_TOKEN_STREAM) is->super;
ts->debugger->rewind(ts->debugger, marker);
is->seek(is, (ANTLR3_UINT32)(marker));
}
static void
seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
{
pANTLR3_COMMON_TOKEN_STREAM cts;
pANTLR3_TOKEN_STREAM ts;
ts = (pANTLR3_TOKEN_STREAM) is->super;
cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
cts->p = (ANTLR3_UINT32)index;
}
static void
dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
{
// TODO: Implement seek in debugger when Ter adds it to Java
//
seek(is, index);
}
ANTLR3_API void
fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream)
{
fillBuffer(tokenStream);
}
static void
fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) {
ANTLR3_UINT32 index;
pANTLR3_COMMON_TOKEN tok;
ANTLR3_BOOLEAN discard;
void * channelI;
/* Start at index 0 of course
*/
index = 0;
/* Pick out the next token from the token source
* Remember we just get a pointer (reference if you like) here
* and so if we store it anywhere, we don't set any pointers to auto free it.
*/
tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF)
{
discard = ANTLR3_FALSE; /* Assume we are not discarding */
/* I employ a bit of a trick, or perhaps hack here. Rather than
* store a pointer to a structure in the override map and discard set
* we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0
* we can distinguish "not being there" from "being channel or type 0"
*/
if (tokenStream->discardSet != NULL
&& tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL)
{
discard = ANTLR3_TRUE;
}
else if ( tokenStream->discardOffChannel == ANTLR3_TRUE
&& tok->getChannel(tok) != tokenStream->channel
)
{
discard = ANTLR3_TRUE;
}
else if (tokenStream->channelOverrides != NULL)
{
/* See if this type is in the override map
*/
channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1);
if (channelI != NULL)
{
/* Override found
*/
tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1);
}
}
/* If not discarding it, add it to the list at the current index
*/
if (discard == ANTLR3_FALSE)
{
/* Add it, indicating that we will delete it and the table should not
*/
tok->setTokenIndex(tok, index);
tokenStream->p++;
tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL);
index++;
}
tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
}
/* Cache the size so we don't keep doing indirect method calls. We do this as
* early as possible so that anything after this may utilize the cached value.
*/
tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count;
/* Set the consume pointer to the first token that is on our channel
*/
tokenStream->p = 0;
tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p);
}
/// Given a starting index, return the index of the first on-channel
/// token.
///
static ANTLR3_UINT32
skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) {
ANTLR3_INT32 n;
pANTLR3_COMMON_TOKEN tok;
n = tokenStream->tstream->istream->cachedSize;
while (i < n)
{
tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element;
if (tok->channel!= tokenStream->channel)
{
i++;
}
else
{
return i;
}
}
return i;
}
static ANTLR3_UINT32
skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x)
{
pANTLR3_COMMON_TOKEN tok;
while (x >= 0)
{
tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element;
if ((tok->channel != tokenStream->channel))
{
x--;
}
else
{
return x;
}
}
return x;
}
/// Return a string that represents the name assoicated with the input source
///
/// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream.
///
/// /returns
/// /implements ANTLR3_INT_STREAM_struct::getSourceName()
///
static pANTLR3_STRING
getSourceName (pANTLR3_INT_STREAM is)
{
// Slightly convoluted as we must trace back to the lexer's input source
// via the token source. The streamName that is here is not initialized
// because this is a token stream, not a file or string stream, which are the
// only things that have a context for a source name.
//
return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName;
}