/*
* [The "BSD license"]
* Copyright (c) 2011 Terence Parr
* All rights reserved.
*
* Conversion to C#:
* Copyright (c) 2011 Sam Harwell, Pixel Mine, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
namespace Antlr.Runtime
{
using System.Collections.Generic;
using System;
using System.Text;
/** <summary>
* A pretty quick CharStream that pulls all data from an array
* directly. Every method call counts in the lexer. Java's
* strings aren't very good so I'm avoiding.
* </summary>
*/
[System.Serializable]
public class FastTokenStream
: ITokenStream<SlimToken>
{
[System.NonSerialized]
ITokenSource<SlimToken> _tokenSource;
/** <summary>
* Record every single token pulled from the source so we can reproduce
* chunks of it later.
* </summary>
*/
protected List<SlimToken> tokens;
/** <summary>Skip tokens on any channel but this one; this is how we skip whitespace...</summary> */
protected int channel = TokenChannels.Default;
/** <summary>Track the last mark() call result value for use in rewind().</summary> */
protected int lastMarker;
/** <summary>
* The index into the tokens list of the current token (next token
* to consume). p==-1 indicates that the tokens list is empty
* </summary>
*/
protected int p = -1;
public FastTokenStream()
{
tokens = new List<SlimToken>( 500 );
}
public FastTokenStream( ITokenSource<SlimToken> tokenSource )
: this()
{
this._tokenSource = tokenSource;
}
public FastTokenStream( ITokenSource<SlimToken> tokenSource, int channel )
: this( tokenSource )
{
this.channel = channel;
}
public int Index
{
get
{
return p;
}
}
/** <summary>Reset this token stream by setting its token source.</summary> */
public void SetTokenSource( ITokenSource<SlimToken> tokenSource )
{
this._tokenSource = tokenSource;
tokens.Clear();
p = -1;
channel = TokenChannels.Default;
}
/** <summary>
* Load all tokens from the token source and put in tokens.
* This is done upon first LT request because you might want to
* set some token type / channel overrides before filling buffer.
* </summary>
*/
public void FillBuffer()
{
// fast return if the buffer is already full
if ( p != -1 )
return;
int index = 0;
SlimToken t = _tokenSource.NextToken();
while ( t.Type != CharStreamConstants.EndOfFile )
{
//t.TokenIndex = index;
tokens.Add( t );
index++;
t = _tokenSource.NextToken();
}
// leave p pointing at first token on channel
p = 0;
p = SkipOffTokenChannels( p );
}
/** <summary>
* Move the input pointer to the next incoming token. The stream
* must become active with LT(1) available. consume() simply
* moves the input pointer so that LT(1) points at the next
* input symbol. Consume at least one token.
* </summary>
*
* <remarks>
* Walk past any token not on the channel the parser is listening to.
* </remarks>
*/
public void Consume()
{
if ( p < tokens.Count )
{
p++;
p = SkipOffTokenChannels( p ); // leave p on valid token
}
}
/** <summary>Given a starting index, return the index of the first on-channel token.</summary> */
protected int SkipOffTokenChannels( int i )
{
int n = tokens.Count;
while ( i < n && tokens[i].Channel != channel )
{
i++;
}
return i;
}
protected int SkipOffTokenChannelsReverse( int i )
{
while ( i >= 0 && tokens[i].Channel != channel )
{
i--;
}
return i;
}
public IList<SlimToken> GetTokens()
{
if ( p == -1 )
{
FillBuffer();
}
return tokens;
}
public IList<SlimToken> GetTokens( int start, int stop )
{
return GetTokens( start, stop, (BitSet)null );
}
/** <summary>
* Given a start and stop index, return a List of all tokens in
* the token type BitSet. Return null if no tokens were found. This
* method looks at both on and off channel tokens.
* </summary>
*/
public IList<SlimToken> GetTokens( int start, int stop, BitSet types )
{
if ( p == -1 )
{
FillBuffer();
}
if ( stop >= tokens.Count )
{
stop = tokens.Count - 1;
}
if ( start < 0 )
{
start = 0;
}
if ( start > stop )
{
return null;
}
// list = tokens[start:stop]:{Token t, t.getType() in types}
List<SlimToken> filteredTokens = new List<SlimToken>();
for ( int i = start; i <= stop; i++ )
{
SlimToken t = tokens[i];
if ( types == null || types.Member( t.Type ) )
{
filteredTokens.Add( t );
}
}
if ( filteredTokens.Count == 0 )
{
filteredTokens = null;
}
return filteredTokens;
}
public IList<SlimToken> GetTokens( int start, int stop, IList<int> types )
{
return GetTokens( start, stop, new BitSet( types ) );
}
public IList<SlimToken> GetTokens( int start, int stop, int ttype )
{
return GetTokens( start, stop, BitSet.Of( ttype ) );
}
/** <summary>
* Get the ith token from the current position 1..n where k=1 is the
* first symbol of lookahead.
* </summary>
*/
public SlimToken LT( int k )
{
if ( p == -1 )
{
FillBuffer();
}
if ( k == 0 )
{
return default( SlimToken );
}
if ( k < 0 )
{
return LB( -k );
}
//System.out.print("LT(p="+p+","+k+")=");
if ( ( p + k - 1 ) >= tokens.Count )
{
return new SlimToken(TokenTypes.EndOfFile);
}
//System.out.println(tokens.get(p+k-1));
int i = p;
int n = 1;
// find k good tokens
while ( n < k )
{
// skip off-channel tokens
i = SkipOffTokenChannels( i + 1 ); // leave p on valid token
n++;
}
if ( i >= tokens.Count )
{
return new SlimToken(TokenTypes.EndOfFile);
}
return tokens[i];
}
/** <summary>Look backwards k tokens on-channel tokens</summary> */
protected SlimToken LB( int k )
{
//System.out.print("LB(p="+p+","+k+") ");
if ( p == -1 )
{
FillBuffer();
}
if ( k == 0 )
{
return default( SlimToken );
}
if ( ( p - k ) < 0 )
{
return default( SlimToken );
}
int i = p;
int n = 1;
// find k good tokens looking backwards
while ( n <= k )
{
// skip off-channel tokens
i = SkipOffTokenChannelsReverse( i - 1 ); // leave p on valid token
n++;
}
if ( i < 0 )
{
return default( SlimToken );
}
return tokens[i];
}
/** <summary>
* Return absolute token i; ignore which channel the tokens are on;
* that is, count all tokens not just on-channel tokens.
* </summary>
*/
public SlimToken Get( int i )
{
return tokens[i];
}
public int LA( int i )
{
return LT( i ).Type;
}
public int Mark()
{
if ( p == -1 )
{
FillBuffer();
}
lastMarker = Index;
return lastMarker;
}
public void Release( int marker )
{
// no resources to release
}
public int Count
{
get
{
return tokens.Count;
}
}
public void Rewind( int marker )
{
Seek( marker );
}
public void Rewind()
{
Seek( lastMarker );
}
public void Reset()
{
p = 0;
lastMarker = 0;
}
public void Seek( int index )
{
p = index;
}
public ITokenSource<SlimToken> TokenSource
{
get
{
return _tokenSource;
}
}
public string SourceName
{
get
{
return TokenSource.SourceName;
}
}
public override string ToString()
{
if ( p == -1 )
{
throw new InvalidOperationException( "Buffer is not yet filled." );
}
return ToString( 0, tokens.Count - 1 );
}
public virtual string ToString( int start, int stop )
{
if ( start < 0 || stop < 0 )
{
return null;
}
if ( p == -1 )
{
throw new InvalidOperationException( "Buffer is not yet filled." );
}
if ( stop >= tokens.Count )
{
stop = tokens.Count - 1;
}
StringBuilder buf = new StringBuilder();
for ( int i = start; i <= stop; i++ )
{
SlimToken t = tokens[i];
SlimLexer lexer = _tokenSource as SlimLexer;
if ( lexer != null )
{
SlimStringStream input = lexer.CharStream as SlimStringStream;
if ( input != null )
{
string text = input.Substring( t.StartIndex, t.StopIndex - t.StartIndex + 1 );
buf.Append( text );
}
}
}
return buf.ToString();
}
public virtual string ToString( IToken start, IToken stop )
{
if ( start != null && stop != null )
{
return ToString( start.TokenIndex, stop.TokenIndex );
}
return null;
}
}
}