/*
* [The "BSD licence"]
* Copyright (c) 2005-2008 Terence Parr
* All rights reserved.
*
* Conversion to C#:
* Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
namespace Antlr.Runtime
{
using System.Collections.Generic;
using InvalidOperationException = System.InvalidOperationException;
using StringBuilder = System.Text.StringBuilder;
/** <summary>
* The most common stream of tokens is one where every token is buffered up
* and tokens are prefiltered for a certain channel (the parser will only
* see these tokens and cannot change the filter channel number during the
* parse).
* </summary>
*
* <remarks>TODO: how to access the full token stream? How to track all tokens matched per rule?</remarks>
*/
[System.Serializable]
public class LegacyCommonTokenStream : ITokenStream
{
[System.NonSerialized]
ITokenSource _tokenSource;
/** <summary>
* Record every single token pulled from the source so we can reproduce
* chunks of it later.
* </summary>
*/
protected List<IToken> tokens;
/** <summary>Map<tokentype, channel> to override some Tokens' channel numbers</summary> */
protected IDictionary<int, int> channelOverrideMap;
/** <summary>Set<tokentype>; discard any tokens with this type</summary> */
protected List<int> discardSet;
/** <summary>Skip tokens on any channel but this one; this is how we skip whitespace...</summary> */
protected int channel = TokenChannels.Default;
/** <summary>By default, track all incoming tokens</summary> */
protected bool discardOffChannelTokens = false;
/** <summary>Track the last mark() call result value for use in rewind().</summary> */
protected int lastMarker;
/** <summary>
* The index into the tokens list of the current token (next token
* to consume). p==-1 indicates that the tokens list is empty
* </summary>
*/
protected int p = -1;
public LegacyCommonTokenStream()
{
tokens = new List<IToken>( 500 );
}
public LegacyCommonTokenStream(ITokenSource tokenSource)
: this()
{
this._tokenSource = tokenSource;
}
public LegacyCommonTokenStream( ITokenSource tokenSource, int channel )
: this( tokenSource )
{
this.channel = channel;
}
public virtual int Index
{
get
{
return p;
}
}
/// <summary>
/// How deep have we gone?
/// </summary>
public virtual int Range
{
get;
protected set;
}
/** <summary>Reset this token stream by setting its token source.</summary> */
public virtual void SetTokenSource( ITokenSource tokenSource )
{
this._tokenSource = tokenSource;
tokens.Clear();
p = -1;
channel = TokenChannels.Default;
}
/** <summary>
* Load all tokens from the token source and put in tokens.
* This is done upon first LT request because you might want to
* set some token type / channel overrides before filling buffer.
* </summary>
*/
public virtual void FillBuffer()
{
// fast return if the buffer is already full
if ( p != -1 )
return;
int index = 0;
IToken t = _tokenSource.NextToken();
while ( t != null && t.Type != CharStreamConstants.EndOfFile )
{
bool discard = false;
// is there a channel override for token type?
int channelI;
if ( channelOverrideMap != null && channelOverrideMap.TryGetValue( t.Type, out channelI ) )
t.Channel = channelI;
//if ( channelOverrideMap != null && channelOverrideMap.ContainsKey( t.getType() ) )
//{
// object channelI = channelOverrideMap.get( t.getType() );
// if ( channelI != null )
// {
// t.setChannel( (int)channelI );
// }
//}
if ( discardSet != null &&
discardSet.Contains( t.Type ) )
{
discard = true;
}
else if ( discardOffChannelTokens && t.Channel != this.channel )
{
discard = true;
}
if ( !discard )
{
t.TokenIndex = index;
tokens.Add( t );
index++;
}
t = _tokenSource.NextToken();
}
// leave p pointing at first token on channel
p = 0;
p = SkipOffTokenChannels( p );
}
/** <summary>
* Move the input pointer to the next incoming token. The stream
* must become active with LT(1) available. consume() simply
* moves the input pointer so that LT(1) points at the next
* input symbol. Consume at least one token.
* </summary>
*
* <remarks>
* Walk past any token not on the channel the parser is listening to.
* </remarks>
*/
public virtual void Consume()
{
if ( p < tokens.Count )
{
p++;
p = SkipOffTokenChannels( p ); // leave p on valid token
}
}
/** <summary>Given a starting index, return the index of the first on-channel token.</summary> */
protected virtual int SkipOffTokenChannels( int i )
{
int n = tokens.Count;
while ( i < n && ( (IToken)tokens[i] ).Channel != channel )
{
i++;
}
return i;
}
protected virtual int SkipOffTokenChannelsReverse( int i )
{
while ( i >= 0 && ( (IToken)tokens[i] ).Channel != channel )
{
i--;
}
return i;
}
/** <summary>
* A simple filter mechanism whereby you can tell this token stream
* to force all tokens of type ttype to be on channel. For example,
* when interpreting, we cannot exec actions so we need to tell
* the stream to force all WS and NEWLINE to be a different, ignored
* channel.
* </summary>
*/
public virtual void SetTokenTypeChannel( int ttype, int channel )
{
if ( channelOverrideMap == null )
{
channelOverrideMap = new Dictionary<int, int>();
}
channelOverrideMap[ttype] = channel;
}
public virtual void DiscardTokenType( int ttype )
{
if ( discardSet == null )
{
discardSet = new List<int>();
}
discardSet.Add( ttype );
}
public virtual void SetDiscardOffChannelTokens( bool discardOffChannelTokens )
{
this.discardOffChannelTokens = discardOffChannelTokens;
}
public virtual IList<IToken> GetTokens()
{
if ( p == -1 )
{
FillBuffer();
}
return tokens;
}
public virtual IList<IToken> GetTokens( int start, int stop )
{
return GetTokens( start, stop, (BitSet)null );
}
/** <summary>
* Given a start and stop index, return a List of all tokens in
* the token type BitSet. Return null if no tokens were found. This
* method looks at both on and off channel tokens.
* </summary>
*/
public virtual IList<IToken> GetTokens( int start, int stop, BitSet types )
{
if ( p == -1 )
{
FillBuffer();
}
if ( stop >= tokens.Count )
{
stop = tokens.Count - 1;
}
if ( start < 0 )
{
start = 0;
}
if ( start > stop )
{
return null;
}
// list = tokens[start:stop]:{Token t, t.getType() in types}
IList<IToken> filteredTokens = new List<IToken>();
for ( int i = start; i <= stop; i++ )
{
IToken t = tokens[i];
if ( types == null || types.Member( t.Type ) )
{
filteredTokens.Add( t );
}
}
if ( filteredTokens.Count == 0 )
{
filteredTokens = null;
}
return filteredTokens;
}
public virtual IList<IToken> GetTokens( int start, int stop, IList<int> types )
{
return GetTokens( start, stop, new BitSet( types ) );
}
public virtual IList<IToken> GetTokens( int start, int stop, int ttype )
{
return GetTokens( start, stop, BitSet.Of( ttype ) );
}
/** <summary>
* Get the ith token from the current position 1..n where k=1 is the
* first symbol of lookahead.
* </summary>
*/
public virtual IToken LT( int k )
{
if ( p == -1 )
{
FillBuffer();
}
if ( k == 0 )
{
return null;
}
if ( k < 0 )
{
return LB( -k );
}
//System.out.print("LT(p="+p+","+k+")=");
if ( ( p + k - 1 ) >= tokens.Count )
{
return tokens[tokens.Count - 1];
}
//System.out.println(tokens.get(p+k-1));
int i = p;
int n = 1;
// find k good tokens
while ( n < k )
{
// skip off-channel tokens
i = SkipOffTokenChannels( i + 1 ); // leave p on valid token
n++;
}
if ( i >= tokens.Count )
{
return tokens[tokens.Count - 1];
}
if (i > Range)
Range = i;
return (IToken)tokens[i];
}
/** <summary>Look backwards k tokens on-channel tokens</summary> */
protected virtual IToken LB( int k )
{
//System.out.print("LB(p="+p+","+k+") ");
if ( p == -1 )
{
FillBuffer();
}
if ( k == 0 )
{
return null;
}
if ( ( p - k ) < 0 )
{
return null;
}
int i = p;
int n = 1;
// find k good tokens looking backwards
while ( n <= k )
{
// skip off-channel tokens
i = SkipOffTokenChannelsReverse( i - 1 ); // leave p on valid token
n++;
}
if ( i < 0 )
{
return null;
}
return (IToken)tokens[i];
}
/** <summary>
* Return absolute token i; ignore which channel the tokens are on;
* that is, count all tokens not just on-channel tokens.
* </summary>
*/
public virtual IToken Get( int i )
{
return (IToken)tokens[i];
}
#if false
/** Get all tokens from start..stop inclusively */
public virtual List<IToken> Get(int start, int count)
{
if (start < 0)
throw new ArgumentOutOfRangeException("start");
if (count < 0)
throw new ArgumentOutOfRangeException("count");
if (p == -1)
FillBuffer();
return new List<IToken>(tokens.Skip(start).Take(count));
}
#endif
public virtual int LA( int i )
{
return LT( i ).Type;
}
public virtual int Mark()
{
if ( p == -1 )
{
FillBuffer();
}
lastMarker = Index;
return lastMarker;
}
public virtual void Release( int marker )
{
// no resources to release
}
public virtual int Count
{
get
{
return tokens.Count;
}
}
public virtual void Rewind( int marker )
{
Seek( marker );
}
public virtual void Rewind()
{
Seek( lastMarker );
}
public virtual void Reset()
{
p = 0;
lastMarker = 0;
}
public virtual void Seek( int index )
{
p = index;
}
public virtual ITokenSource TokenSource
{
get
{
return _tokenSource;
}
}
public virtual string SourceName
{
get
{
return TokenSource.SourceName;
}
}
public override string ToString()
{
if ( p == -1 )
{
throw new InvalidOperationException( "Buffer is not yet filled." );
}
return ToString( 0, tokens.Count - 1 );
}
public virtual string ToString( int start, int stop )
{
if ( start < 0 || stop < 0 )
{
return null;
}
if ( p == -1 )
{
throw new InvalidOperationException( "Buffer is not yet filled." );
}
if ( stop >= tokens.Count )
{
stop = tokens.Count - 1;
}
StringBuilder buf = new StringBuilder();
for ( int i = start; i <= stop; i++ )
{
IToken t = tokens[i];
buf.Append( t.Text );
}
return buf.ToString();
}
public virtual string ToString( IToken start, IToken stop )
{
if ( start != null && stop != null )
{
return ToString( start.TokenIndex, stop.TokenIndex );
}
return null;
}
}
}