import unittest
import imp
import os
import errno
import sys
import glob
import re
import tempfile
import shutil
import inspect
import hashlib
from distutils.errors import *
import antlr3
def unlink(path):
try:
os.unlink(path)
except OSError, exc:
if exc.errno != errno.ENOENT:
raise
class GrammarCompileError(Exception):
"""Grammar failed to compile."""
pass
# At least on MacOSX tempdir (/tmp) is a symlink. It's sometimes dereferences,
# sometimes not, breaking the inspect.getmodule() function.
testbasedir = os.path.join(
os.path.realpath(tempfile.gettempdir()),
'antlr3-test')
class BrokenTest(unittest.TestCase.failureException):
def __repr__(self):
name, reason = self.args
return '%s: %s: %s works now' % (
(self.__class__.__name__, name, reason))
def broken(reason, *exceptions):
'''Indicates a failing (or erroneous) test case fails that should succeed.
If the test fails with an exception, list the exception type in args'''
def wrapper(test_method):
def replacement(*args, **kwargs):
try:
test_method(*args, **kwargs)
except exceptions or unittest.TestCase.failureException:
pass
else:
raise BrokenTest(test_method.__name__, reason)
replacement.__doc__ = test_method.__doc__
replacement.__name__ = 'XXX_' + test_method.__name__
replacement.todo = reason
return replacement
return wrapper
dependencyCache = {}
compileErrorCache = {}
# setup java CLASSPATH
if 'CLASSPATH' not in os.environ:
cp = []
baseDir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
libDir = os.path.join(baseDir, 'lib')
jar = os.path.join(libDir, 'ST-4.0.1.jar')
if not os.path.isfile(jar):
raise DistutilsFileError(
"Missing file '%s'. Grap it from a distribution package."
% jar,
)
cp.append(jar)
jar = os.path.join(libDir, 'antlr-2.7.7.jar')
if not os.path.isfile(jar):
raise DistutilsFileError(
"Missing file '%s'. Grap it from a distribution package."
% jar,
)
cp.append(jar)
jar = os.path.join(libDir, 'junit-4.2.jar')
if not os.path.isfile(jar):
raise DistutilsFileError(
"Missing file '%s'. Grap it from a distribution package."
% jar,
)
cp.append(jar)
cp.append(os.path.join(baseDir, 'runtime', 'Python', 'build'))
classpath = '-cp "' + ':'.join([os.path.abspath(p) for p in cp]) + '"'
else:
classpath = ''
class ANTLRTest(unittest.TestCase):
def __init__(self, *args, **kwargs):
unittest.TestCase.__init__(self, *args, **kwargs)
self.moduleName = os.path.splitext(os.path.basename(sys.modules[self.__module__].__file__))[0]
self.className = self.__class__.__name__
self._baseDir = None
self.lexerModule = None
self.parserModule = None
self.grammarName = None
self.grammarType = None
def assertListEqual(self, a, b):
if a == b:
return
import difflib
a = [str(l) + '\n' for l in a]
b = [str(l) + '\n' for l in b]
raise AssertionError(''.join(difflib.unified_diff(a, b)))
@property
def baseDir(self):
if self._baseDir is None:
testName = 'unknownTest'
for frame in inspect.stack():
code = frame[0].f_code
codeMod = inspect.getmodule(code)
if codeMod is None:
continue
# skip frames not in requested module
if codeMod is not sys.modules[self.__module__]:
continue
# skip some unwanted names
if code.co_name in ('nextToken', '<module>'):
continue
if code.co_name.startswith('test'):
testName = code.co_name
break
self._baseDir = os.path.join(
testbasedir,
self.moduleName, self.className, testName)
if not os.path.isdir(self._baseDir):
os.makedirs(self._baseDir)
return self._baseDir
def _invokeantlr(self, dir, file, options, javaOptions=''):
cmd = 'cd %s; java %s %s org.antlr.Tool -o . %s %s 2>&1' % (
dir, javaOptions, classpath, options, file
)
fp = os.popen(cmd)
output = ''
failed = False
for line in fp:
output += line
if line.startswith('error('):
failed = True
rc = fp.close()
if rc is not None:
failed = True
if failed:
raise GrammarCompileError(
"Failed to compile grammar '%s':\n%s\n\n" % (file, cmd)
+ output
)
def compileGrammar(self, grammarName=None, options='', javaOptions=''):
if grammarName is None:
grammarName = self.moduleName + '.g'
self._baseDir = os.path.join(
testbasedir,
self.moduleName)
if not os.path.isdir(self._baseDir):
os.makedirs(self._baseDir)
if self.grammarName is None:
self.grammarName = os.path.splitext(grammarName)[0]
grammarPath = os.path.join(os.path.dirname(os.path.abspath(__file__)), grammarName)
# get type and name from first grammar line
grammar = open(grammarPath, 'r').read()
m = re.match(r'\s*((lexer|parser|tree)\s+|)grammar\s+(\S+);', grammar, re.MULTILINE)
assert m is not None, grammar
self.grammarType = m.group(2)
if self.grammarType is None:
self.grammarType = 'combined'
if self.grammarType is None:
assert self.grammarType in ('lexer', 'parser', 'tree', 'combined'), self.grammarType
# don't try to rebuild grammar, if it already failed
if grammarName in compileErrorCache:
return
try:
# # get dependencies from antlr
# if grammarName in dependencyCache:
# dependencies = dependencyCache[grammarName]
# else:
# dependencies = []
# cmd = ('cd %s; java %s %s org.antlr.Tool -o . -depend %s 2>&1'
# % (self.baseDir, javaOptions, classpath, grammarPath))
# output = ""
# failed = False
# fp = os.popen(cmd)
# for line in fp:
# output += line
# if line.startswith('error('):
# failed = True
# elif ':' in line:
# a, b = line.strip().split(':', 1)
# dependencies.append(
# (os.path.join(self.baseDir, a.strip()),
# [os.path.join(self.baseDir, b.strip())])
# )
# rc = fp.close()
# if rc is not None:
# failed = True
# if failed:
# raise GrammarCompileError(
# "antlr -depend failed with code %s on grammar '%s':\n\n"
# % (rc, grammarName)
# + cmd
# + "\n"
# + output
# )
# # add dependencies to my .stg files
# templateDir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'tool', 'src', 'main', 'resources', 'org', 'antlr', 'codegen', 'templates', 'Python'))
# templates = glob.glob(os.path.join(templateDir, '*.stg'))
# for dst, src in dependencies:
# src.extend(templates)
# dependencyCache[grammarName] = dependencies
# rebuild = False
# for dest, sources in dependencies:
# if not os.path.isfile(dest):
# rebuild = True
# break
# for source in sources:
# if os.path.getmtime(source) > os.path.getmtime(dest):
# rebuild = True
# break
# if rebuild:
# self._invokeantlr(self.baseDir, grammarPath, options, javaOptions)
self._invokeantlr(self.baseDir, grammarPath, options, javaOptions)
except:
# mark grammar as broken
compileErrorCache[grammarName] = True
raise
def lexerClass(self, base):
"""Optionally build a subclass of generated lexer class"""
return base
def parserClass(self, base):
"""Optionally build a subclass of generated parser class"""
return base
def walkerClass(self, base):
"""Optionally build a subclass of generated walker class"""
return base
def __load_module(self, name):
modFile, modPathname, modDescription \
= imp.find_module(name, [self.baseDir])
return imp.load_module(
name, modFile, modPathname, modDescription
)
def getLexer(self, *args, **kwargs):
"""Build lexer instance. Arguments are passed to lexer.__init__()."""
if self.grammarType == 'lexer':
self.lexerModule = self.__load_module(self.grammarName)
cls = getattr(self.lexerModule, self.grammarName)
else:
self.lexerModule = self.__load_module(self.grammarName + 'Lexer')
cls = getattr(self.lexerModule, self.grammarName + 'Lexer')
cls = self.lexerClass(cls)
lexer = cls(*args, **kwargs)
return lexer
def getParser(self, *args, **kwargs):
"""Build parser instance. Arguments are passed to parser.__init__()."""
if self.grammarType == 'parser':
self.lexerModule = self.__load_module(self.grammarName)
cls = getattr(self.lexerModule, self.grammarName)
else:
self.parserModule = self.__load_module(self.grammarName + 'Parser')
cls = getattr(self.parserModule, self.grammarName + 'Parser')
cls = self.parserClass(cls)
parser = cls(*args, **kwargs)
return parser
def getWalker(self, *args, **kwargs):
"""Build walker instance. Arguments are passed to walker.__init__()."""
self.walkerModule = self.__load_module(self.grammarName + 'Walker')
cls = getattr(self.walkerModule, self.grammarName + 'Walker')
cls = self.walkerClass(cls)
walker = cls(*args, **kwargs)
return walker
def writeInlineGrammar(self, grammar):
# Create a unique ID for this test and use it as the grammar name,
# to avoid class name reuse. This kinda sucks. Need to find a way so
# tests can use the same grammar name without messing up the namespace.
# Well, first I should figure out what the exact problem is...
id = hashlib.md5(self.baseDir).hexdigest()[-8:]
grammar = grammar.replace('$TP', 'TP' + id)
grammar = grammar.replace('$T', 'T' + id)
# get type and name from first grammar line
m = re.match(r'\s*((lexer|parser|tree)\s+|)grammar\s+(\S+);', grammar, re.MULTILINE)
assert m is not None, grammar
grammarType = m.group(2)
if grammarType is None:
grammarType = 'combined'
grammarName = m.group(3)
assert grammarType in ('lexer', 'parser', 'tree', 'combined'), grammarType
grammarPath = os.path.join(self.baseDir, grammarName + '.g')
# dump temp grammar file
fp = open(grammarPath, 'w')
fp.write(grammar)
fp.close()
return grammarName, grammarPath, grammarType
def writeFile(self, name, contents):
testDir = os.path.dirname(os.path.abspath(__file__))
path = os.path.join(self.baseDir, name)
fp = open(path, 'w')
fp.write(contents)
fp.close()
return path
def compileInlineGrammar(self, grammar, options='', javaOptions='',
returnModule=False):
# write grammar file
grammarName, grammarPath, grammarType = self.writeInlineGrammar(grammar)
# compile it
self._invokeantlr(
os.path.dirname(grammarPath),
os.path.basename(grammarPath),
options,
javaOptions
)
if grammarType == 'combined':
lexerMod = self.__load_module(grammarName + 'Lexer')
parserMod = self.__load_module(grammarName + 'Parser')
if returnModule:
return lexerMod, parserMod
lexerCls = getattr(lexerMod, grammarName + 'Lexer')
lexerCls = self.lexerClass(lexerCls)
parserCls = getattr(parserMod, grammarName + 'Parser')
parserCls = self.parserClass(parserCls)
return lexerCls, parserCls
if grammarType == 'lexer':
lexerMod = self.__load_module(grammarName)
if returnModule:
return lexerMod
lexerCls = getattr(lexerMod, grammarName)
lexerCls = self.lexerClass(lexerCls)
return lexerCls
if grammarType == 'parser':
parserMod = self.__load_module(grammarName)
if returnModule:
return parserMod
parserCls = getattr(parserMod, grammarName)
parserCls = self.parserClass(parserCls)
return parserCls
if grammarType == 'tree':
walkerMod = self.__load_module(grammarName)
if returnModule:
return walkerMod
walkerCls = getattr(walkerMod, grammarName)
walkerCls = self.walkerClass(walkerCls)
return walkerCls