Adds selector tests.

Troy Melhase · Troy Melhase · commit 646fe5feff88 · 2011-10-20T03:00:15.000Z
diff --git a/bin/j2py b/bin/j2py
@@ -93,15 +93,15 @@ def main(options):
         print 'IOError: %s.' % (msg, )
         return code
 
-    config = Config(configs)
     timed['comp']
     try:
-	tree = buildAST(source, config)
+	tree = buildAST(source)
     except (Exception, ), exc:
         exception('exception while parsing')
         return 1
     timed['comp_finish']
 
+    config = Config(configs)
     timed['xform']
     transformAST(tree, config)
     timed['xform_finish']
@@ -118,6 +118,11 @@ def main(options):
     timed['encode_finish']
     timed['overall_finish']
 
+    if options.lexertokens:
+        for idx, tok in enumerate(tree.parser.input.tokens):
+            print >> sys.stderr, '{0}  {1}'.format(idx, tok)
+	print >> sys.stderr
+
     if options.javaast:
 	tree.dump(sys.stderr)
 	print >> sys.stderr
@@ -171,6 +176,9 @@ def config(argv):
     addopt('-l', '--loglevel', dest='loglevel',
 	   help='Set log level by name or value.',
 	   default='WARN', type='loglevel')
+    addopt('-t', '--lexer-tokens', dest='lexertokens',
+	   help='Print lexer tokens to stderr.',
+	   default=False, action='store_true')
     addopt('-p', '--python-tree', dest='pytree',
 	   help='Print python object tree to stderr.',
 	   default=False, action='store_true')
diff --git a/java2python/compiler/__init__.py b/java2python/compiler/__init__.py
@@ -7,4 +7,31 @@
 # instead of using directly referencing items within the subpackage.
 
 from java2python.compiler.block import Module
-from java2python.compiler.tool import buildAST, transformAST
+from java2python.lang import Lexer, Parser, StringStream, TokenStream, TreeAdaptor
+
+
+def buildAST(source):
+    """ Returns an AST for the given source. """
+    lexer = Lexer(StringStream(source))
+    parser = Parser(TokenStream(lexer))
+    adapter = TreeAdaptor(lexer, parser)
+    parser.setTreeAdaptor(adapter)
+    scope = parser.javaSource()
+    return scope.tree
+
+
+def buildJavaDocAST(source):
+    """ Returns an AST for the given javadoc source. """
+    from java2python.lang.JavaDocLexer import JavaDocLexer
+    from java2python.lang.JavaDocParser import JavaDocParser
+    lexer = JavaDocLexer(StringStream(source))
+    parser = JavaDocParser(TokenStream(lexer))
+    scope = parser.commentBody()
+    return scope.tree
+
+
+def transformAST(tree, config):
+    """ Walk the tree and apply the transforms in the config. """
+    for selector, call in config.last('astTransforms', ()):
+        for node in selector.walk(tree):
+	    call(node, config)
diff --git a/java2python/compiler/tool.py b/java2python/compiler/tool.py
diff --git a/java2python/lang/selector.py b/java2python/lang/selector.py
@@ -5,9 +5,8 @@
 
 
 class Selector(object):
-    """ Selector -> base for selectors; provides operator methods.
+    """ Selector -> base for selectors; provides operator methods. """
 
-    """
     def __add__(self, other):
         """ C + C => S
 
@@ -24,6 +23,10 @@ def __and__(self, other):
         """
 	return Descendant(self, other)
 
+    def __call__(self, *args, **kwds):
+        """ Subclasses must implement. """
+        raise NotImplementedError('Selector class cannot be called.')
+
     def __getitem__(self, key):
         """ C[n] => S
 
@@ -41,15 +44,44 @@ def __gt__(self, other):
         """
 	return Child(self, other)
 
-
     def walk(self, tree):
+        """ Select items from the tree and from the tree children. """
         for item in self(tree):
             yield item
         for child in tree.children:
             for item in self.walk(child):
                 yield item
 
 
+class Token(Selector):
+    """ Token(...) -> select tokens by matching attributes.
+
+    """
+    # channel=None, index=None, input=None, line=None, start=None, stop=None, text=None, type=None
+
+    def __init__(self, **attrs):
+        self.attrs = attrs
+        if isinstance(attrs.get('type'), (basestring, )):
+            self.attrs['type'] = getattr(tokens, attrs.get('type'))
+
+    def __call__(self, tree):
+        items = self.attrs.items()
+        token = tree.token
+
+        def match_or_call(k, v):
+            if callable(v):
+                return v(token)
+            return getattr(token, k)==v
+
+        if all(match_or_call(k, v) for k, v in items if v is not None):
+            yield tree
+
+    def __str__(self):
+        items = self.attrs.items()
+        keys = ('{}={}'.format(k, v) for k, v in items if v is not None)
+	return 'Token({})'.format(', '.join(keys))
+
+
 class Nth(Selector):
     """ E[n] ->  match any slice n of E
 
@@ -109,29 +141,6 @@ def __str__(self):
 	return 'Type({0}{1}:{2})'.format(tokens.map[self.key], val, self.key)
 
 
-class Token(Selector):
-    """ Token(T)    select any token by matching attributes.
-
-    Similar to the type selector in CSS.
-    """
-    # channel=None, index=None, input=None, line=None, start=None, stop=None, text=None, type=None
-
-    def __init__(self, **attrs):
-        self.attrs = attrs
-        if isinstance(attrs.get('type'), (basestring, )):
-            self.attrs['type'] = getattr(tokens, attrs.get('type'))
-
-    def __call__(self, tree):
-        items = self.attrs.items()
-        if all(getattr(tree.token, k)==v for k, v in items if v is not None):
-            yield tree
-
-    def __str__(self):
-        items = self.attrs.items()
-        keys = ('{}={}'.format(k, v) for k, v in items if v is not None)
-	return 'Token({})'.format(', '.join(keys))
-
-
 class Star(Selector):
     """ *    select any
 
diff --git a/test/Makefile b/test/Makefile
@@ -14,6 +14,7 @@ test_targets := $(sort $(notdir $(basename $(wildcard *.java))))
 
 all:
 	$(MAKE) $(test_targets)
+	@cd selector && make
 
 
 clean:
diff --git a/test/selector/Makefile b/test/selector/Makefile
@@ -0,0 +1,12 @@
+.PHONY: all
+
+test_targets := $(sort $(notdir $(basename $(wildcard *.py))))
+
+
+all:
+	$(MAKE) $(test_targets)
+
+%:
+	@python -m unittest -v $@
+
+
diff --git a/test/selector/Selector1.java b/test/selector/Selector1.java
@@ -1,7 +1,9 @@
 class Selector1 {
-    void bar() {
+    void bar(int x, int y) {
         int foo = 3;
+	String z = "foo";
     }
 
-    void foo() {}
+    void foo() {};
+
 }
diff --git a/test/selector/test_all.py b/test/selector/test_all.py
@@ -1,37 +1,104 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
+import os
+import sys
+import unittest
 
-from java2python.config import Config
 from java2python.compiler import buildAST
 from java2python.lang import tokens
 from java2python.lang.selector import Token, Type
+from java2python.lib import colortools
 
 
-if __name__ == '__main__':
-    import sys, os
+def setUpModule():
     fn = os.path.join(os.path.dirname(__file__), 'Selector1.java')
-    source = open(fn).read()
-    tree = buildAST(source, Config(()))
-    tree.dump(sys.stdout)
-
-    selectors = [
-#	Type('EXPR'),
-#	Type('QUALIFIED_TYPE_IDENT') > Type('IDENT'),
-#        Type('CLASS')[2],
-#	Type('METHOD_CALL') & Type('IDENT'),
-#	Type('IDENT') + Type('IDENT'),
-        Type('CLASS') > Type('IDENT'),
-        Type('IDENT', 'foo'),
-        Token(text='foo'),
-        Token(type='BLOCK_SCOPE', line=2) & Token(type='IDENT', text='foo')
-    ]
-
-    for index, selector in enumerate(selectors):
-	print 'Selector Test {0}:\n   ==== {1}'.format(index, selector)
-        for node in selector.walk(tree):
-	    name = str(node)
-	    ntype = tokens.map[node.type]
-	    args = (name, '') if name == ntype else (ntype, name)
-	    print '{0}---- match: {1} {2}'.format(*(' '*8, )+args)
-            print '{0}---- token: {1}'.format(' '*8, node.dumps() )
-	print
+    SelectorTest.tree = buildAST(open(fn).read())
+    SelectorTest.tree.dump(sys.stdout)
+
+
+class SelectorTest(unittest.TestCase):
+    def walk(self, selector):
+        return list(selector.walk(self.tree))
+
+    def assertNodes(self, nodes, length):
+        self.assertTrue(nodes)
+        self.assertEqual(len(nodes), length)
+
+    def shortDescription(self):
+        fs = 'Description: {}\nSelector: {}\n'
+        args = (colortools.cyan(self.description), colortools.yellow(self.selector))
+        return fs.format(*args)
+
+    @classmethod
+    def make(cls, count):
+        def t(self):
+            nodes = self.walk(self.selector)
+            self.assertNodes(nodes, count)
+        return t
+
+
+class TestIdentChildOfClass(SelectorTest):
+    description = 'select one IDENT node that is a child of a CLASS node'
+    selector = Type('CLASS') > Type('IDENT')
+    test = SelectorTest.make(1)
+
+
+class TestIdentWithText(SelectorTest):
+    description = 'select two IDENT nodes with text "foo"'
+    selector = Type('IDENT', 'foo')
+    test = SelectorTest.make(2)
+
+
+class TestTokensWithText(SelectorTest):
+    description = 'select two nodes with text "foo"'
+    selector = Token(text='foo')
+    test = SelectorTest.make(2)
+
+
+class TestTokenCallableCombo(SelectorTest):
+    description = 'select BLOCK_SCOPE on line 7'
+    selector = Token(type=lambda t: t.type == tokens.BLOCK_SCOPE, line=7)
+    test = SelectorTest.make(1)
+
+
+class TestTokenMultipleCallables(SelectorTest):
+    description = 'select BLOCK_SCOPE on line 2 or 7'
+    selector = Token(type=lambda t: t.type == tokens.BLOCK_SCOPE, line=lambda t:t.line in (2, 7))
+    test = SelectorTest.make(2)
+
+
+class TestTokenChildCallable(SelectorTest):
+    description = 'select BLOCK_SCOPE with one child IDENT starting with "f"'
+    selector = Token(type=lambda t: t.type == tokens.BLOCK_SCOPE) & Token(type='IDENT', text=lambda tok:tok.text.startswith('f'))
+    test = SelectorTest.make(1)
+
+
+class TestNthChildWithExtraChecks(SelectorTest):
+    description = 'select two children of FORMAL_PARAM_STD_DECL at index 2 '
+    selector = Type('FORMAL_PARAM_STD_DECL')[2]
+
+    def test(self):
+        nodes = self.walk(self.selector)
+        self.assertNodes(nodes, 2)
+        self.assertEquals(nodes[0].type, tokens.IDENT)
+        self.assertEquals(nodes[1].type, tokens.IDENT)
+        self.assertEquals(nodes[0].text, 'x')
+        self.assertEquals(nodes[1].text, 'y')
+
+
+class TestDirectChildren(SelectorTest):
+    description = 'select two TYPE nodes that are children of a VAR_DECLARATION node'
+    selector = Type('VAR_DECLARATION') > Type('TYPE')
+    test = SelectorTest.make(2)
+
+
+class TestSimpleSiblings(SelectorTest):
+    description = 'select three IDENT nodes that are siblings of a MODIFIER_LIST'
+    selector = Type('MODIFIER_LIST') + Type('IDENT')
+    test = SelectorTest.make(3)
+
+
+class TestClassIdent(SelectorTest):
+    description = 'select one IDENT node that is a child of a CLASS node'
+    selector = Type('CLASS') > Type('IDENT')
+    test = SelectorTest.make(1)

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,9 @@`
`1`	`1`	`class Selector1 {`
`2`		`- void bar() {`
	`2`	`+ void bar(int x, int y) {`
`3`	`3`	`int foo = 3;`
	`4`	`+ String z = "foo";`
`4`	`5`	`}`
`5`	`6`
`6`		`- void foo() {}`
	`7`	`+ void foo() {};`
	`8`	`+`
`7`	`9`	`}`