ply: update PLY to version 3.2

1 files changed, 564 insertions, 40 deletions

diff --git a/ext/ply/test/testlex.py b/ext/ply/test/testlex.py
index df000b83d..606387d1d 100755
--- a/ext/ply/test/testlex.py
+++ b/ext/ply/test/testlex.py
@@ -1,57 +1,581 @@
-#!/usr/local/bin
-# ----------------------------------------------------------------------
 # testlex.py
-#
-# Run tests for the lexing module
-# ----------------------------------------------------------------------
 
-import sys,os,glob
+import unittest
+try:
+    import StringIO
+except ImportError:
+    import io as StringIO
 
-if len(sys.argv) < 2:
-    print "Usage: python testlex.py directory"
-    raise SystemExit
+import sys
+sys.path.insert(0,"..")
+sys.tracebacklimit = 0
 
-dirname = None
-make = 0
+import ply.lex
 
-for o in sys.argv[1:]:
-    if o == '-make':
-        make = 1
-    else:
-        dirname = o
-        break
+def check_expected(result,expected):
+    if sys.version_info[0] >= 3:
+        if isinstance(result,str):
+            result = result.encode('ascii')
+        if isinstance(expected,str):
+            expected = expected.encode('ascii')
+    resultlines = result.splitlines()
+    expectedlines = expected.splitlines()
 
-if not dirname:
-    print "Usage: python testlex.py [-make] directory"
-    raise SystemExit
 
-f = glob.glob("%s/%s" % (dirname,"lex_*.py"))
+    if len(resultlines) != len(expectedlines):
+        return False
 
-print "**** Running tests for lex ****"
+    for rline,eline in zip(resultlines,expectedlines):
+        if not rline.endswith(eline):
+            return False
+    return True
 
-for t in f:
-    name = t[:-3]
-    print "Testing %-32s" % name,
-    if make:
-        if not os.path.exists("%s.exp" % name):
-            os.system("python %s.py >%s.exp 2>&1" % (name,name))
-        passed = 1
-    else:
-        os.system("python %s.py >%s.out 2>&1" % (name,name))
-        a = os.system("diff %s.out %s.exp >%s.dif" % (name,name,name))
-        if a == 0:
-            passed = 1
-        else:
-            passed = 0
+def run_import(module):
+    code = "import "+module
+    exec(code)
+    del sys.modules[module]
+    
+# Tests related to errors and warnings when building lexers
+class LexErrorWarningTests(unittest.TestCase):
+    def setUp(self):
+        sys.stderr = StringIO.StringIO()
+        sys.stdout = StringIO.StringIO()
+    def tearDown(self):
+        sys.stderr = sys.__stderr__
+        sys.stdout = sys.__stdout__
+    def test_lex_doc1(self):
+        self.assertRaises(SyntaxError,run_import,"lex_doc1")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                              "lex_doc1.py:18: No regular expression defined for rule 't_NUMBER'\n"))
+    def test_lex_dup1(self):
+        self.assertRaises(SyntaxError,run_import,"lex_dup1")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "lex_dup1.py:20: Rule t_NUMBER redefined. Previously defined on line 18\n" ))        
+        
+    def test_lex_dup2(self):
+        self.assertRaises(SyntaxError,run_import,"lex_dup2")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "lex_dup2.py:22: Rule t_NUMBER redefined. Previously defined on line 18\n" ))
+            
+    def test_lex_dup3(self):
+        self.assertRaises(SyntaxError,run_import,"lex_dup3")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "lex_dup3.py:20: Rule t_NUMBER redefined. Previously defined on line 18\n" ))
 
-    if passed:
-        print "Passed"
-    else:
-        print "Failed. See %s.dif" % name
+    def test_lex_empty(self):
+        self.assertRaises(SyntaxError,run_import,"lex_empty")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "No rules of the form t_rulename are defined\n"
+                                    "No rules defined for state 'INITIAL'\n"))
 
+    def test_lex_error1(self):
+        run_import("lex_error1")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "No t_error rule is defined\n"))
 
+    def test_lex_error2(self):
+        self.assertRaises(SyntaxError,run_import,"lex_error2")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "Rule 't_error' must be defined as a function\n")
+                     )
 
+    def test_lex_error3(self):
+        self.assertRaises(SyntaxError,run_import,"lex_error3")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "lex_error3.py:20: Rule 't_error' requires an argument\n"))
 
+    def test_lex_error4(self):
+        self.assertRaises(SyntaxError,run_import,"lex_error4")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "lex_error4.py:20: Rule 't_error' has too many arguments\n"))
 
+    def test_lex_ignore(self):
+        self.assertRaises(SyntaxError,run_import,"lex_ignore")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "lex_ignore.py:20: Rule 't_ignore' must be defined as a string\n"))
 
+    def test_lex_ignore2(self):
+        run_import("lex_ignore2")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "t_ignore contains a literal backslash '\\'\n"))
 
+
+    def test_lex_re1(self):
+        self.assertRaises(SyntaxError,run_import,"lex_re1")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "Invalid regular expression for rule 't_NUMBER'. unbalanced parenthesis\n"))
+
+    def test_lex_re2(self):
+        self.assertRaises(SyntaxError,run_import,"lex_re2")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "Regular expression for rule 't_PLUS' matches empty string\n"))
+
+    def test_lex_re3(self):
+        self.assertRaises(SyntaxError,run_import,"lex_re3")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "Invalid regular expression for rule 't_POUND'. unbalanced parenthesis\n"
+                                    "Make sure '#' in rule 't_POUND' is escaped with '\\#'\n"))
+
+    def test_lex_rule1(self):
+        self.assertRaises(SyntaxError,run_import,"lex_rule1")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "t_NUMBER not defined as a function or string\n"))
+
+    def test_lex_rule2(self):
+        self.assertRaises(SyntaxError,run_import,"lex_rule2")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "lex_rule2.py:18: Rule 't_NUMBER' requires an argument\n"))
+
+    def test_lex_rule3(self):
+        self.assertRaises(SyntaxError,run_import,"lex_rule3")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "lex_rule3.py:18: Rule 't_NUMBER' has too many arguments\n"))
+
+
+    def test_lex_state1(self):
+        self.assertRaises(SyntaxError,run_import,"lex_state1")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                   "states must be defined as a tuple or list\n"))
+
+    def test_lex_state2(self):
+        self.assertRaises(SyntaxError,run_import,"lex_state2")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "Invalid state specifier 'comment'. Must be a tuple (statename,'exclusive|inclusive')\n"
+                                    "Invalid state specifier 'example'. Must be a tuple (statename,'exclusive|inclusive')\n"))
+
+    def test_lex_state3(self):
+        self.assertRaises(SyntaxError,run_import,"lex_state3")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "State name 1 must be a string\n"
+                                    "No rules defined for state 'example'\n"))
+
+    def test_lex_state4(self):
+        self.assertRaises(SyntaxError,run_import,"lex_state4")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "State type for state comment must be 'inclusive' or 'exclusive'\n"))
+
+
+    def test_lex_state5(self):
+        self.assertRaises(SyntaxError,run_import,"lex_state5")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "State 'comment' already defined\n"))
+
+    def test_lex_state_noerror(self):
+        run_import("lex_state_noerror")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "No error rule is defined for exclusive state 'comment'\n"))    
+
+    def test_lex_state_norule(self):
+        self.assertRaises(SyntaxError,run_import,"lex_state_norule")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "No rules defined for state 'example'\n"))
+
+    def test_lex_token1(self):
+        self.assertRaises(SyntaxError,run_import,"lex_token1")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "No token list is defined\n"
+                                    "Rule 't_NUMBER' defined for an unspecified token NUMBER\n"
+                                    "Rule 't_PLUS' defined for an unspecified token PLUS\n"
+                                    "Rule 't_MINUS' defined for an unspecified token MINUS\n"
+))
+
+    def test_lex_token2(self):
+        self.assertRaises(SyntaxError,run_import,"lex_token2")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "tokens must be a list or tuple\n"
+                                    "Rule 't_NUMBER' defined for an unspecified token NUMBER\n"
+                                    "Rule 't_PLUS' defined for an unspecified token PLUS\n"
+                                    "Rule 't_MINUS' defined for an unspecified token MINUS\n"
+))
+    
+    def test_lex_token3(self):
+        self.assertRaises(SyntaxError,run_import,"lex_token3")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "Rule 't_MINUS' defined for an unspecified token MINUS\n"))
+
+
+    def test_lex_token4(self):
+        self.assertRaises(SyntaxError,run_import,"lex_token4")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "Bad token name '-'\n"))
+
+
+    def test_lex_token5(self):
+        try:
+            run_import("lex_token5")
+        except ply.lex.LexError:
+            e = sys.exc_info()[1]
+        self.assert_(check_expected(str(e),"lex_token5.py:19: Rule 't_NUMBER' returned an unknown token type 'NUM'"))
+
+    def test_lex_token_dup(self):
+        run_import("lex_token_dup")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "Token 'MINUS' multiply defined\n"))   
+
+        
+    def test_lex_literal1(self):
+        self.assertRaises(SyntaxError,run_import,"lex_literal1")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "Invalid literal '**'. Must be a single character\n"))
+
+    def test_lex_literal2(self):
+        self.assertRaises(SyntaxError,run_import,"lex_literal2")
+        result = sys.stderr.getvalue()
+        self.assert_(check_expected(result,
+                                    "Invalid literals specification. literals must be a sequence of characters\n"))
+
+import os
+import subprocess
+import shutil
+
+# Tests related to various build options associated with lexers
+class LexBuildOptionTests(unittest.TestCase):
+    def setUp(self):
+        sys.stderr = StringIO.StringIO()
+        sys.stdout = StringIO.StringIO()
+    def tearDown(self):
+        sys.stderr = sys.__stderr__
+        sys.stdout = sys.__stdout__
+        try:
+            shutil.rmtree("lexdir")
+        except OSError:
+            pass
+
+    def test_lex_module(self):
+        run_import("lex_module")
+        result = sys.stdout.getvalue()
+        self.assert_(check_expected(result,
+                                    "(NUMBER,3,1,0)\n"
+                                    "(PLUS,'+',1,1)\n"
+                                    "(NUMBER,4,1,2)\n"))
+        
+    def test_lex_object(self):
+        run_import("lex_object")
+        result = sys.stdout.getvalue()
+        self.assert_(check_expected(result,
+                                    "(NUMBER,3,1,0)\n"
+                                    "(PLUS,'+',1,1)\n"
+                                    "(NUMBER,4,1,2)\n"))
+
+    def test_lex_closure(self):
+        run_import("lex_closure")
+        result = sys.stdout.getvalue()
+        self.assert_(check_expected(result,
+                                    "(NUMBER,3,1,0)\n"
+                                    "(PLUS,'+',1,1)\n"
+                                    "(NUMBER,4,1,2)\n"))
+    def test_lex_optimize(self):
+        try:
+            os.remove("lextab.py")
+        except OSError:
+            pass
+        try:
+            os.remove("lextab.pyc")
+        except OSError:
+            pass
+        try:
+            os.remove("lextab.pyo")
+        except OSError:
+            pass
+        run_import("lex_optimize")
+
+        result = sys.stdout.getvalue()
+        self.assert_(check_expected(result,
+                                    "(NUMBER,3,1,0)\n"
+                                    "(PLUS,'+',1,1)\n"
+                                    "(NUMBER,4,1,2)\n"))
+        self.assert_(os.path.exists("lextab.py"))
+
+
+        p = subprocess.Popen([sys.executable,'-O','lex_optimize.py'],
+                             stdout=subprocess.PIPE)
+        result = p.stdout.read()
+
+        self.assert_(check_expected(result,
+                                    "(NUMBER,3,1,0)\n"
+                                    "(PLUS,'+',1,1)\n"
+                                    "(NUMBER,4,1,2)\n"))
+        self.assert_(os.path.exists("lextab.pyo"))
+
+        os.remove("lextab.pyo")
+        p = subprocess.Popen([sys.executable,'-OO','lex_optimize.py'],
+                             stdout=subprocess.PIPE)
+        result = p.stdout.read()
+        self.assert_(check_expected(result,
+                                    "(NUMBER,3,1,0)\n"
+                                    "(PLUS,'+',1,1)\n"
+                                    "(NUMBER,4,1,2)\n"))
+        self.assert_(os.path.exists("lextab.pyo"))
+        try:
+            os.remove("lextab.py")
+        except OSError:
+            pass
+        try:
+            os.remove("lextab.pyc")
+        except OSError:
+            pass
+        try:
+            os.remove("lextab.pyo")
+        except OSError:
+            pass
+
+    def test_lex_optimize2(self):
+        try:
+            os.remove("opt2tab.py")
+        except OSError:
+            pass
+        try:
+            os.remove("opt2tab.pyc")
+        except OSError:
+            pass
+        try:
+            os.remove("opt2tab.pyo")
+        except OSError:
+            pass
+        run_import("lex_optimize2")
+        result = sys.stdout.getvalue()
+        self.assert_(check_expected(result,
+                                    "(NUMBER,3,1,0)\n"
+                                    "(PLUS,'+',1,1)\n"
+                                    "(NUMBER,4,1,2)\n"))
+        self.assert_(os.path.exists("opt2tab.py"))
+
+        p = subprocess.Popen([sys.executable,'-O','lex_optimize2.py'],
+                             stdout=subprocess.PIPE)
+        result = p.stdout.read()
+        self.assert_(check_expected(result,
+                                    "(NUMBER,3,1,0)\n"
+                                    "(PLUS,'+',1,1)\n"
+                                    "(NUMBER,4,1,2)\n"))
+        self.assert_(os.path.exists("opt2tab.pyo"))
+        os.remove("opt2tab.pyo")
+        p = subprocess.Popen([sys.executable,'-OO','lex_optimize2.py'],
+                             stdout=subprocess.PIPE)
+        result = p.stdout.read()
+        self.assert_(check_expected(result,
+                                    "(NUMBER,3,1,0)\n"
+                                    "(PLUS,'+',1,1)\n"
+                                    "(NUMBER,4,1,2)\n"))
+        self.assert_(os.path.exists("opt2tab.pyo"))
+        try:
+            os.remove("opt2tab.py")
+        except OSError:
+            pass
+        try:
+            os.remove("opt2tab.pyc")
+        except OSError:
+            pass
+        try:
+            os.remove("opt2tab.pyo")
+        except OSError:
+            pass
+
+    def test_lex_optimize3(self):
+        try:
+            shutil.rmtree("lexdir")
+        except OSError:
+            pass
+        
+        os.mkdir("lexdir")
+        os.mkdir("lexdir/sub")
+        open("lexdir/__init__.py","w").write("")
+        open("lexdir/sub/__init__.py","w").write("")
+        run_import("lex_optimize3")
+        result = sys.stdout.getvalue()
+        self.assert_(check_expected(result,
+                                    "(NUMBER,3,1,0)\n"
+                                    "(PLUS,'+',1,1)\n"
+                                    "(NUMBER,4,1,2)\n"))
+        self.assert_(os.path.exists("lexdir/sub/calctab.py"))
+
+        p = subprocess.Popen([sys.executable,'-O','lex_optimize3.py'],
+                             stdout=subprocess.PIPE)
+        result = p.stdout.read()
+        self.assert_(check_expected(result,
+                                    "(NUMBER,3,1,0)\n"
+                                    "(PLUS,'+',1,1)\n"
+                                    "(NUMBER,4,1,2)\n"))
+        self.assert_(os.path.exists("lexdir/sub/calctab.pyo"))
+        os.remove("lexdir/sub/calctab.pyo")
+        p = subprocess.Popen([sys.executable,'-OO','lex_optimize3.py'],
+                             stdout=subprocess.PIPE)
+        result = p.stdout.read()
+        self.assert_(check_expected(result,
+                                    "(NUMBER,3,1,0)\n"
+                                    "(PLUS,'+',1,1)\n"
+                                    "(NUMBER,4,1,2)\n"))
+        self.assert_(os.path.exists("lexdir/sub/calctab.pyo"))
+        try:
+            shutil.rmtree("lexdir")
+        except OSError:
+            pass
+
+    def test_lex_opt_alias(self):
+        try:
+            os.remove("aliastab.py")
+        except OSError:
+            pass
+        try:
+            os.remove("aliastab.pyc")
+        except OSError:
+            pass
+        try:
+            os.remove("aliastab.pyo")
+        except OSError:
+            pass
+        run_import("lex_opt_alias")
+        result = sys.stdout.getvalue()
+        self.assert_(check_expected(result,
+                                    "(NUMBER,3,1,0)\n"
+                                    "(+,'+',1,1)\n"
+                                    "(NUMBER,4,1,2)\n"))
+        self.assert_(os.path.exists("aliastab.py"))
+
+        p = subprocess.Popen([sys.executable,'-O','lex_opt_alias.py'],
+                             stdout=subprocess.PIPE)
+        result = p.stdout.read()
+        self.assert_(check_expected(result,
+                                    "(NUMBER,3,1,0)\n"
+                                    "(+,'+',1,1)\n"
+                                    "(NUMBER,4,1,2)\n"))
+        self.assert_(os.path.exists("aliastab.pyo"))
+        os.remove("aliastab.pyo")
+        p = subprocess.Popen([sys.executable,'-OO','lex_opt_alias.py'],
+                             stdout=subprocess.PIPE)
+        result = p.stdout.read()
+        self.assert_(check_expected(result,
+                                    "(NUMBER,3,1,0)\n"
+                                    "(+,'+',1,1)\n"
+                                    "(NUMBER,4,1,2)\n"))
+        self.assert_(os.path.exists("aliastab.pyo"))
+        try:
+            os.remove("aliastab.py")
+        except OSError:
+            pass
+        try:
+            os.remove("aliastab.pyc")
+        except OSError:
+            pass
+        try:
+            os.remove("aliastab.pyo")
+        except OSError:
+            pass
+
+    def test_lex_many_tokens(self):
+        try:
+            os.remove("manytab.py")
+        except OSError:
+            pass
+        try:
+            os.remove("manytab.pyc")
+        except OSError:
+            pass
+        try:
+            os.remove("manytab.pyo")
+        except OSError:
+            pass
+        run_import("lex_many_tokens")
+        result = sys.stdout.getvalue()
+        self.assert_(check_expected(result,
+                                    "(TOK34,'TOK34:',1,0)\n"
+                                    "(TOK143,'TOK143:',1,7)\n"
+                                    "(TOK269,'TOK269:',1,15)\n"
+                                    "(TOK372,'TOK372:',1,23)\n"
+                                    "(TOK452,'TOK452:',1,31)\n"
+                                    "(TOK561,'TOK561:',1,39)\n"
+                                    "(TOK999,'TOK999:',1,47)\n"
+                                    ))
+
+        self.assert_(os.path.exists("manytab.py"))
+
+        p = subprocess.Popen([sys.executable,'-O','lex_many_tokens.py'],
+                             stdout=subprocess.PIPE)
+        result = p.stdout.read()
+        self.assert_(check_expected(result,
+                                    "(TOK34,'TOK34:',1,0)\n"
+                                    "(TOK143,'TOK143:',1,7)\n"
+                                    "(TOK269,'TOK269:',1,15)\n"
+                                    "(TOK372,'TOK372:',1,23)\n"
+                                    "(TOK452,'TOK452:',1,31)\n"
+                                    "(TOK561,'TOK561:',1,39)\n"
+                                    "(TOK999,'TOK999:',1,47)\n"
+                                    ))
+
+        self.assert_(os.path.exists("manytab.pyo"))
+        os.remove("manytab.pyo")
+        try:
+            os.remove("manytab.py")
+        except OSError:
+            pass
+        try:
+            os.remove("manytab.pyc")
+        except OSError:
+            pass
+        try:
+            os.remove("manytab.pyo")
+        except OSError:
+            pass
+        
+# Tests related to run-time behavior of lexers
+class LexRunTests(unittest.TestCase):
+    def setUp(self):
+        sys.stderr = StringIO.StringIO()
+        sys.stdout = StringIO.StringIO()
+    def tearDown(self):
+        sys.stderr = sys.__stderr__
+        sys.stdout = sys.__stdout__
+
+    def test_lex_hedit(self):
+        run_import("lex_hedit")
+        result = sys.stdout.getvalue()
+        self.assert_(check_expected(result,
+                                    "(H_EDIT_DESCRIPTOR,'abc',1,0)\n"
+                                    "(H_EDIT_DESCRIPTOR,'abcdefghij',1,6)\n"
+                                    "(H_EDIT_DESCRIPTOR,'xy',1,20)\n"))
+       
+    def test_lex_state_try(self):
+        run_import("lex_state_try")
+        result = sys.stdout.getvalue()
+        self.assert_(check_expected(result,
+                                    "(NUMBER,'3',1,0)\n"
+                                    "(PLUS,'+',1,2)\n"
+                                    "(NUMBER,'4',1,4)\n"
+                                    "Entering comment state\n"
+                                    "comment body LexToken(body_part,'This is a comment */',1,9)\n"
+                                    "(PLUS,'+',1,30)\n"
+                                    "(NUMBER,'10',1,32)\n"
+                                    ))    
+
+
+
+unittest.main()