summaryrefslogtreecommitdiff
path: root/ext/ply/example/yply/ylex.py
blob: 61bc0c7ef99cecd1edf4b26108ed30333702dbda (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
# lexer for yacc-grammars
#
# Author: David Beazley (dave@dabeaz.com)
# Date  : October 2, 2006

import sys
sys.path.append("../..")

from ply import *

tokens = (
    'LITERAL','SECTION','TOKEN','LEFT','RIGHT','PREC','START','TYPE','NONASSOC','UNION','CODE',
    'ID','QLITERAL','NUMBER',
)

states = (('code','exclusive'),)

literals = [ ';', ',', '<', '>', '|',':' ]
t_ignore = ' \t'

t_TOKEN     = r'%token'
t_LEFT      = r'%left'
t_RIGHT     = r'%right'
t_NONASSOC  = r'%nonassoc'
t_PREC      = r'%prec'
t_START     = r'%start'
t_TYPE      = r'%type'
t_UNION     = r'%union'
t_ID        = r'[a-zA-Z_][a-zA-Z_0-9]*'
t_QLITERAL  = r'''(?P<quote>['"]).*?(?P=quote)'''
t_NUMBER    = r'\d+'

def t_SECTION(t):
    r'%%'
    if getattr(t.lexer,"lastsection",0):
         t.value = t.lexer.lexdata[t.lexpos+2:]
         t.lexer.lexpos = len(t.lexer.lexdata)
    else:
         t.lexer.lastsection = 0
    return t

# Comments
def t_ccomment(t):
    r'/\*(.|\n)*?\*/'
    t.lineno += t.value.count('\n')

t_ignore_cppcomment = r'//.*'

def t_LITERAL(t):
   r'%\{(.|\n)*?%\}'
   t.lexer.lineno += t.value.count("\n")
   return t

def t_NEWLINE(t):
   r'\n'
   t.lexer.lineno += 1

def t_code(t):
   r'\{'
   t.lexer.codestart = t.lexpos
   t.lexer.level = 1
   t.lexer.begin('code')

def t_code_ignore_string(t):
    r'\"([^\\\n]|(\\.))*?\"'

def t_code_ignore_char(t):
    r'\'([^\\\n]|(\\.))*?\''

def t_code_ignore_comment(t):
   r'/\*(.|\n)*?\*/'

def t_code_ignore_cppcom(t):
   r'//.*'

def t_code_lbrace(t):
    r'\{'
    t.lexer.level += 1

def t_code_rbrace(t):
    r'\}'
    t.lexer.level -= 1
    if t.lexer.level == 0:
         t.type = 'CODE'
         t.value = t.lexer.lexdata[t.lexer.codestart:t.lexpos+1]
         t.lexer.begin('INITIAL')
         t.lexer.lineno += t.value.count('\n')
         return t

t_code_ignore_nonspace   = r'[^\s\}\'\"\{]+'
t_code_ignore_whitespace = r'\s+'
t_code_ignore = ""

def t_code_error(t):
    raise RuntimeError

def t_error(t):
    print "%d: Illegal character '%s'" % (t.lineno, t.value[0])
    print t.value
    t.lexer.skip(1)

lex.lex()

if __name__ == '__main__':
    lex.runmain()