My favorites | Sign in
Project Home Downloads Wiki Issues Source
Repository:
Checkout   Browse   Changes   Clones  
Changes to /src/lepl/regexp/_test/rewriters.py
13a59b816b7c vs. d93e5d662c5d Compare: vs.  Format:
Revision d93e5d662c5d
Go to: 
Project members, sign in to write a code review
/src/lepl/regexp/_test/rewriters.py   13a59b816b7c /src/lepl/regexp/_test/rewriters.py   d93e5d662c5d
1 from lepl.stream.core import DUMMY_HELPER 1 from lepl.stream.core import DUMMY_HELPER
2 2
3 # The contents of this file are subject to the Mozilla Public License 3 # The contents of this file are subject to the Mozilla Public License
4 # (MPL) Version 1.1 (the "License"); you may not use this file except 4 # (MPL) Version 1.1 (the "License"); you may not use this file except
5 # in compliance with the License. You may obtain a copy of the License 5 # in compliance with the License. You may obtain a copy of the License
6 # at http://www.mozilla.org/MPL/ 6 # at http://www.mozilla.org/MPL/
7 # 7 #
8 # Software distributed under the License is distributed on an "AS IS" 8 # Software distributed under the License is distributed on an "AS IS"
9 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 9 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
10 # the License for the specific language governing rights and 10 # the License for the specific language governing rights and
11 # limitations under the License. 11 # limitations under the License.
12 # 12 #
13 # The Original Code is LEPL (http://www.acooke.org/lepl) 13 # The Original Code is LEPL (http://www.acooke.org/lepl)
14 # The Initial Developer of the Original Code is Andrew Cooke. 14 # The Initial Developer of the Original Code is Andrew Cooke.
15 # Portions created by the Initial Developer are Copyright (C) 2009-2010 15 # Portions created by the Initial Developer are Copyright (C) 2009-2010
16 # Andrew Cooke (andrew@acooke.org). All Rights Reserved. 16 # Andrew Cooke (andrew@acooke.org). All Rights Reserved.
17 # 17 #
18 # Alternatively, the contents of this file may be used under the terms 18 # Alternatively, the contents of this file may be used under the terms
19 # of the LGPL license (the GNU Lesser General Public License, 19 # of the LGPL license (the GNU Lesser General Public License,
20 # http://www.gnu.org/licenses/lgpl.html), in which case the provisions 20 # http://www.gnu.org/licenses/lgpl.html), in which case the provisions
21 # of the LGPL License are applicable instead of those above. 21 # of the LGPL License are applicable instead of those above.
22 # 22 #
23 # If you wish to allow use of your version of this file only under the 23 # If you wish to allow use of your version of this file only under the
24 # terms of the LGPL License and not to allow others to use your version 24 # terms of the LGPL License and not to allow others to use your version
25 # of this file under the MPL, indicate your decision by deleting the 25 # of this file under the MPL, indicate your decision by deleting the
26 # provisions above and replace them with the notice and other provisions 26 # provisions above and replace them with the notice and other provisions
27 # required by the LGPL License. If you do not delete the provisions 27 # required by the LGPL License. If you do not delete the provisions
28 # above, a recipient may use your version of this file under either the 28 # above, a recipient may use your version of this file under either the
29 # MPL or the LGPL License. 29 # MPL or the LGPL License.
30 30
31 ''' 31 '''
32 Tests for the lepl.regexp.rewriters module. 32 Tests for the lepl.regexp.rewriters module.
33 ''' 33 '''
34 34
35 from logging import basicConfig, DEBUG 35 from logging import basicConfig, DEBUG
36 from string import ascii_letters 36 from string import ascii_letters
37 from unittest import TestCase 37 from unittest import TestCase
38 38
39 from lepl import * 39 from lepl import *
40 from lepl.regexp.rewriters import CompileRegexp 40 from lepl.regexp.rewriters import CompileRegexp
41 41
42 # pylint: disable-msg=C0103, C0111, C0301, C0324 42 # pylint: disable-msg=C0103, C0111, C0301, C0324
43 # (dude this is just a test) 43 # (dude this is just a test)
44 44
45 45
46 class RewriteTest(TestCase): 46 class RewriteTest(TestCase):
47 47
48 def test_any(self): 48 def test_any(self):
49 #basicConfig(level=DEBUG) 49 #basicConfig(level=DEBUG)
50 char = Any() 50 char = Any()
51 51
52 char.config.clear().compile_to_nfa(force=True).no_memoize() 52 char.config.clear().compile_to_nfa(force=True).no_memoize()
53 matcher = char.get_match_sequence() 53 matcher = char.get_match_sequence()
54 results = list(matcher('abc')) 54 results = list(matcher('abc'))
55 assert results == [(['a'], (1, DUMMY_HELPER))], results 55 assert results == [(['a'], (1, DUMMY_HELPER))], results
56 assert isinstance(matcher.matcher, NfaRegexp) 56 assert isinstance(matcher.matcher, NfaRegexp)
57 57
58 char.config.clear().compile_to_nfa(force=True).compose_transforms().no_memoize() 58 char.config.clear().compile_to_nfa(force=True).compose_transforms().no_memoize()
59 matcher = char.get_match_sequence() 59 matcher = char.get_match_sequence()
60 results = list(matcher('abc')) 60 results = list(matcher('abc'))
61 assert results == [(['a'], (1, DUMMY_HELPER))], results 61 assert results == [(['a'], (1, DUMMY_HELPER))], results
62 assert isinstance(matcher.matcher, NfaRegexp) 62 assert isinstance(matcher.matcher, NfaRegexp)
63 63
64 def test_or(self): 64 def test_or(self):
65 #basicConfig(level=DEBUG) 65 #basicConfig(level=DEBUG)
66 rx = Any('a') | Any('b') 66 rx = Any('a') | Any('b')
67 67
68 rx.config.clear().compile_to_nfa(force=True).no_memoize() 68 rx.config.clear().compile_to_nfa(force=True).no_memoize()
69 matcher = rx.get_match_sequence() 69 matcher = rx.get_match_sequence()
70 results = list(matcher('bq')) 70 results = list(matcher('bq'))
71 assert results == [(['b'], (1, DUMMY_HELPER))], results 71 assert results == [(['b'], (1, DUMMY_HELPER))], results
72 results = list(matcher('aq')) 72 results = list(matcher('aq'))
73 assert results == [(['a'], (1, DUMMY_HELPER))], results 73 assert results == [(['a'], (1, DUMMY_HELPER))], results
74 assert isinstance(matcher.matcher, NfaRegexp) 74 assert isinstance(matcher.matcher, NfaRegexp)
75 75
76 rx.config.clear().compile_to_nfa(force=True).compose_transforms().no_memoize() 76 rx.config.clear().compile_to_nfa(force=True).compose_transforms().no_memoize()
77 matcher = rx.get_match_sequence() 77 matcher = rx.get_match_sequence()
78 results = list(matcher('bq')) 78 results = list(matcher('bq'))
79 assert results == [(['b'], (1, DUMMY_HELPER))], results 79 assert results == [(['b'], (1, DUMMY_HELPER))], results
80 results = list(matcher('aq')) 80 results = list(matcher('aq'))
81 assert results == [(['a'], (1, DUMMY_HELPER))], results 81 assert results == [(['a'], (1, DUMMY_HELPER))], results
82 assert isinstance(matcher.matcher, NfaRegexp) 82 assert isinstance(matcher.matcher, NfaRegexp)
83 83
84 def test_plus(self): 84 def test_plus(self):
85 rx = Any('a') + Any('b') 85 rx = Any('a') + Any('b')
86 86
87 rx.config.clear().compile_to_nfa(force=True).no_memoize() 87 rx.config.clear().compile_to_nfa(force=True).no_memoize()
88 matcher = rx.get_match_sequence() 88 matcher = rx.get_match_sequence()
89 results = list(matcher('abq')) 89 results = list(matcher('abq'))
90 assert results == [(['ab'], (2, DUMMY_HELPER))], results 90 assert results == [(['ab'], (2, DUMMY_HELPER))], results
91 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 91 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
92 92
93 rx.config.clear().compile_to_nfa(force=True).compose_transforms() 93 rx.config.clear().compile_to_nfa(force=True).compose_transforms()
94 matcher = rx.get_match_sequence() 94 matcher = rx.get_match_sequence()
95 results = list(matcher('abq')) 95 results = list(matcher('abq'))
96 assert results == [(['ab'], (2, DUMMY_HELPER))], results 96 assert results == [(['ab'], (2, DUMMY_HELPER))], results
97 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 97 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
98 98
99 def test_add(self): 99 def test_add(self):
100 rx = Add(And(Any('a'), Any('b'))) 100 rx = Add(And(Any('a'), Any('b')))
101 101
102 rx.config.clear().compile_to_nfa(force=True).no_memoize() 102 rx.config.clear().compile_to_nfa(force=True).no_memoize()
103 matcher = rx.get_match_sequence() 103 matcher = rx.get_match_sequence()
104 results = list(matcher('abq')) 104 results = list(matcher('abq'))
105 assert results == [(['ab'], (2, DUMMY_HELPER))], results 105 assert results == [(['ab'], (2, DUMMY_HELPER))], results
106 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 106 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
107 rx.config.clear().compile_to_nfa(force=True).compose_transforms() 107 rx.config.clear().compile_to_nfa(force=True).compose_transforms()
108 108
109 matcher = rx.get_match_sequence() 109 matcher = rx.get_match_sequence()
110 results = list(matcher('abq')) 110 results = list(matcher('abq'))
111 assert results == [(['ab'], (2, DUMMY_HELPER))], results 111 assert results == [(['ab'], (2, DUMMY_HELPER))], results
112 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 112 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
113 113
114 def test_literal(self): 114 def test_literal(self):
115 rx = Literal('abc') 115 rx = Literal('abc')
116 116
117 rx.config.clear().compile_to_nfa(force=True).no_memoize() 117 rx.config.clear().compile_to_nfa(force=True).no_memoize()
118 matcher = rx.get_match_sequence() 118 matcher = rx.get_match_sequence()
119 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 119 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
120 results = list(matcher('abcd')) 120 results = list(matcher('abcd'))
121 assert results == [(['abc'], (3, DUMMY_HELPER))], results 121 assert results == [(['abc'], (3, DUMMY_HELPER))], results
122 122
123 rx.config.clear().compile_to_nfa(force=True).compose_transforms().no_memoize() 123 rx.config.clear().compile_to_nfa(force=True).compose_transforms().no_memoize()
124 matcher = rx.get_match_sequence() 124 matcher = rx.get_match_sequence()
125 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 125 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
126 results = list(matcher('abcd')) 126 results = list(matcher('abcd'))
127 assert results == [(['abc'], (3, DUMMY_HELPER))], results 127 assert results == [(['abc'], (3, DUMMY_HELPER))], results
128 128
129 rx = Literal('abc') >> (lambda x: x+'e') 129 rx = Literal('abc') >> (lambda x: x+'e')
130 130
131 rx.config.clear().compile_to_nfa(force=True).no_memoize() 131 rx.config.clear().compile_to_nfa(force=True).no_memoize()
132 matcher = rx.get_match_sequence() 132 matcher = rx.get_match_sequence()
133 print(matcher.matcher.tree()) 133 print(matcher.matcher.tree())
134 results = list(matcher('abcd')) 134 results = list(matcher('abcd'))
135 assert results == [(['abce'], (3, DUMMY_HELPER))], results 135 assert results == [(['abce'], (3, DUMMY_HELPER))], results
136 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 136 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
137 137
138 rx.config.clear().compile_to_nfa(force=True).compose_transforms().no_memoize() 138 rx.config.clear().compile_to_nfa(force=True).compose_transforms().no_memoize()
139 matcher = rx.get_match_sequence() 139 matcher = rx.get_match_sequence()
140 print(matcher.matcher.tree()) 140 print(matcher.matcher.tree())
141 results = list(matcher('abcd')) 141 results = list(matcher('abcd'))
142 assert results == [(['abce'], (3, DUMMY_HELPER))], results 142 assert results == [(['abce'], (3, DUMMY_HELPER))], results
143 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 143 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
144 144
145 def test_dfs(self): 145 def test_dfs(self):
146 expected = [(['abcd'], (4, DUMMY_HELPER)), 146 expected = [(['abcd'], (4, DUMMY_HELPER)),
147 (['abc'], (3, DUMMY_HELPER)), 147 (['abc'], (3, DUMMY_HELPER)),
148 (['ab'], (2, DUMMY_HELPER)), 148 (['ab'], (2, DUMMY_HELPER)),
149 (['a'], (1, DUMMY_HELPER)), 149 (['a'], (1, DUMMY_HELPER)),
150 ([], (0, DUMMY_HELPER))] 150 ([], (0, DUMMY_HELPER))]
151 rx = Any()[:, ...] 151 rx = Any()[:, ...]
152 152
153 # do un-rewritten to check whether [] or [''] is correct 153 # do un-rewritten to check whether [] or [''] is correct
154 rx.config.clear().no_memoize() 154 rx.config.clear().no_memoize()
155 matcher = rx.get_match_sequence() 155 matcher = rx.get_match_sequence()
156 results = list(matcher('abcd')) 156 results = list(matcher('abcd'))
157 assert results == expected, results 157 assert results == expected, results
158 158
159 rx.config.clear().compose_transforms().no_memoize() 159 rx.config.clear().compose_transforms().no_memoize()
160 matcher = rx.get_match_sequence() 160 matcher = rx.get_match_sequence()
161 results = list(matcher('abcd')) 161 results = list(matcher('abcd'))
162 assert results == expected, results 162 assert results == expected, results
163 163
164 #basicConfig(level=DEBUG) 164 #basicConfig(level=DEBUG)
165 rx.config.clear().compile_to_nfa().no_memoize() 165 rx.config.clear().compile_to_nfa().no_memoize()
166 matcher = rx.get_match_sequence() 166 matcher = rx.get_match_sequence()
167 results = list(matcher('abcd')) 167 results = list(matcher('abcd'))
168 assert results == expected, results 168 assert results == expected, results
169 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 169 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
170 170
171 rx.config.clear().compile_to_nfa().compose_transforms().no_memoize() 171 rx.config.clear().compile_to_nfa().compose_transforms().no_memoize()
172 matcher = rx.get_match_sequence() 172 matcher = rx.get_match_sequence()
173 results = list(matcher('abcd')) 173 results = list(matcher('abcd'))
174 assert results == expected, results 174 assert results == expected, results
175 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 175 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
176 176
177 def test_complex(self): 177 def test_complex(self):
178 #basicConfig(level=DEBUG) 178 #basicConfig(level=DEBUG)
179 rx = Literal('foo') | (Literal('ba') + Any('a')[1:,...]) 179 rx = Literal('foo') | (Literal('ba') + Any('a')[1:,...])
180 180
181 rx.config.compile_to_nfa().no_full_first_match().no_memoize() 181 rx.config.compile_to_nfa().no_full_first_match().no_memoize()
182 matcher = rx.get_match_sequence() 182 matcher = rx.get_match_sequence()
183 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 183 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
184 results = list(matcher('foo')) 184 results = list(matcher('foo'))
185 assert results == [(['foo'], (3, DUMMY_HELPER))], results 185 assert results == [(['foo'], (3, DUMMY_HELPER))], results
186 results = list(matcher('baaaaax')) 186 results = list(matcher('baaaaax'))
187 assert results == [(['baaaaa'], (6, DUMMY_HELPER)), 187 assert results == [(['baaaaa'], (6, DUMMY_HELPER)),
188 (['baaaa'], (5, DUMMY_HELPER)), 188 (['baaaa'], (5, DUMMY_HELPER)),
189 (['baaa'], (4, DUMMY_HELPER)), 189 (['baaa'], (4, DUMMY_HELPER)),
190 (['baa'], (3, DUMMY_HELPER))], results 190 (['baa'], (3, DUMMY_HELPER))], results
191 results = list(matcher('ba')) 191 results = list(matcher('ba'))
192 assert results == [], results 192 assert results == [], results
193 193
194 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize() 194 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize()
195 matcher = rx.get_match_sequence() 195 matcher = rx.get_match_sequence()
196 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 196 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
197 results = list(matcher('foo')) 197 results = list(matcher('foo'))
198 assert results == [(['foo'], (3, DUMMY_HELPER))], results 198 assert results == [(['foo'], (3, DUMMY_HELPER))], results
199 results = list(matcher('baaaaax')) 199 results = list(matcher('baaaaax'))
200 assert results == [(['baaaaa'], (6, DUMMY_HELPER)), 200 assert results == [(['baaaaa'], (6, DUMMY_HELPER)),
201 (['baaaa'], (5, DUMMY_HELPER)), 201 (['baaaa'], (5, DUMMY_HELPER)),
202 (['baaa'], (4, DUMMY_HELPER)), 202 (['baaa'], (4, DUMMY_HELPER)),
203 (['baa'], (3, DUMMY_HELPER))], results 203 (['baa'], (3, DUMMY_HELPER))], results
204 results = list(matcher('ba')) 204 results = list(matcher('ba'))
205 assert results == [], results 205 assert results == [], results
206 206
207 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize() 207 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize()
208 matcher = rx.get_match_sequence() 208 matcher = rx.get_match_sequence()
209 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 209 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
210 results = list(matcher('foo')) 210 results = list(matcher('foo'))
211 assert results == [(['foo'], (3, DUMMY_HELPER))], results 211 assert results == [(['foo'], (3, DUMMY_HELPER))], results
212 results = list(matcher('baaaaax')) 212 results = list(matcher('baaaaax'))
213 assert results == [(['baaaaa'], (6, DUMMY_HELPER)), 213 assert results == [(['baaaaa'], (6, DUMMY_HELPER)),
214 (['baaaa'], (5, DUMMY_HELPER)), 214 (['baaaa'], (5, DUMMY_HELPER)),
215 (['baaa'], (4, DUMMY_HELPER)), 215 (['baaa'], (4, DUMMY_HELPER)),
216 (['baa'], (3, DUMMY_HELPER))], results 216 (['baa'], (3, DUMMY_HELPER))], results
217 results = list(matcher('ba')) 217 results = list(matcher('ba'))
218 assert results == [], results 218 assert results == [], results
219 219
220 def test_integer(self): 220 def test_integer(self):
221 rx = Integer() 221 rx = Integer()
222 222
223 rx.config.compile_to_nfa().no_full_first_match().no_memoize() 223 rx.config.compile_to_nfa().no_full_first_match().no_memoize()
224 matcher = rx.get_match_sequence() 224 matcher = rx.get_match_sequence()
225 results = list(matcher('12x')) 225 results = list(matcher('12x'))
226 assert results == [(['12'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results 226 assert results == [(['12'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results
227 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 227 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
228 228
229 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize() 229 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize()
230 matcher = rx.get_match_sequence() 230 matcher = rx.get_match_sequence()
231 results = list(matcher('12x')) 231 results = list(matcher('12x'))
232 assert results == [(['12'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results 232 assert results == [(['12'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results
233 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 233 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
234 234
235 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize() 235 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize()
236 matcher = rx.get_match_sequence() 236 matcher = rx.get_match_sequence()
237 results = list(matcher('12x')) 237 results = list(matcher('12x'))
238 assert results == [(['12'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results 238 assert results == [(['12'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results
239 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 239 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
240 240
241 def test_real(self): 241 def test_real(self):
242 rx = Real() 242 rx = Real()
243 243
244 rx.config.compile_to_nfa().no_full_first_match().no_memoize() 244 rx.config.compile_to_nfa().no_full_first_match().no_memoize()
245 matcher = rx.get_match_sequence() 245 matcher = rx.get_match_sequence()
246 results = list(matcher('1.2x')) 246 results = list(matcher('1.2x'))
247 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results 247 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results
248 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 248 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
249 249
250 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize() 250 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize()
251 matcher = rx.get_match_sequence() 251 matcher = rx.get_match_sequence()
252 results = list(matcher('1.2x')) 252 results = list(matcher('1.2x'))
253 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results 253 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results
254 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 254 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
255 255
256 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize() 256 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize()
257 matcher = rx.get_match_sequence() 257 matcher = rx.get_match_sequence()
258 results = list(matcher('1.2x')) 258 results = list(matcher('1.2x'))
259 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results 259 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results
260 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 260 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
261 261
262 def test_float(self): 262 def test_float(self):
263 rx = Float() 263 rx = Float()
264 264
265 rx.config.compile_to_nfa().no_full_first_match().no_memoize() 265 rx.config.compile_to_nfa().no_full_first_match().no_memoize()
266 matcher = rx.get_match_sequence() 266 matcher = rx.get_match_sequence()
267 results = list(matcher('1.2x')) 267 results = list(matcher('1.2x'))
268 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER))], results 268 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER))], results
269 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 269 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
270 270
271 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize() 271 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize()
272 matcher = rx.get_match_sequence() 272 matcher = rx.get_match_sequence()
273 results = list(matcher('1.2x')) 273 results = list(matcher('1.2x'))
274 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER))], results 274 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER))], results
275 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree().no_memoize() 275 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree().no_memoize()
276 276
277 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms() 277 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms()
278 matcher = rx.get_match_sequence() 278 matcher = rx.get_match_sequence()
279 results = list(matcher('1.2x')) 279 results = list(matcher('1.2x'))
280 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER))], results 280 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER))], results
281 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree().no_memoize() 281 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree().no_memoize()
282 282
283 def test_star(self): 283 def test_star(self):
284 rx = Add(Star('a')) 284 rx = Add(Star('a'))
285 285
286 rx.config.compile_to_nfa().no_full_first_match().no_memoize() 286 rx.config.compile_to_nfa().no_full_first_match().no_memoize()
287 matcher = rx.get_match_sequence() 287 matcher = rx.get_match_sequence()
288 results = list(matcher('aa')) 288 results = list(matcher('aa'))
289 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER)), ([], (0, DUMMY_HELPER))], results 289 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER)), ([], (0, DUMMY_HELPER))], results
290 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 290 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
291 291
292 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize() 292 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize()
293 matcher = rx.get_match_sequence() 293 matcher = rx.get_match_sequence()
294 results = list(matcher('aa')) 294 results = list(matcher('aa'))
295 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER)), ([], (0, DUMMY_HELPER))], results 295 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER)), ([], (0, DUMMY_HELPER))], results
296 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 296 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
297 297
298 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize() 298 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize()
299 matcher = rx.get_match_sequence() 299 matcher = rx.get_match_sequence()
300 results = list(matcher('aa')) 300 results = list(matcher('aa'))
301 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER)), ([], (0, DUMMY_HELPER))], results 301 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER)), ([], (0, DUMMY_HELPER))], results
302 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 302 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
303 303
304 def test_word(self): 304 def test_word(self):
305 #basicConfig(level=DEBUG) 305 #basicConfig(level=DEBUG)
306 rx = Word('a') 306 rx = Word('a')
307 307
308 rx.config.compile_to_nfa().no_full_first_match().no_memoize() 308 rx.config.compile_to_nfa().no_full_first_match().no_memoize()
309 matcher = rx.get_match_sequence() 309 matcher = rx.get_match_sequence()
310 results = list(matcher('aa')) 310 results = list(matcher('aa'))
311 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER))], results 311 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER))], results
312 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 312 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
313 313
314 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize() 314 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize()
315 matcher = rx.get_match_sequence() 315 matcher = rx.get_match_sequence()
316 results = list(matcher('aa')) 316 results = list(matcher('aa'))
317 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER))], results 317 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER))], results
318 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 318 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
319 319
320 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize() 320 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize()
321 matcher = rx.get_match_sequence() 321 matcher = rx.get_match_sequence()
322 results = list(matcher('aa')) 322 results = list(matcher('aa'))
323 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER))], results 323 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER))], results
324 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 324 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
325 325
326 326
327 class CompileTest(TestCase): 327 class CompileTest(TestCase):
328 ''' 328 '''
329 Test the rewrite routine directly. 329 Test the rewrite routine directly.
330 ''' 330 '''
331 331
332 def assert_regexp(self, matcher, regexp): 332 def assert_regexp(self, matcher, regexp):
333 compiler = CompileRegexp(use=True) 333 compiler = CompileRegexp(use=True)
334 matcher = compiler(matcher) 334 matcher = compiler(matcher)
335 assert isinstance(matcher, NfaRegexp), matcher.tree() 335 assert isinstance(matcher, NfaRegexp), matcher.tree()
336 assert str(matcher.regexp) == regexp, matcher.regexp 336 assert str(matcher.regexp) == regexp, matcher.regexp
337 337
338 def test_any(self): 338 def test_any(self):
339 self.assert_regexp(Any(), '.') 339 self.assert_regexp(Any(), '.')
340 self.assert_regexp(Any('abc'), '[a-c]') 340 self.assert_regexp(Any('abc'), '[a-c]')
341 341
342 def test_literal(self): 342 def test_literal(self):
343 self.assert_regexp(Literal('foo'), 'foo') 343 self.assert_regexp(Literal('foo'), 'foo')
344 344
345 def test_repeat(self): 345 def test_repeat(self):
346 self.assert_regexp(Any()[:, ...], '(?:.)*') 346 self.assert_regexp(Any()[:, ...], '(?:.)*')
347 self.assert_regexp(Any()[1:, ...], '.(?:.)*') 347 self.assert_regexp(Any()[1:, ...], '.(?:.)*')
348 self.assert_regexp(Any()[1, ...], '.') 348 self.assert_regexp(Any()[1, ...], '.')
349 self.assert_regexp(Any()[1:2, ...], '.(?:.)?') 349 self.assert_regexp(Any()[1:2, ...], '.(?:.)?')
350 self.assert_regexp(Any()[2, ...], '..') 350 self.assert_regexp(Any()[2, ...], '..')
351 self.assert_regexp(Any()[2:4, ...], '..(?:.(?:.)?)?') 351 self.assert_regexp(Any()[2:4, ...], '..(?:.(?:.)?)?')
352 self.assert_regexp(Any()[:, 'x', ...], '(?:.(?:x.)*|)') 352 self.assert_regexp(Any()[:, 'x', ...], '(?:.(?:x.)*|)')
353 self.assert_regexp(Any()[1:, 'x', ...], '.(?:x.)*') 353 self.assert_regexp(Any()[1:, 'x', ...], '.(?:x.)*')
354 self.assert_regexp(Any()[1, 'x', ...], '.') 354 self.assert_regexp(Any()[1, 'x', ...], '.')
355 self.assert_regexp(Any()[1:2, 'x', ...], '.(?:x.)?') 355 self.assert_regexp(Any()[1:2, 'x', ...], '.(?:x.)?')
356 self.assert_regexp(Any()[2, 'x', ...], '.x.') 356 self.assert_regexp(Any()[2, 'x', ...], '.x.')
357 self.assert_regexp(Any()[2:4, 'x', ...], '.x.(?:x.(?:x.)?)?') 357 self.assert_regexp(Any()[2:4, 'x', ...], '.x.(?:x.(?:x.)?)?')
358 self.assert_regexp(Literal('foo')[:, ...], '(?:foo)*') 358 self.assert_regexp(Literal('foo')[:, ...], '(?:foo)*')
359 359
360 def test_and(self): 360 def test_and(self):
361 self.assert_regexp(Any('ab')[:, ...] + Any('p'), '(?:[a-b])*p') 361 self.assert_regexp(Any('ab')[:, ...] + Any('p'), '(?:[a-b])*p')
362 362
363 def test_or(self): 363 def test_or(self):
364 self.assert_regexp(Any('ab')[:, ...] | Any('p'), '(?:(?:[a-b])*|p)') 364 self.assert_regexp(Any('ab')[:, ...] | Any('p'), '(?:(?:[a-b])*|p)')
365 365
366 def test_complex(self): 366 def test_complex(self):
367 self.assert_regexp((Any('ab') + Literal('q')) | Literal('z'), '(?:[a-b]q|z)') 367 self.assert_regexp((Any('ab') + Literal('q')) | Literal('z'), '(?:[a-b]q|z)')
368 self.assert_regexp((Any('ab') + 'q') | 'z', '(?:[a-b]q|z)') 368 self.assert_regexp((Any('ab') + 'q') | 'z', '(?:[a-b]q|z)')
369 369
370 370
371 class RepeatBugTest(TestCase): 371 class RepeatBugTest(TestCase):
372 372
373 def test_bug(self): 373 def test_bug(self):
374 #basicConfig(level=DEBUG) 374 #basicConfig(level=DEBUG)
375 matcher = Any()[2, ...] 375 matcher = Any()[2, ...]
376 matcher.config.no_full_first_match().compile_to_nfa() 376 matcher.config.no_full_first_match().compile_to_nfa()
377 parser = matcher.get_parse_all() 377 parser = matcher.get_parse_all()
378 results = list(parser('abc')) 378 results = list(parser('abc'))
379 assert results == [['ab']], results 379 assert results == [['ab']], results
380 380
381 def test_bug2(self): 381 def test_bug2(self):
382 matcher = NfaRegexp('..') 382 matcher = NfaRegexp('..')
383 matcher.config.no_full_first_match() 383 matcher.config.no_full_first_match()
384 parser = matcher.get_parse_all() 384 parser = matcher.get_parse_all()
385 results = list(parser('abc')) 385 results = list(parser('abc'))
386 assert results == [['ab']], results 386 assert results == [['ab']], results
387 387
388 388
389 class WordBugTest(TestCase): 389 class WordBugTest(TestCase):
390 ''' 390 '''
391 Used to not be possible to compile a raw Word() 391 Used to not be possible to compile a raw Word()
392 ''' 392 '''
393 393
394 def test_word(self): 394 def test_word(self):
395 #basicConfig(level=DEBUG) 395 #basicConfig(level=DEBUG)
396 rx = Word() 396 rx = Word()
397 397
398 rx.config.compile_to_nfa().no_full_first_match().no_memoize() 398 rx.config.compile_to_nfa().no_full_first_match().no_memoize()
399 matcher = rx.get_match_sequence() 399 matcher = rx.get_match_sequence()
400 results = list(matcher('aa')) 400 results = list(matcher('aa'))
401 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER))], results 401 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER))], results
402 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 402 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
403 403
404 404
405 class TokenBugTest(TestCase): 405 class TokenBugTest(TestCase):
406 ''' 406 '''
407 Token(Word()) and Token(Any(ascii_letters)[1:]) gave errors. 407 Token(Word()) and Token(Any(ascii_letters)[1:]) gave errors.
408 ''' 408 '''
409 409
410 def test_token_word(self): 410 def test_token_word(self):
411 tk = Token(Word()) 411 tk = Token(Word())
412 tk.config.offside(block_policy=explicit) 412 tk.config.lines(block_policy=explicit)
413 tk.get_parse() 413 tk.get_parse()
414 414
415 def test_token_any(self): 415 def test_token_any(self):
416 tk = Token(Any(ascii_letters)[1:,...]) 416 tk = Token(Any(ascii_letters)[1:,...])
417 tk.config.offside(block_policy=explicit) 417 tk.config.lines(block_policy=explicit)
418 tk.get_parse() 418 tk.get_parse()
419 419
420 def test_simple_word(self): 420 def test_simple_word(self):
421 #basicConfig(level=DEBUG) 421 #basicConfig(level=DEBUG)
422 rx = Word() 422 rx = Word()
423 rx.config.no_memoize() 423 rx.config.no_memoize()
424 matcher = rx.get_parse().matcher 424 matcher = rx.get_parse().matcher
425 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 425 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
426 426
427 def test_simple_any(self): 427 def test_simple_any(self):
428 rx = Any(ascii_letters)[1:,...] 428 rx = Any(ascii_letters)[1:,...]
429 rx.config.compile_to_nfa().no_memoize() 429 rx.config.compile_to_nfa().no_memoize()
430 matcher = rx.get_parse().matcher 430 matcher = rx.get_parse().matcher
431 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 431 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
432 432
Powered by Google Project Hosting