My favorites | Sign in
Project Home Downloads Wiki Issues Source
Repository:
Checkout   Browse   Changes   Clones  
Changes to /src/lepl/regexp/_test/rewriters.py
f5a97799d820 vs. 1165a341b0b8 Compare: vs.  Format:
Revision 1165a341b0b8
Go to: 
Project members, sign in to write a code review
/src/lepl/regexp/_test/rewriters.py   f5a97799d820 /src/lepl/regexp/_test/rewriters.py   1165a341b0b8
1
2 # The contents of this file are subject to the Mozilla Public License 1 # The contents of this file are subject to the Mozilla Public License
3 # (MPL) Version 1.1 (the "License"); you may not use this file except 2 # (MPL) Version 1.1 (the "License"); you may not use this file except
4 # in compliance with the License. You may obtain a copy of the License 3 # in compliance with the License. You may obtain a copy of the License
5 # at http://www.mozilla.org/MPL/ 4 # at http://www.mozilla.org/MPL/
6 # 5 #
7 # Software distributed under the License is distributed on an "AS IS" 6 # Software distributed under the License is distributed on an "AS IS"
8 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 7 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
9 # the License for the specific language governing rights and 8 # the License for the specific language governing rights and
10 # limitations under the License. 9 # limitations under the License.
11 # 10 #
12 # The Original Code is LEPL (http://www.acooke.org/lepl) 11 # The Original Code is LEPL (http://www.acooke.org/lepl)
13 # The Initial Developer of the Original Code is Andrew Cooke. 12 # The Initial Developer of the Original Code is Andrew Cooke.
14 # Portions created by the Initial Developer are Copyright (C) 2009-2010 13 # Portions created by the Initial Developer are Copyright (C) 2009-2010
15 # Andrew Cooke (andrew@acooke.org). All Rights Reserved. 14 # Andrew Cooke (andrew@acooke.org). All Rights Reserved.
16 # 15 #
17 # Alternatively, the contents of this file may be used under the terms 16 # Alternatively, the contents of this file may be used under the terms
18 # of the LGPL license (the GNU Lesser General Public License, 17 # of the LGPL license (the GNU Lesser General Public License,
19 # http://www.gnu.org/licenses/lgpl.html), in which case the provisions 18 # http://www.gnu.org/licenses/lgpl.html), in which case the provisions
20 # of the LGPL License are applicable instead of those above. 19 # of the LGPL License are applicable instead of those above.
21 # 20 #
22 # If you wish to allow use of your version of this file only under the 21 # If you wish to allow use of your version of this file only under the
23 # terms of the LGPL License and not to allow others to use your version 22 # terms of the LGPL License and not to allow others to use your version
24 # of this file under the MPL, indicate your decision by deleting the 23 # of this file under the MPL, indicate your decision by deleting the
25 # provisions above and replace them with the notice and other provisions 24 # provisions above and replace them with the notice and other provisions
26 # required by the LGPL License. If you do not delete the provisions 25 # required by the LGPL License. If you do not delete the provisions
27 # above, a recipient may use your version of this file under either the 26 # above, a recipient may use your version of this file under either the
28 # MPL or the LGPL License. 27 # MPL or the LGPL License.
29 28
30 ''' 29 '''
31 Tests for the lepl.regexp.rewriters module. 30 Tests for the lepl.regexp.rewriters module.
32 ''' 31 '''
33 32
34 from logging import basicConfig, DEBUG 33 from logging import basicConfig, DEBUG
35 from string import ascii_letters 34 from string import ascii_letters
36 from unittest import TestCase 35 from unittest import TestCase
37 36
38 from lepl import * 37 from lepl import *
39 from lepl.regexp.rewriters import CompileRegexp 38 from lepl.regexp.rewriters import CompileRegexp
40 from lepl.stream.core import DUMMY_HELPER 39 from lepl.stream.core import DUMMY_HELPER
41 40
42 # pylint: disable-msg=C0103, C0111, C0301, C0324 41 # pylint: disable-msg=C0103, C0111, C0301, C0324
43 # (dude this is just a test) 42 # (dude this is just a test)
44 43
45 44
46 class RewriteTest(TestCase): 45 class RewriteTest(TestCase):
47 46
48 def test_any(self): 47 def test_any(self):
49 #basicConfig(level=DEBUG) 48 #basicConfig(level=DEBUG)
50 char = Any() 49 char = Any()
51 50
52 char.config.clear().compile_to_nfa(force=True).no_memoize() 51 char.config.clear().compile_to_nfa(force=True).no_memoize()
53 matcher = char.get_match_sequence() 52 matcher = char.get_match_sequence()
54 results = list(matcher('abc')) 53 results = list(matcher('abc'))
55 assert results == [(['a'], (1, DUMMY_HELPER))], results 54 assert results == [(['a'], (1, DUMMY_HELPER))], results
56 assert isinstance(matcher.matcher, NfaRegexp) 55 assert isinstance(matcher.matcher, NfaRegexp)
57 56
58 char.config.clear().compile_to_nfa(force=True).compose_transforms().no_memoize() 57 char.config.clear().compile_to_nfa(force=True).compose_transforms().no_memoize()
59 matcher = char.get_match_sequence() 58 matcher = char.get_match_sequence()
60 results = list(matcher('abc')) 59 results = list(matcher('abc'))
61 assert results == [(['a'], (1, DUMMY_HELPER))], results 60 assert results == [(['a'], (1, DUMMY_HELPER))], results
62 assert isinstance(matcher.matcher, NfaRegexp) 61 assert isinstance(matcher.matcher, NfaRegexp)
63 62
64 def test_or(self): 63 def test_or(self):
65 #basicConfig(level=DEBUG) 64 #basicConfig(level=DEBUG)
66 rx = Any('a') | Any('b') 65 rx = Any('a') | Any('b')
67 66
68 rx.config.clear().compile_to_nfa(force=True).no_memoize() 67 rx.config.clear().compile_to_nfa(force=True).no_memoize()
69 matcher = rx.get_match_sequence() 68 matcher = rx.get_match_sequence()
70 results = list(matcher('bq')) 69 results = list(matcher('bq'))
71 assert results == [(['b'], (1, DUMMY_HELPER))], results 70 assert results == [(['b'], (1, DUMMY_HELPER))], results
72 results = list(matcher('aq')) 71 results = list(matcher('aq'))
73 assert results == [(['a'], (1, DUMMY_HELPER))], results 72 assert results == [(['a'], (1, DUMMY_HELPER))], results
74 assert isinstance(matcher.matcher, NfaRegexp) 73 assert isinstance(matcher.matcher, NfaRegexp)
75 74
76 rx.config.clear().compile_to_nfa(force=True).compose_transforms().no_memoize() 75 rx.config.clear().compile_to_nfa(force=True).compose_transforms().no_memoize()
77 matcher = rx.get_match_sequence() 76 matcher = rx.get_match_sequence()
78 results = list(matcher('bq')) 77 results = list(matcher('bq'))
79 assert results == [(['b'], (1, DUMMY_HELPER))], results 78 assert results == [(['b'], (1, DUMMY_HELPER))], results
80 results = list(matcher('aq')) 79 results = list(matcher('aq'))
81 assert results == [(['a'], (1, DUMMY_HELPER))], results 80 assert results == [(['a'], (1, DUMMY_HELPER))], results
82 assert isinstance(matcher.matcher, NfaRegexp) 81 assert isinstance(matcher.matcher, NfaRegexp)
83 82
84 def test_plus(self): 83 def test_plus(self):
85 rx = Any('a') + Any('b') 84 rx = Any('a') + Any('b')
86 85
87 rx.config.clear().compile_to_nfa(force=True).no_memoize() 86 rx.config.clear().compile_to_nfa(force=True).no_memoize()
88 matcher = rx.get_match_sequence() 87 matcher = rx.get_match_sequence()
89 results = list(matcher('abq')) 88 results = list(matcher('abq'))
90 assert results == [(['ab'], (2, DUMMY_HELPER))], results 89 assert results == [(['ab'], (2, DUMMY_HELPER))], results
91 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 90 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
92 91
93 rx.config.clear().compile_to_nfa(force=True).compose_transforms() 92 rx.config.clear().compile_to_nfa(force=True).compose_transforms()
94 matcher = rx.get_match_sequence() 93 matcher = rx.get_match_sequence()
95 results = list(matcher('abq')) 94 results = list(matcher('abq'))
96 assert results == [(['ab'], (2, DUMMY_HELPER))], results 95 assert results == [(['ab'], (2, DUMMY_HELPER))], results
97 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 96 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
98 97
99 def test_add(self): 98 def test_add(self):
100 rx = Add(And(Any('a'), Any('b'))) 99 rx = Add(And(Any('a'), Any('b')))
101 100
102 rx.config.clear().compile_to_nfa(force=True).no_memoize() 101 rx.config.clear().compile_to_nfa(force=True).no_memoize()
103 matcher = rx.get_match_sequence() 102 matcher = rx.get_match_sequence()
104 results = list(matcher('abq')) 103 results = list(matcher('abq'))
105 assert results == [(['ab'], (2, DUMMY_HELPER))], results 104 assert results == [(['ab'], (2, DUMMY_HELPER))], results
106 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 105 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
107 rx.config.clear().compile_to_nfa(force=True).compose_transforms() 106 rx.config.clear().compile_to_nfa(force=True).compose_transforms()
108 107
109 matcher = rx.get_match_sequence() 108 matcher = rx.get_match_sequence()
110 results = list(matcher('abq')) 109 results = list(matcher('abq'))
111 assert results == [(['ab'], (2, DUMMY_HELPER))], results 110 assert results == [(['ab'], (2, DUMMY_HELPER))], results
112 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 111 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
113 112
114 def test_literal(self): 113 def test_literal(self):
115 rx = Literal('abc') 114 rx = Literal('abc')
116 115
117 rx.config.clear().compile_to_nfa(force=True).no_memoize() 116 rx.config.clear().compile_to_nfa(force=True).no_memoize()
118 matcher = rx.get_match_sequence() 117 matcher = rx.get_match_sequence()
119 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 118 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
120 results = list(matcher('abcd')) 119 results = list(matcher('abcd'))
121 assert results == [(['abc'], (3, DUMMY_HELPER))], results 120 assert results == [(['abc'], (3, DUMMY_HELPER))], results
122 121
123 rx.config.clear().compile_to_nfa(force=True).compose_transforms().no_memoize() 122 rx.config.clear().compile_to_nfa(force=True).compose_transforms().no_memoize()
124 matcher = rx.get_match_sequence() 123 matcher = rx.get_match_sequence()
125 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 124 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
126 results = list(matcher('abcd')) 125 results = list(matcher('abcd'))
127 assert results == [(['abc'], (3, DUMMY_HELPER))], results 126 assert results == [(['abc'], (3, DUMMY_HELPER))], results
128 127
129 rx = Literal('abc') >> (lambda x: x+'e') 128 rx = Literal('abc') >> (lambda x: x+'e')
130 129
131 rx.config.clear().compile_to_nfa(force=True).no_memoize() 130 rx.config.clear().compile_to_nfa(force=True).no_memoize()
132 matcher = rx.get_match_sequence() 131 matcher = rx.get_match_sequence()
133 print(matcher.matcher.tree()) 132 print(matcher.matcher.tree())
134 results = list(matcher('abcd')) 133 results = list(matcher('abcd'))
135 assert results == [(['abce'], (3, DUMMY_HELPER))], results 134 assert results == [(['abce'], (3, DUMMY_HELPER))], results
136 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 135 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
137 136
138 rx.config.clear().compile_to_nfa(force=True).compose_transforms().no_memoize() 137 rx.config.clear().compile_to_nfa(force=True).compose_transforms().no_memoize()
139 matcher = rx.get_match_sequence() 138 matcher = rx.get_match_sequence()
140 print(matcher.matcher.tree()) 139 print(matcher.matcher.tree())
141 results = list(matcher('abcd')) 140 results = list(matcher('abcd'))
142 assert results == [(['abce'], (3, DUMMY_HELPER))], results 141 assert results == [(['abce'], (3, DUMMY_HELPER))], results
143 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 142 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
144 143
145 def test_dfs(self): 144 def test_dfs(self):
146 expected = [(['abcd'], (4, DUMMY_HELPER)), 145 expected = [(['abcd'], (4, DUMMY_HELPER)),
147 (['abc'], (3, DUMMY_HELPER)), 146 (['abc'], (3, DUMMY_HELPER)),
148 (['ab'], (2, DUMMY_HELPER)), 147 (['ab'], (2, DUMMY_HELPER)),
149 (['a'], (1, DUMMY_HELPER)), 148 (['a'], (1, DUMMY_HELPER)),
150 ([], (0, DUMMY_HELPER))] 149 ([], (0, DUMMY_HELPER))]
151 rx = Any()[:, ...] 150 rx = Any()[:, ...]
152 151
153 # do un-rewritten to check whether [] or [''] is correct 152 # do un-rewritten to check whether [] or [''] is correct
154 rx.config.clear().no_memoize() 153 rx.config.clear().no_memoize()
155 matcher = rx.get_match_sequence() 154 matcher = rx.get_match_sequence()
156 results = list(matcher('abcd')) 155 results = list(matcher('abcd'))
157 assert results == expected, results 156 assert results == expected, results
158 157
159 rx.config.clear().compose_transforms().no_memoize() 158 rx.config.clear().compose_transforms().no_memoize()
160 matcher = rx.get_match_sequence() 159 matcher = rx.get_match_sequence()
161 results = list(matcher('abcd')) 160 results = list(matcher('abcd'))
162 assert results == expected, results 161 assert results == expected, results
163 162
164 #basicConfig(level=DEBUG) 163 #basicConfig(level=DEBUG)
165 rx.config.clear().compile_to_nfa().no_memoize() 164 rx.config.clear().compile_to_nfa().no_memoize()
166 matcher = rx.get_match_sequence() 165 matcher = rx.get_match_sequence()
167 results = list(matcher('abcd')) 166 results = list(matcher('abcd'))
168 assert results == expected, results 167 assert results == expected, results
169 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 168 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
170 169
171 rx.config.clear().compile_to_nfa().compose_transforms().no_memoize() 170 rx.config.clear().compile_to_nfa().compose_transforms().no_memoize()
172 matcher = rx.get_match_sequence() 171 matcher = rx.get_match_sequence()
173 results = list(matcher('abcd')) 172 results = list(matcher('abcd'))
174 assert results == expected, results 173 assert results == expected, results
175 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 174 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
176 175
177 def test_complex(self): 176 def test_complex(self):
178 #basicConfig(level=DEBUG) 177 #basicConfig(level=DEBUG)
179 rx = Literal('foo') | (Literal('ba') + Any('a')[1:,...]) 178 rx = Literal('foo') | (Literal('ba') + Any('a')[1:,...])
180 179
181 rx.config.compile_to_nfa().no_full_first_match().no_memoize() 180 rx.config.compile_to_nfa().no_full_first_match().no_memoize()
182 matcher = rx.get_match_sequence() 181 matcher = rx.get_match_sequence()
183 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 182 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
184 results = list(matcher('foo')) 183 results = list(matcher('foo'))
185 assert results == [(['foo'], (3, DUMMY_HELPER))], results 184 assert results == [(['foo'], (3, DUMMY_HELPER))], results
186 results = list(matcher('baaaaax')) 185 results = list(matcher('baaaaax'))
187 assert results == [(['baaaaa'], (6, DUMMY_HELPER)), 186 assert results == [(['baaaaa'], (6, DUMMY_HELPER)),
188 (['baaaa'], (5, DUMMY_HELPER)), 187 (['baaaa'], (5, DUMMY_HELPER)),
189 (['baaa'], (4, DUMMY_HELPER)), 188 (['baaa'], (4, DUMMY_HELPER)),
190 (['baa'], (3, DUMMY_HELPER))], results 189 (['baa'], (3, DUMMY_HELPER))], results
191 results = list(matcher('ba')) 190 results = list(matcher('ba'))
192 assert results == [], results 191 assert results == [], results
193 192
194 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize() 193 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize()
195 matcher = rx.get_match_sequence() 194 matcher = rx.get_match_sequence()
196 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 195 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
197 results = list(matcher('foo')) 196 results = list(matcher('foo'))
198 assert results == [(['foo'], (3, DUMMY_HELPER))], results 197 assert results == [(['foo'], (3, DUMMY_HELPER))], results
199 results = list(matcher('baaaaax')) 198 results = list(matcher('baaaaax'))
200 assert results == [(['baaaaa'], (6, DUMMY_HELPER)), 199 assert results == [(['baaaaa'], (6, DUMMY_HELPER)),
201 (['baaaa'], (5, DUMMY_HELPER)), 200 (['baaaa'], (5, DUMMY_HELPER)),
202 (['baaa'], (4, DUMMY_HELPER)), 201 (['baaa'], (4, DUMMY_HELPER)),
203 (['baa'], (3, DUMMY_HELPER))], results 202 (['baa'], (3, DUMMY_HELPER))], results
204 results = list(matcher('ba')) 203 results = list(matcher('ba'))
205 assert results == [], results 204 assert results == [], results
206 205
207 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize() 206 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize()
208 matcher = rx.get_match_sequence() 207 matcher = rx.get_match_sequence()
209 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 208 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
210 results = list(matcher('foo')) 209 results = list(matcher('foo'))
211 assert results == [(['foo'], (3, DUMMY_HELPER))], results 210 assert results == [(['foo'], (3, DUMMY_HELPER))], results
212 results = list(matcher('baaaaax')) 211 results = list(matcher('baaaaax'))
213 assert results == [(['baaaaa'], (6, DUMMY_HELPER)), 212 assert results == [(['baaaaa'], (6, DUMMY_HELPER)),
214 (['baaaa'], (5, DUMMY_HELPER)), 213 (['baaaa'], (5, DUMMY_HELPER)),
215 (['baaa'], (4, DUMMY_HELPER)), 214 (['baaa'], (4, DUMMY_HELPER)),
216 (['baa'], (3, DUMMY_HELPER))], results 215 (['baa'], (3, DUMMY_HELPER))], results
217 results = list(matcher('ba')) 216 results = list(matcher('ba'))
218 assert results == [], results 217 assert results == [], results
219 218
220 def test_integer(self): 219 def test_integer(self):
221 rx = Integer() 220 rx = Integer()
222 221
223 rx.config.compile_to_nfa().no_full_first_match().no_memoize() 222 rx.config.compile_to_nfa().no_full_first_match().no_memoize()
224 matcher = rx.get_match_sequence() 223 matcher = rx.get_match_sequence()
225 results = list(matcher('12x')) 224 results = list(matcher('12x'))
226 assert results == [(['12'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results 225 assert results == [(['12'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results
227 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 226 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
228 227
229 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize() 228 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize()
230 matcher = rx.get_match_sequence() 229 matcher = rx.get_match_sequence()
231 results = list(matcher('12x')) 230 results = list(matcher('12x'))
232 assert results == [(['12'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results 231 assert results == [(['12'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results
233 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 232 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
234 233
235 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize() 234 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize()
236 matcher = rx.get_match_sequence() 235 matcher = rx.get_match_sequence()
237 results = list(matcher('12x')) 236 results = list(matcher('12x'))
238 assert results == [(['12'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results 237 assert results == [(['12'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results
239 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 238 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
240 239
241 def test_real(self): 240 def test_real(self):
242 rx = Real() 241 rx = Real()
243 242
244 rx.config.compile_to_nfa().no_full_first_match().no_memoize() 243 rx.config.compile_to_nfa().no_full_first_match().no_memoize()
245 matcher = rx.get_match_sequence() 244 matcher = rx.get_match_sequence()
246 results = list(matcher('1.2x')) 245 results = list(matcher('1.2x'))
247 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results 246 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results
248 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 247 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
249 248
250 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize() 249 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize()
251 matcher = rx.get_match_sequence() 250 matcher = rx.get_match_sequence()
252 results = list(matcher('1.2x')) 251 results = list(matcher('1.2x'))
253 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results 252 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results
254 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 253 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
255 254
256 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize() 255 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize()
257 matcher = rx.get_match_sequence() 256 matcher = rx.get_match_sequence()
258 results = list(matcher('1.2x')) 257 results = list(matcher('1.2x'))
259 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results 258 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER)), (['1'], (1, DUMMY_HELPER))], results
260 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 259 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
261 260
262 def test_float(self): 261 def test_float(self):
263 rx = Float() 262 rx = Float()
264 263
265 rx.config.compile_to_nfa().no_full_first_match().no_memoize() 264 rx.config.compile_to_nfa().no_full_first_match().no_memoize()
266 matcher = rx.get_match_sequence() 265 matcher = rx.get_match_sequence()
267 results = list(matcher('1.2x')) 266 results = list(matcher('1.2x'))
268 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER))], results 267 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER))], results
269 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 268 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
270 269
271 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize() 270 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize()
272 matcher = rx.get_match_sequence() 271 matcher = rx.get_match_sequence()
273 results = list(matcher('1.2x')) 272 results = list(matcher('1.2x'))
274 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER))], results 273 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER))], results
275 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree().no_memoize() 274 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree().no_memoize()
276 275
277 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms() 276 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms()
278 matcher = rx.get_match_sequence() 277 matcher = rx.get_match_sequence()
279 results = list(matcher('1.2x')) 278 results = list(matcher('1.2x'))
280 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER))], results 279 assert results == [(['1.2'], (3, DUMMY_HELPER)), (['1.'], (2, DUMMY_HELPER))], results
281 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree().no_memoize() 280 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree().no_memoize()
282 281
283 def test_star(self): 282 def test_star(self):
284 rx = Add(Star('a')) 283 rx = Add(Star('a'))
285 284
286 rx.config.compile_to_nfa().no_full_first_match().no_memoize() 285 rx.config.compile_to_nfa().no_full_first_match().no_memoize()
287 matcher = rx.get_match_sequence() 286 matcher = rx.get_match_sequence()
288 results = list(matcher('aa')) 287 results = list(matcher('aa'))
289 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER)), ([], (0, DUMMY_HELPER))], results 288 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER)), ([], (0, DUMMY_HELPER))], results
290 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 289 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
291 290
292 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize() 291 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize()
293 matcher = rx.get_match_sequence() 292 matcher = rx.get_match_sequence()
294 results = list(matcher('aa')) 293 results = list(matcher('aa'))
295 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER)), ([], (0, DUMMY_HELPER))], results 294 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER)), ([], (0, DUMMY_HELPER))], results
296 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 295 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
297 296
298 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize() 297 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize()
299 matcher = rx.get_match_sequence() 298 matcher = rx.get_match_sequence()
300 results = list(matcher('aa')) 299 results = list(matcher('aa'))
301 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER)), ([], (0, DUMMY_HELPER))], results 300 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER)), ([], (0, DUMMY_HELPER))], results
302 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 301 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
303 302
304 def test_word(self): 303 def test_word(self):
305 #basicConfig(level=DEBUG) 304 #basicConfig(level=DEBUG)
306 rx = Word('a') 305 rx = Word('a')
307 306
308 rx.config.compile_to_nfa().no_full_first_match().no_memoize() 307 rx.config.compile_to_nfa().no_full_first_match().no_memoize()
309 matcher = rx.get_match_sequence() 308 matcher = rx.get_match_sequence()
310 results = list(matcher('aa')) 309 results = list(matcher('aa'))
311 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER))], results 310 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER))], results
312 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 311 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
313 312
314 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize() 313 rx.config.clear().compile_to_nfa().no_full_first_match().no_memoize()
315 matcher = rx.get_match_sequence() 314 matcher = rx.get_match_sequence()
316 results = list(matcher('aa')) 315 results = list(matcher('aa'))
317 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER))], results 316 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER))], results
318 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 317 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
319 318
320 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize() 319 rx.config.clear().compile_to_nfa().no_full_first_match().compose_transforms().no_memoize()
321 matcher = rx.get_match_sequence() 320 matcher = rx.get_match_sequence()
322 results = list(matcher('aa')) 321 results = list(matcher('aa'))
323 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER))], results 322 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER))], results
324 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 323 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
325 324
326 325
327 class CompileTest(TestCase): 326 class CompileTest(TestCase):
328 ''' 327 '''
329 Test the rewrite routine directly. 328 Test the rewrite routine directly.
330 ''' 329 '''
331 330
332 def assert_regexp(self, matcher, regexp): 331 def assert_regexp(self, matcher, regexp):
333 compiler = CompileRegexp(use=True) 332 compiler = CompileRegexp(use=True)
334 matcher = compiler(matcher) 333 matcher = compiler(matcher)
335 assert isinstance(matcher, NfaRegexp), matcher.tree() 334 assert isinstance(matcher, NfaRegexp), matcher.tree()
336 assert str(matcher.regexp) == regexp, matcher.regexp 335 assert str(matcher.regexp) == regexp, matcher.regexp
337 336
338 def test_any(self): 337 def test_any(self):
339 self.assert_regexp(Any(), '.') 338 self.assert_regexp(Any(), '.')
340 self.assert_regexp(Any('abc'), '[a-c]') 339 self.assert_regexp(Any('abc'), '[a-c]')
341 340
342 def test_literal(self): 341 def test_literal(self):
343 self.assert_regexp(Literal('foo'), 'foo') 342 self.assert_regexp(Literal('foo'), 'foo')
344 343
345 def test_repeat(self): 344 def test_repeat(self):
346 self.assert_regexp(Any()[:, ...], '(?:.)*') 345 self.assert_regexp(Any()[:, ...], '(?:.)*')
347 self.assert_regexp(Any()[1:, ...], '.(?:.)*') 346 self.assert_regexp(Any()[1:, ...], '.(?:.)*')
348 self.assert_regexp(Any()[1, ...], '.') 347 self.assert_regexp(Any()[1, ...], '.')
349 self.assert_regexp(Any()[1:2, ...], '.(?:.)?') 348 self.assert_regexp(Any()[1:2, ...], '.(?:.)?')
350 self.assert_regexp(Any()[2, ...], '..') 349 self.assert_regexp(Any()[2, ...], '..')
351 self.assert_regexp(Any()[2:4, ...], '..(?:.(?:.)?)?') 350 self.assert_regexp(Any()[2:4, ...], '..(?:.(?:.)?)?')
352 self.assert_regexp(Any()[:, 'x', ...], '(?:.(?:x.)*|)') 351 self.assert_regexp(Any()[:, 'x', ...], '(?:.(?:x.)*|)')
353 self.assert_regexp(Any()[1:, 'x', ...], '.(?:x.)*') 352 self.assert_regexp(Any()[1:, 'x', ...], '.(?:x.)*')
354 self.assert_regexp(Any()[1, 'x', ...], '.') 353 self.assert_regexp(Any()[1, 'x', ...], '.')
355 self.assert_regexp(Any()[1:2, 'x', ...], '.(?:x.)?') 354 self.assert_regexp(Any()[1:2, 'x', ...], '.(?:x.)?')
356 self.assert_regexp(Any()[2, 'x', ...], '.x.') 355 self.assert_regexp(Any()[2, 'x', ...], '.x.')
357 self.assert_regexp(Any()[2:4, 'x', ...], '.x.(?:x.(?:x.)?)?') 356 self.assert_regexp(Any()[2:4, 'x', ...], '.x.(?:x.(?:x.)?)?')
358 self.assert_regexp(Literal('foo')[:, ...], '(?:foo)*') 357 self.assert_regexp(Literal('foo')[:, ...], '(?:foo)*')
359 358
360 def test_and(self): 359 def test_and(self):
361 self.assert_regexp(Any('ab')[:, ...] + Any('p'), '(?:[a-b])*p') 360 self.assert_regexp(Any('ab')[:, ...] + Any('p'), '(?:[a-b])*p')
362 361
363 def test_or(self): 362 def test_or(self):
364 self.assert_regexp(Any('ab')[:, ...] | Any('p'), '(?:(?:[a-b])*|p)') 363 self.assert_regexp(Any('ab')[:, ...] | Any('p'), '(?:(?:[a-b])*|p)')
365 364
366 def test_complex(self): 365 def test_complex(self):
367 self.assert_regexp((Any('ab') + Literal('q')) | Literal('z'), '(?:[a-b]q|z)') 366 self.assert_regexp((Any('ab') + Literal('q')) | Literal('z'), '(?:[a-b]q|z)')
368 self.assert_regexp((Any('ab') + 'q') | 'z', '(?:[a-b]q|z)') 367 self.assert_regexp((Any('ab') + 'q') | 'z', '(?:[a-b]q|z)')
369 368
370 369
371 class RepeatBugTest(TestCase): 370 class RepeatBugTest(TestCase):
372 371
373 def test_bug(self): 372 def test_bug(self):
374 #basicConfig(level=DEBUG) 373 #basicConfig(level=DEBUG)
375 matcher = Any()[2, ...] 374 matcher = Any()[2, ...]
376 matcher.config.no_full_first_match().compile_to_nfa() 375 matcher.config.no_full_first_match().compile_to_nfa()
377 parser = matcher.get_parse_all() 376 parser = matcher.get_parse_all()
378 results = list(parser('abc')) 377 results = list(parser('abc'))
379 assert results == [['ab']], results 378 assert results == [['ab']], results
380 379
381 def test_bug2(self): 380 def test_bug2(self):
382 matcher = NfaRegexp('..') 381 matcher = NfaRegexp('..')
383 matcher.config.no_full_first_match() 382 matcher.config.no_full_first_match()
384 parser = matcher.get_parse_all() 383 parser = matcher.get_parse_all()
385 results = list(parser('abc')) 384 results = list(parser('abc'))
386 assert results == [['ab']], results 385 assert results == [['ab']], results
387 386
388 387
389 class WordBugTest(TestCase): 388 class WordBugTest(TestCase):
390 ''' 389 '''
391 Used to not be possible to compile a raw Word() 390 Used to not be possible to compile a raw Word()
392 ''' 391 '''
393 392
394 def test_word(self): 393 def test_word(self):
395 #basicConfig(level=DEBUG) 394 #basicConfig(level=DEBUG)
396 rx = Word() 395 rx = Word()
397 396
398 rx.config.compile_to_nfa().no_full_first_match().no_memoize() 397 rx.config.compile_to_nfa().no_full_first_match().no_memoize()
399 matcher = rx.get_match_sequence() 398 matcher = rx.get_match_sequence()
400 results = list(matcher('aa')) 399 results = list(matcher('aa'))
401 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER))], results 400 assert results == [(['aa'], (2, DUMMY_HELPER)), (['a'], (1, DUMMY_HELPER))], results
402 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 401 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
403 402
404 403
405 class TokenBugTest(TestCase): 404 class TokenBugTest(TestCase):
406 ''' 405 '''
407 Token(Word()) and Token(Any(ascii_letters)[1:]) gave errors. 406 Token(Word()) and Token(Any(ascii_letters)[1:]) gave errors.
408 ''' 407 '''
409 408
410 def test_token_word(self): 409 def test_token_word(self):
411 tk = Token(Word()) 410 tk = Token(Word())
412 tk.config.lines(block_policy=explicit) 411 tk.config.lines(block_policy=explicit)
413 tk.get_parse() 412 tk.get_parse()
414 413
415 def test_token_any(self): 414 def test_token_any(self):
416 tk = Token(Any(ascii_letters)[1:,...]) 415 tk = Token(Any(ascii_letters)[1:,...])
417 tk.config.lines(block_policy=explicit) 416 tk.config.lines(block_policy=explicit)
418 tk.get_parse() 417 tk.get_parse()
419 418
420 def test_simple_word(self): 419 def test_simple_word(self):
421 #basicConfig(level=DEBUG) 420 #basicConfig(level=DEBUG)
422 rx = Word() 421 rx = Word()
423 rx.config.no_memoize() 422 rx.config.no_memoize()
424 matcher = rx.get_parse().matcher 423 matcher = rx.get_parse().matcher
425 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 424 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
426 425
427 def test_simple_any(self): 426 def test_simple_any(self):
428 rx = Any(ascii_letters)[1:,...] 427 rx = Any(ascii_letters)[1:,...]
429 rx.config.compile_to_nfa().no_memoize() 428 rx.config.compile_to_nfa().no_memoize()
430 matcher = rx.get_parse().matcher 429 matcher = rx.get_parse().matcher
431 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree() 430 assert isinstance(matcher.matcher, NfaRegexp), matcher.matcher.tree()
432 431
Powered by Google Project Hosting