My favorites | Sign in
Project Home Downloads Wiki Issues Source
Repository:
Checkout   Browse   Changes   Clones  
Changes to /src/lepl/stream/core.py
c49f34c9c2a4 vs. a7dc412647e7 Compare: vs.  Format:
Revision a7dc412647e7
Go to: 
Project members, sign in to write a code review
/src/lepl/stream/core.py   c49f34c9c2a4 /src/lepl/stream/core.py   a7dc412647e7
1 1
2 # The contents of this file are subject to the Mozilla Public License 2 # The contents of this file are subject to the Mozilla Public License
3 # (MPL) Version 1.1 (the "License"); you may not use this file except 3 # (MPL) Version 1.1 (the "License"); you may not use this file except
4 # in compliance with the License. You may obtain a copy of the License 4 # in compliance with the License. You may obtain a copy of the License
5 # at http://www.mozilla.org/MPL/ 5 # at http://www.mozilla.org/MPL/
6 # 6 #
7 # Software distributed under the License is distributed on an "AS IS" 7 # Software distributed under the License is distributed on an "AS IS"
8 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 8 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
9 # the License for the specific language governing rights and 9 # the License for the specific language governing rights and
10 # limitations under the License. 10 # limitations under the License.
11 # 11 #
12 # The Original Code is LEPL (http://www.acooke.org/lepl) 12 # The Original Code is LEPL (http://www.acooke.org/lepl)
13 # The Initial Developer of the Original Code is Andrew Cooke. 13 # The Initial Developer of the Original Code is Andrew Cooke.
14 # Portions created by the Initial Developer are Copyright (C) 2009-2010 14 # Portions created by the Initial Developer are Copyright (C) 2009-2010
15 # Andrew Cooke (andrew@acooke.org). All Rights Reserved. 15 # Andrew Cooke (andrew@acooke.org). All Rights Reserved.
16 # 16 #
17 # Alternatively, the contents of this file may be used under the terms 17 # Alternatively, the contents of this file may be used under the terms
18 # of the LGPL license (the GNU Lesser General Public License, 18 # of the LGPL license (the GNU Lesser General Public License,
19 # http://www.gnu.org/licenses/lgpl.html), in which case the provisions 19 # http://www.gnu.org/licenses/lgpl.html), in which case the provisions
20 # of the LGPL License are applicable instead of those above. 20 # of the LGPL License are applicable instead of those above.
21 # 21 #
22 # If you wish to allow use of your version of this file only under the 22 # If you wish to allow use of your version of this file only under the
23 # terms of the LGPL License and not to allow others to use your version 23 # terms of the LGPL License and not to allow others to use your version
24 # of this file under the MPL, indicate your decision by deleting the 24 # of this file under the MPL, indicate your decision by deleting the
25 # provisions above and replace them with the notice and other provisions 25 # provisions above and replace them with the notice and other provisions
26 # required by the LGPL License. If you do not delete the provisions 26 # required by the LGPL License. If you do not delete the provisions
27 # above, a recipient may use your version of this file under either the 27 # above, a recipient may use your version of this file under either the
28 # MPL or the LGPL License. 28 # MPL or the LGPL License.
29 29
30 ''' 30 '''
31 Default implementations of the stream classes. 31 Default implementations of the stream classes.
32 32
33 A stream is a tuple (state, helper), where `state` will vary from location to 33 A stream is a tuple (state, helper), where `state` will vary from location to
34 location, while `helper` is an "unchanging" instance of `StreamHelper`, 34 location, while `helper` is an "unchanging" instance of `StreamHelper`,
35 defined below. 35 defined below.
36 36
37 For simple streams state can be a simple integer and this approach avoids the 37 For simple streams state can be a simple integer and this approach avoids the
38 repeated creation of objects. More complex streams may choose to not use 38 repeated creation of objects. More complex streams may choose to not use
39 the state at all, simply creating a new helper at each point. 39 the state at all, simply creating a new helper at each point.
40 ''' 40 '''
41 41
42 from abc import ABCMeta 42 from abc import ABCMeta
43 43
44 from lepl.support.lib import fmt 44 from lepl.support.lib import fmt
45 45
46 46
47 #class _SimpleStream(metaclass=ABCMeta): 47 #class _SimpleStream(metaclass=ABCMeta):
48 # Python 2.6 48 # Python 2.6
49 # pylint: disable-msg=W0105, C0103 49 # pylint: disable-msg=W0105, C0103
50 _StreamHelper = ABCMeta('_StreamHelper', (object, ), {}) 50 _StreamHelper = ABCMeta('_StreamHelper', (object, ), {})
51 '''ABC used to identify streams.''' 51 '''ABC used to identify streams.'''
52 52
53 DUMMY_HELPER = object() 53 DUMMY_HELPER = object()
54 '''Allows tests to specify an arbitrary helper in results.''' 54 '''Allows tests to specify an arbitrary helper in results.'''
55 55
56 OFFSET, LINENO, CHAR = range(3) 56 OFFSET, LINENO, CHAR = range(3)
57 '''Indices into delta.''' 57 '''Indices into delta.'''
58 58
59 59
60 class StreamHelper(_StreamHelper): 60 class StreamHelper(_StreamHelper):
61 ''' 61 '''
62 The interface that all helpers should implement. 62 The interface that all helpers should implement.
63 ''' 63 '''
64 64
65 def __init__(self, id=None, factory=None, max=None, global_kargs=None, 65 def __init__(self, id=None, factory=None, max=None, global_kargs=None,
66 cache_level=None): 66 cache_level=None):
67 from lepl.stream.factory import DEFAULT_STREAM_FACTORY 67 from lepl.stream.factory import DEFAULT_STREAM_FACTORY
68 self.id = id if id is not None else hash(self) 68 self.id = id if id is not None else hash(self)
69 self.factory = factory if factory else DEFAULT_STREAM_FACTORY 69 self.factory = factory if factory else DEFAULT_STREAM_FACTORY
70 self.max = max if max else MutableMaxDepth() 70 self.max = max if max else MutableMaxDepth()
71 self.global_kargs = global_kargs if global_kargs else {} 71 self.global_kargs = global_kargs if global_kargs else {}
72 self.cache_level = 1 if cache_level is None else cache_level 72 self.cache_level = 1 if cache_level is None else cache_level
73 73
74 def __repr__(self): 74 def __repr__(self):
75 '''Simplify for comparison in tests''' 75 '''Simplify for comparison in tests'''
76 return '<helper>' 76 return '<helper>'
77 77
78 def __eq__(self, other): 78 def __eq__(self, other):
79 return other is DUMMY_HELPER or super(StreamHelper, self).__eq__(other) 79 return other is DUMMY_HELPER or super(StreamHelper, self).__eq__(other)
80 80
81 def __hash__(self): 81 def __hash__(self):
82 return super(StreamHelper, self).__hash__() 82 return super(StreamHelper, self).__hash__()
83 83
84 def key(self, state, other): 84 def key(self, state, other):
85 ''' 85 '''
86 Generate an object that can be hashed (implements __hash__ and __eq__). 86 Generate an object that can be hashed (implements __hash__ and __eq__).
87 See `HashKey`. 87 See `HashKey`.
88 ''' 88 '''
89 raise NotImplementedError 89 raise NotImplementedError
90 90
91 def kargs(self, state, prefix='', kargs=None): 91 def kargs(self, state, prefix='', kargs=None):
92 ''' 92 '''
93 Generate a dictionary of values that describe the stream. These 93 Generate a dictionary of values that describe the stream. These
94 may be extended by subclasses. They are provided to 94 may be extended by subclasses. They are provided to
95 `syntax_error_kargs`, for example. 95 `syntax_error_kargs`, for example.
96 96
97 `prefix` modifies the property names 97 `prefix` modifies the property names
98 98
99 `kargs` allows values to be provided. These are *not* overwritten, 99 `kargs` allows values to be provided. These are *not* overwritten,
100 so if there is a name clash the provided value remains. 100 so if there is a name clash the provided value remains.
101 101
102 Note: Calculating this can be expensive; use only for error messages, 102 Note: Calculating this can be expensive; use only for error messages,
103 not debug messages (that may be discarded). 103 not debug messages (that may be discarded).
104 104
105 The following names will be defined (at a minimum). 105 The following names will be defined (at a minimum).
106 106
107 For these value the "global" prefix indicates the underlying stream 107 For these value the "global" prefix indicates the underlying stream
108 when, for example, tokens are used (other values will be relative to 108 when, for example, tokens are used (other values will be relative to
109 the token). If tokens etc are not in use then global and non-global 109 the token). If tokens etc are not in use then global and non-global
110 values will agree. 110 values will agree.
111 - data: a line representing the data, highlighting the current offset 111 - data: a line representing the data, highlighting the current offset
112 - global_data: as data, but for the entire sequence 112 - global_data: as data, but for the entire sequence
113 - text: as data, but without a "[...]" at the end 113 - text: as data, but without a "[...]" at the end
114 - global_text: as text, but for the entire sequence 114 - global_text: as text, but for the entire sequence
115 - type: the type of the sequence 115 - type: the type of the sequence
116 - global_type: the type of the entire sequence 116 - global_type: the type of the entire sequence
117 - global_offset: a 0-based index into the underlying sequence 117 - global_offset: a 0-based index into the underlying sequence
118 118
119 These values are always local: 119 These values are always local:
120 - offset: a 0-based index into the sequence 120 - offset: a 0-based index into the sequence
121 - rest: the data following the current point 121 - rest: the data following the current point
122 - repr: the current value, or <EOS> 122 - repr: the current value, or <EOS>
123 - str: the current value, or an empty string 123 - str: the current value, or an empty string
124 124
125 These values are always global: 125 These values are always global:
126 - filename: a filename, if available, or the type 126 - filename: a filename, if available, or the type
127 - lineno: a 1-based line number for the current offset 127 - lineno: a 1-based line number for the current offset
128 - char: a 1-based character count within the line for the current offset 128 - char: a 1-based character count within the line for the current offset
129 - location: a summary of the current location 129 - location: a summary of the current location
130 ''' 130 '''
131 raise NotImplementedError 131 raise NotImplementedError
132 132
133 def fmt(self, state, template, prefix='', kargs=None): 133 def fmt(self, state, template, prefix='', kargs=None):
134 '''fmt a message using the expensive kargs function.''' 134 '''fmt a message using the expensive kargs function.'''
135 return fmt(template, **self.kargs(state, prefix=prefix, kargs=kargs)) 135 return fmt(template, **self.kargs(state, prefix=prefix, kargs=kargs))
136 136
137 def debug(self, state): 137 def debug(self, state):
138 '''Generate an inexpensive debug message.''' 138 '''Generate an inexpensive debug message.'''
139 raise NotImplementedError 139 raise NotImplementedError
140 140
141 def next(self, state, count=1): 141 def next(self, state, count=1):
142 ''' 142 '''
143 Return (value, stream) where `value` is the next value (or 143 Return (value, stream) where `value` is the next value (or
144 values if count > 1) from the stream and `stream` is advanced to the 144 values if count > 1) from the stream and `stream` is advanced to the
145 next character. Note that `value` is always a sequence (so if the 145 next character. Note that `value` is always a sequence (so if the
146 stream is a list of integers, and `count`=1, then it will be a 146 stream is a list of integers, and `count`=1, then it will be a
147 unitary list, for example). 147 unitary list, for example).
148 148
149 Should raise StopIteration when no more data are available. 149 Should raise StopIteration when no more data are available.
150 ''' 150 '''
151 raise StopIteration 151 raise StopIteration
152 152
153 def join(self, state, *values): 153 def join(self, state, *values):
154 ''' 154 '''
155 Join sequences of values into a single sequence. 155 Join sequences of values into a single sequence.
156 ''' 156 '''
157 raise NotImplementedError 157 raise NotImplementedError
158 158
159 def empty(self, state): 159 def empty(self, state):
160 ''' 160 '''
161 Return true if no more data available. 161 Return true if no more data available.
162 ''' 162 '''
163 raise NotImplementedError 163 raise NotImplementedError
164 164
165 def line(self, state, empty_ok): 165 def line(self, state, empty_ok):
166 ''' 166 '''
167 Return (values, stream) where `values` correspond to something 167 Return (values, stream) where `values` correspond to something
168 like "the rest of the line" from the current point and `stream` 168 like "the rest of the line" from the current point and `stream`
169 is advanced to the point after the line ends. 169 is advanced to the point after the line ends.
170 170
171 If `empty_ok` is true and we are at the end of a line, return an 171 If `empty_ok` is true and we are at the end of a line, return an
172 empty line, otherwise advance (and maybe raise a StopIteration). 172 empty line, otherwise advance (and maybe raise a StopIteration).
173 ''' 173 '''
174 raise NotImplementedError 174 raise NotImplementedError
175 175
176 def len(self, state): 176 def len(self, state):
177 ''' 177 '''
178 Return the remaining length of the stream. Streams of unknown 178 Return the remaining length of the stream. Streams of unknown
179 length (iterables) should raise a TypeError. 179 length (iterables) should raise a TypeError.
180 ''' 180 '''
181 raise NotImplementedError 181 raise NotImplementedError
182 182
183 def stream(self, state, value, id_=None, max=None): 183 def stream(self, state, value, id_=None, max=None):
184 ''' 184 '''
185 Return a new stream that encapsulates the value given, starting at 185 Return a new stream that encapsulates the value given, starting at
186 `state`. IMPORTANT: the stream used is the one that corresponds to 186 `state`. IMPORTANT: the stream used is the one that corresponds to
187 the start of the value. 187 the start of the value.
188 188
189 For example: 189 For example:
190 (line, next_stream) = s_line(stream, False) 190 (line, next_stream) = s_line(stream, False)
191 token_stream = s_stream(stream, line) # uses stream, not next_stream 191 token_stream = s_stream(stream, line) # uses stream, not next_stream
192 192
193 This is used when processing Tokens, for example, or columns (where 193 This is used when processing Tokens, for example, or columns (where
194 fragments in the correct column area are parsed separately). 194 fragments in the correct column area are parsed separately).
195 ''' 195 '''
196 raise NotImplementedError 196 raise NotImplementedError
197 197
198 def deepest(self): 198 def deepest(self):
199 ''' 199 '''
200 Return a stream that represents the deepest match. The stream may be 200 Return a stream that represents the deepest match. The stream may be
201 incomplete in some sense (it may not be possible to use it for 201 incomplete in some sense (it may not be possible to use it for
202 parsing more data), but it will have usable fmt and kargs methods. 202 parsing more data), but it will have usable fmt and kargs methods.
203 ''' 203 '''
204 raise NotImplementedError 204 raise NotImplementedError
205 205
206 def delta(self, state): 206 def delta(self, state):
207 ''' 207 '''
208 Return the offset, lineno and char of the current point, relative to 208 Return the offset, lineno and char of the current point, relative to
209 the entire stream, as a tuple. 209 the entire stream, as a tuple.
210 ''' 210 '''
211 raise NotImplementedError 211 raise NotImplementedError
212 212
213 def eq(self, state1, state2): 213 def eq(self, state1, state2):
214 ''' 214 '''
215 Are the two states equal? 215 Are the two states equal?
216 ''' 216 '''
217 return state1 == state2 217 return state1 == state2
218 218
219 def new_max(self, state): 219 def new_max(self, state):
220 ''' 220 '''
221 Return a new stream, with a new helper, that uses a new max. 221 Return (old max, new stream), where new stream uses a new max.
222 This is used when we want to read from the stream without
223 affecting the max (eg when looking ahead to generate tokens).
222 ''' 224 '''
223 raise NotImplementedError 225 raise NotImplementedError
224 226
225 def cacheable(self): 227 def cacheable(self):
226 ''' 228 '''
227 Is this stream cacheable? 229 Is this stream cacheable?
228 ''' 230 '''
229 return self.cache_level > 0 231 return self.cache_level > 0
230 232
231 233
232 # The following are helper functions that allow the methods above to be 234 # The following are helper functions that allow the methods above to be
233 # called on (state, helper) tuples 235 # called on (state, helper) tuples
234 236
235 s_key = lambda stream, other=None: stream[1].key(stream[0], other) 237 s_key = lambda stream, other=None: stream[1].key(stream[0], other)
236 '''Invoke helper.key(state, other)''' 238 '''Invoke helper.key(state, other)'''
237 239
238 s_kargs = lambda stream, prefix='', kargs=None: stream[1].kargs(stream[0], prefix=prefix, kargs=kargs) 240 s_kargs = lambda stream, prefix='', kargs=None: stream[1].kargs(stream[0], prefix=prefix, kargs=kargs)
239 '''Invoke helper.kargs(state, prefix, kargs)''' 241 '''Invoke helper.kargs(state, prefix, kargs)'''
240 242
241 s_fmt = lambda stream, template, prefix='', kargs=None: stream[1].fmt(stream[0], template, prefix=prefix, kargs=kargs) 243 s_fmt = lambda stream, template, prefix='', kargs=None: stream[1].fmt(stream[0], template, prefix=prefix, kargs=kargs)
242 '''Invoke helper.fmt(state, template, prefix, kargs)''' 244 '''Invoke helper.fmt(state, template, prefix, kargs)'''
243 245
244 s_debug = lambda stream: stream[1].debug(stream[0]) 246 s_debug = lambda stream: stream[1].debug(stream[0])
245 '''Invoke helper.debug()''' 247 '''Invoke helper.debug()'''
246 248
247 s_next = lambda stream, count=1: stream[1].next(stream[0], count=count) 249 s_next = lambda stream, count=1: stream[1].next(stream[0], count=count)
248 '''Invoke helper.next(state, count)''' 250 '''Invoke helper.next(state, count)'''
249 251
250 s_join = lambda stream, *values: stream[1].join(stream[0], *values) 252 s_join = lambda stream, *values: stream[1].join(stream[0], *values)
251 '''Invoke helper.join(*values)''' 253 '''Invoke helper.join(*values)'''
252 254
253 s_empty = lambda stream: stream[1].empty(stream[0]) 255 s_empty = lambda stream: stream[1].empty(stream[0])
254 '''Invoke helper.empty(state)''' 256 '''Invoke helper.empty(state)'''
255 257
256 s_line = lambda stream, empty_ok: stream[1].line(stream[0], empty_ok) 258 s_line = lambda stream, empty_ok: stream[1].line(stream[0], empty_ok)
257 '''Invoke helper.line(state, empty_ok)''' 259 '''Invoke helper.line(state, empty_ok)'''
258 260
259 s_len = lambda stream: stream[1].len(stream[0]) 261 s_len = lambda stream: stream[1].len(stream[0])
260 '''Invoke helper.len(state)''' 262 '''Invoke helper.len(state)'''
261 263
262 s_stream = lambda stream, value, id_=None, max=None: stream[1].stream(stream[0], value, id_=id_, max=max) 264 s_stream = lambda stream, value, id_=None, max=None: stream[1].stream(stream[0], value, id_=id_, max=max)
263 '''Invoke helper.stream(state, value)''' 265 '''Invoke helper.stream(state, value)'''
264 266
265 s_deepest = lambda stream: stream[1].deepest() 267 s_deepest = lambda stream: stream[1].deepest()
266 '''Invoke helper.deepest()''' 268 '''Invoke helper.deepest()'''
267 269
268 s_delta = lambda stream: stream[1].delta(stream[0]) 270 s_delta = lambda stream: stream[1].delta(stream[0])
269 '''Invoke helper.delta(state)''' 271 '''Invoke helper.delta(state)'''
270 272
271 s_eq = lambda stream1, stream2: stream1[1].eq(stream1[0], stream2[0]) 273 s_eq = lambda stream1, stream2: stream1[1].eq(stream1[0], stream2[0])
272 '''Compare two streams (which should have identical helpers)''' 274 '''Compare two streams (which should have identical helpers)'''
273 275
274 s_id = lambda stream: stream[1].id 276 s_id = lambda stream: stream[1].id
275 '''Access the ID attribute.''' 277 '''Access the ID attribute.'''
276 278
277 s_factory = lambda stream: stream[1].factory 279 s_factory = lambda stream: stream[1].factory
278 '''Access the factory attribute.''' 280 '''Access the factory attribute.'''
279 281
280 s_max = lambda stream: stream[1].max 282 s_max = lambda stream: stream[1].max
281 '''Access the max attribute.''' 283 '''Access the max attribute.'''
282 284
283 s_new_max = lambda stream: stream[1].new_max(stream[0]) 285 s_new_max = lambda stream: stream[1].new_max(stream[0])
284 '''Invoke helper.new_max(state).''' 286 '''Invoke helper.new_max(state).'''
285 287
286 s_global_kargs = lambda stream: stream[1].global_kargs 288 s_global_kargs = lambda stream: stream[1].global_kargs
287 '''Access the global_kargs attribute.''' 289 '''Access the global_kargs attribute.'''
288 290
289 s_cache_level = lambda stream: stream[1].cache_level 291 s_cache_level = lambda stream: stream[1].cache_level
290 '''Access the cache_level attribute.''' 292 '''Access the cache_level attribute.'''
291 293
292 s_cacheable = lambda stream: stream[1].cacheable() 294 s_cacheable = lambda stream: stream[1].cacheable()
293 '''Is the stream cacheable?''' 295 '''Is the stream cacheable?'''
294 296
295 297
296 class MutableMaxDepth(object): 298 class MutableMaxDepth(object):
297 ''' 299 '''
298 Track maximum depth (offset) reached and the associated stream. Used to 300 Track maximum depth (offset) reached and the associated stream. Used to
299 generate error message for incomplete matches. 301 generate error message for incomplete matches.
300 ''' 302 '''
301 303
302 def __init__(self): 304 def __init__(self):
303 self.depth = 0 305 self.depth = 0
304 self.stream = None 306 self.stream = None
305 307
306 def update(self, depth, stream): 308 def update(self, depth, stream):
307 # the '=' here allows a token to nudge on to the next stream without 309 # the '=' here allows a token to nudge on to the next stream without
308 # changing the offset (when count=0 in s_next) 310 # changing the offset (when count=0 in s_next)
309 if depth >= self.depth or not self.stream: 311 if depth >= self.depth or not self.stream:
310 self.depth = depth 312 self.depth = depth
311 self.stream = stream 313 self.stream = stream
312 314
313 def get(self): 315 def get(self):
314 return self.stream 316 return self.stream
315 317
316 318
317 class HashKey(object): 319 class HashKey(object):
318 ''' 320 '''
319 Used to store a value with a given hash. 321 Used to store a value with a given hash.
320 ''' 322 '''
321 323
322 __slots__ = ['hash', 'eq'] 324 __slots__ = ['hash', 'eq']
323 325
324 def __init__(self, hash, eq=None): 326 def __init__(self, hash, eq=None):
325 self.hash = hash 327 self.hash = hash
326 self.eq = eq 328 self.eq = eq
327 329
328 def __hash__(self): 330 def __hash__(self):
329 return self.hash 331 return self.hash
330 332
331 def __eq__(self, other): 333 def __eq__(self, other):
332 try: 334 try:
333 return other.hash == self.hash and other.eq == self.eq 335 return other.hash == self.hash and other.eq == self.eq
334 except AttributeError: 336 except AttributeError:
335 return False 337 return False
336 338
337 339
Powered by Google Project Hosting