My favorites | Sign in
Project Home Downloads Wiki Issues Source
Repository:
Checkout   Browse   Changes   Clones  
Changes to /src/lepl/lexer/blocks/matchers.py
fd34eef202f5 vs. 1bc735ea4745 Compare: vs.  Format:
Revision 1bc735ea4745
Go to: 
Project members, sign in to write a code review
/src/lepl/lexer/blocks/matchers.py   fd34eef202f5 /src/lepl/lexer/blocks/matchers.py   1bc735ea4745
1 1
2 # The contents of this file are subject to the Mozilla Public License 2 # The contents of this file are subject to the Mozilla Public License
3 # (MPL) Version 1.1 (the "License"); you may not use this file except 3 # (MPL) Version 1.1 (the "License"); you may not use this file except
4 # in compliance with the License. You may obtain a copy of the License 4 # in compliance with the License. You may obtain a copy of the License
5 # at http://www.mozilla.org/MPL/ 5 # at http://www.mozilla.org/MPL/
6 # 6 #
7 # Software distributed under the License is distributed on an "AS IS" 7 # Software distributed under the License is distributed on an "AS IS"
8 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 8 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
9 # the License for the specific language governing rights and 9 # the License for the specific language governing rights and
10 # limitations under the License. 10 # limitations under the License.
11 # 11 #
12 # The Original Code is LEPL (http://www.acooke.org/lepl) 12 # The Original Code is LEPL (http://www.acooke.org/lepl)
13 # The Initial Developer of the Original Code is Andrew Cooke. 13 # The Initial Developer of the Original Code is Andrew Cooke.
14 # Portions created by the Initial Developer are Copyright (C) 2009-2010 14 # Portions created by the Initial Developer are Copyright (C) 2009-2010
15 # Andrew Cooke (andrew@acooke.org). All Rights Reserved. 15 # Andrew Cooke (andrew@acooke.org). All Rights Reserved.
16 # 16 #
17 # Alternatively, the contents of this file may be used under the terms 17 # Alternatively, the contents of this file may be used under the terms
18 # of the LGPL license (the GNU Lesser General Public License, 18 # of the LGPL license (the GNU Lesser General Public License,
19 # http://www.gnu.org/licenses/lgpl.html), in which case the provisions 19 # http://www.gnu.org/licenses/lgpl.html), in which case the provisions
20 # of the LGPL License are applicable instead of those above. 20 # of the LGPL License are applicable instead of those above.
21 # 21 #
22 # If you wish to allow use of your version of this file only under the 22 # If you wish to allow use of your version of this file only under the
23 # terms of the LGPL License and not to allow others to use your version 23 # terms of the LGPL License and not to allow others to use your version
24 # of this file under the MPL, indicate your decision by deleting the 24 # of this file under the MPL, indicate your decision by deleting the
25 # provisions above and replace them with the notice and other provisions 25 # provisions above and replace them with the notice and other provisions
26 # required by the LGPL License. If you do not delete the provisions 26 # required by the LGPL License. If you do not delete the provisions
27 # above, a recipient may use your version of this file under either the 27 # above, a recipient may use your version of this file under either the
28 # MPL or the LGPL License. 28 # MPL or the LGPL License.
29 29
30 30
31 from lepl.lexer.matchers import Token, RestrictTokensBy 31 from lepl.lexer.matchers import Token, RestrictTokensBy
32 from lepl.lexer.blocks.lexer import INDENT 32 from lepl.lexer.blocks.lexer import INDENT
33 from lepl.lexer.blocks.monitor import BlockMonitor 33 from lepl.lexer.blocks.monitor import BlockMonitor
34 from lepl.core.parser import tagged 34 from lepl.core.parser import tagged
35 from lepl.lexer.blocks.support import OffsideError 35 from lepl.lexer.blocks.support import OffsideError
36 from lepl.matchers.support import OperatorMatcher, coerce_ 36 from lepl.matchers.support import OperatorMatcher, coerce_, NoMemo
37 from lepl.matchers.combine import And 37 from lepl.matchers.combine import And
38 from lepl.lexer.lines.matchers import LineEnd 38 from lepl.lexer.lines.matchers import LineEnd
39 from lepl.support.lib import fmt 39 from lepl.support.lib import fmt
40 from lepl.stream.core import s_key 40 from lepl.stream.core import s_key
41 41
42 42
43 NO_BLOCKS = object() 43 NO_BLOCKS = object()
44 ''' 44 '''
45 Magic initial value for block_offset to disable indentation checks. 45 Magic initial value for block_offset to disable indentation checks.
46 ''' 46 '''
47 47
48 48
49 class Indent(Token): 49 class Indent(Token):
50 ''' 50 '''
51 Match an indent (space at start of line with offside lexer). 51 Match an indent (space at start of line with offside lexer).
52 ''' 52 '''
53 53
54 def __init__(self, regexp=None, content=None, id_=None, alphabet=None, 54 def __init__(self, regexp=None, content=None, id_=None, alphabet=None,
55 complete=True, compiled=False): 55 complete=True, compiled=False):
56 ''' 56 '''
57 Arguments used only to support cloning. 57 Arguments used only to support cloning.
58 ''' 58 '''
59 super(Indent, self).__init__(regexp=None, content=None, id_=INDENT, 59 super(Indent, self).__init__(regexp=None, content=None, id_=INDENT,
60 alphabet=None, complete=True, 60 alphabet=None, complete=True,
61 compiled=compiled) 61 compiled=compiled)
62 self.monitor_class = BlockMonitor 62 self.monitor_class = BlockMonitor
63 self._current_indent = None 63 self._current_indent = None
64 64
65 def on_push(self, monitor): 65 def on_push(self, monitor):
66 ''' 66 '''
67 Read the global indentation level. 67 Read the global indentation level.
68 ''' 68 '''
69 self._current_indent = monitor.indent 69 self._current_indent = monitor.indent
70 70
71 def on_pop(self, monitor): 71 def on_pop(self, monitor):
72 ''' 72 '''
73 Unused 73 Unused
74 ''' 74 '''
75 75
76 @tagged 76 @tagged
77 def _match(self, stream_in): 77 def _match(self, stream_in):
78 ''' 78 '''
79 Check that we match the current level 79 Check that we match the current level
80 ''' 80 '''
81 if self._current_indent is None: 81 if self._current_indent is None:
82 raise OffsideError('No initial indentation has been set. ' 82 raise OffsideError('No initial indentation has been set. '
83 'You probably have not specified one of ' 83 'You probably have not specified one of '
84 'block_policy or block_start in the ' 84 'block_policy or block_start in the '
85 'configuration') 85 'configuration')
86 try: 86 try:
87 generator = super(Indent, self)._match(stream_in) 87 generator = super(Indent, self)._match(stream_in)
88 while True: 88 while True:
89 (indent, stream) = yield generator 89 (indent, stream) = yield generator
90 self._debug(fmt('Indent {0!r}', indent)) 90 self._debug(fmt('Indent {0!r}', indent))
91 if indent[0] and indent[0][-1] == '\n': indent[0] = indent[0][:-1] 91 if indent[0] and indent[0][-1] == '\n': indent[0] = indent[0][:-1]
92 if self._current_indent == NO_BLOCKS or \ 92 if self._current_indent == NO_BLOCKS or \
93 len(indent[0]) == self._current_indent: 93 len(indent[0]) == self._current_indent:
94 yield (indent, stream) 94 yield (indent, stream)
95 else: 95 else:
96 self._debug( 96 self._debug(
97 fmt('Incorrect indent ({0:d} != len({1!r}), {2:d})', 97 fmt('Incorrect indent ({0:d} != len({1!r}), {2:d})',
98 self._current_indent, indent[0], 98 self._current_indent, indent[0],
99 len(indent[0]))) 99 len(indent[0])))
100 except StopIteration: 100 except StopIteration:
101 return 101 return
102 102
103 103
104 def constant_indent(n_spaces): 104 def constant_indent(n_spaces):
105 ''' 105 '''
106 Construct a simple policy for `Block` that increments the indent 106 Construct a simple policy for `Block` that increments the indent
107 by some fixed number of spaces. 107 by some fixed number of spaces.
108 ''' 108 '''
109 def policy(current, _indent): 109 def policy(current, _indent):
110 ''' 110 '''
111 Increment current by n_spaces 111 Increment current by n_spaces
112 ''' 112 '''
113 return current + n_spaces 113 return current + n_spaces
114 return policy 114 return policy
115 115
116 116
117 def rightmost(_current, indent): 117 def rightmost(_current, indent):
118 ''' 118 '''
119 Another simple policy that matches whatever indent is used. 119 Another simple policy that matches whatever indent is used.
120 ''' 120 '''
121 return len(indent[0]) 121 return len(indent[0])
122 122
123 123
124 def to_right(current, indent): 124 def to_right(current, indent):
125 ''' 125 '''
126 This allows new blocks to be used without any introduction (eg no colon 126 This allows new blocks to be used without any introduction (eg no colon
127 on the preceding line). See the "closed_bug" test for more details. 127 on the preceding line). See the "closed_bug" test for more details.
128 ''' 128 '''
129 new = len(indent[0]) 129 new = len(indent[0])
130 if new <= current: 130 if new <= current:
131 raise StopIteration 131 raise StopIteration
132 return new 132 return new
133 133
134 134
135 class UncheckedIndent(Indent): 135 class UncheckedIndent(Indent):
136 136
137 def on_push(self, monitor): 137 def on_push(self, monitor):
138 ''' 138 '''
139 Ignore the global indentation level. 139 Ignore the global indentation level.
140 ''' 140 '''
141 self._current_indent = NO_BLOCKS 141 self._current_indent = NO_BLOCKS
142 142
143 143
144 144
145 # pylint: disable-msg=W0105 145 # pylint: disable-msg=W0105
146 # epydoc convention 146 # epydoc convention
147 DEFAULT_TABSIZE = 8 147 DEFAULT_TABSIZE = 8
148 ''' 148 '''
149 The default number of spaces for a tab. 149 The default number of spaces for a tab.
150 ''' 150 '''
151 151
152 DEFAULT_POLICY = constant_indent(DEFAULT_TABSIZE) 152 DEFAULT_POLICY = constant_indent(DEFAULT_TABSIZE)
153 ''' 153 '''
154 By default, expect an indent equivalent to a tab. 154 By default, expect an indent equivalent to a tab.
155 ''' 155 '''
156 156
157 # pylint: disable-msg=E1101, W0212, R0901, R0904 157 # pylint: disable-msg=E1101, W0212, R0901, R0904
158 # pylint conventions 158 # pylint conventions
159 class Block(OperatorMatcher): 159 class Block(OperatorMatcher, NoMemo):
160 ''' 160 '''
161 Set a new indent level for the enclosed matchers (typically `BLine` and 161 Set a new indent level for the enclosed matchers (typically `BLine` and
162 `Block` instances). 162 `Block` instances).
163 163
164 In the simplest case, this might increment the global indent by 4, say. 164 In the simplest case, this might increment the global indent by 4, say.
165 In a more complex case it might look at the current token, expecting an 165 In a more complex case it might look at the current token, expecting an
166 `Indent`, and set the global indent at that amount if it is larger 166 `Indent`, and set the global indent at that amount if it is larger
167 than the current value. 167 than the current value.
168 168
169 A block will always match an `Indent`, but will not consume it 169 A block will always match an `Indent`, but will not consume it
170 (it will remain in the stream after the block has finished). 170 (it will remain in the stream after the block has finished).
171 171
172 The usual memoization of left recursive calls will not detect problems 172 The usual memoization of left recursive calls will not detect problems
173 with nested blocks (because the indentation changes), so instead we 173 with nested blocks (because the indentation changes), so instead we
174 track and block nested calls manually. 174 track and block nested calls manually.
175 ''' 175 '''
176 176
177 POLICY = 'policy' 177 POLICY = 'policy'
178 # class-wide default 178 # class-wide default
179 __indent = UncheckedIndent() 179 __indent = UncheckedIndent()
180 180
181 # Python 2.6 does not support this syntax 181 # Python 2.6 does not support this syntax
182 # def __init__(self, *lines, policy=None, indent=None): 182 # def __init__(self, *lines, policy=None, indent=None):
183 def __init__(self, *lines, **kargs): 183 def __init__(self, *lines, **kargs):
184 ''' 184 '''
185 Lines are invoked in sequence (like `And()`). 185 Lines are invoked in sequence (like `And()`).
186 186
187 The policy is passed the current level and the indent and must 187 The policy is passed the current level and the indent and must
188 return a new level. Typically it is set globally by rewriting with 188 return a new level. Typically it is set globally by rewriting with
189 a default in the configuration. If it is given as an integer then 189 a default in the configuration. If it is given as an integer then
190 `constant_indent` is used to create a policy from that. 190 `constant_indent` is used to create a policy from that.
191 191
192 indent is the matcher used to match indents, and is exposed for 192 indent is the matcher used to match indents, and is exposed for
193 rewriting/extension (in other words, ignore it). 193 rewriting/extension (in other words, ignore it).
194 ''' 194 '''
195 super(Block, self).__init__() 195 super(Block, self).__init__()
196 self._args(lines=lines) 196 self._args(lines=lines)
197 policy = kargs.get(self.POLICY, DEFAULT_POLICY) 197 policy = kargs.get(self.POLICY, DEFAULT_POLICY)
198 if isinstance(policy, int): 198 if isinstance(policy, int):
199 policy = constant_indent(policy) 199 policy = constant_indent(policy)
200 self._karg(policy=policy) 200 self._karg(policy=policy)
201 indent = kargs.get('indent', self.__indent) 201 indent = kargs.get('indent', self.__indent)
202 self._karg(indent=indent) 202 self._karg(indent=indent)
203 self.monitor_class = BlockMonitor 203 self.monitor_class = BlockMonitor
204 self.__monitor = None 204 self.__monitor = None
205 self.__streams = set() 205 self.__streams = set()
206 206
207 def on_push(self, monitor): 207 def on_push(self, monitor):
208 ''' 208 '''
209 Store a reference to the monitor which we will update when _match 209 Store a reference to the monitor which we will update when _match
210 is invoked (ie immediately). 210 is invoked (ie immediately).
211 ''' 211 '''
212 self.__monitor = monitor 212 self.__monitor = monitor
213 213
214 def on_pop(self, monitor): 214 def on_pop(self, monitor):
215 ''' 215 '''
216 Remove the indent we added. 216 Remove the indent we added.
217 ''' 217 '''
218 # only if we pushed a value to monitor (see below) 218 # only if we pushed a value to monitor (see below)
219 if self.__monitor: 219 if self.__monitor:
220 self.__monitor = None 220 self.__monitor = None
221 else: 221 else:
222 monitor.pop_level() 222 monitor.pop_level()
223 223
224 @tagged 224 @tagged
225 def _match(self, stream_in): 225 def _match(self, stream_in):
226 ''' 226 '''
227 Pull indent and call the policy and update the global value, 227 Pull indent and call the policy and update the global value,
228 then evaluate the contents. 228 then evaluate the contents.
229 ''' 229 '''
230 # detect a nested call 230 # detect a nested call
231 key = s_key(stream_in) 231 key = s_key(stream_in)
232 if key in self.__streams: 232 if key in self.__streams:
233 self._debug('Avoided left recursive call to Block.') 233 self._debug('Avoided left recursive call to Block.')
234 return 234 return
235 self.__streams.add(key) 235 self.__streams.add(key)
236 try: 236 try:
237 (indent, _stream) = yield self.indent._match(stream_in) 237 (indent, _stream) = yield self.indent._match(stream_in)
238 current = self.__monitor.indent 238 current = self.__monitor.indent
239 self.__monitor.push_level(self.policy(current, indent)) 239 self.__monitor.push_level(self.policy(current, indent))
240 # this flags we have pushed and need to pop 240 # this flags we have pushed and need to pop
241 self.__monitor = None 241 self.__monitor = None
242 242
243 generator = And(*self.lines)._match(stream_in) 243 generator = And(*self.lines)._match(stream_in)
244 while True: 244 while True:
245 yield (yield generator) 245 yield (yield generator)
246 finally: 246 finally:
247 self.__streams.remove(key) 247 self.__streams.remove(key)
248 248
249 249
250 # pylint: disable-msg=C0103 250 # pylint: disable-msg=C0103
251 # consistent interface 251 # consistent interface
252 def BLine(matcher, indent=True): 252 def BLine(matcher, indent=True):
253 ''' 253 '''
254 Match the matcher within a line with block indent. 254 Match the matcher within a line with block indent.
255 ''' 255 '''
256 if indent: 256 if indent:
257 return ~Indent() & matcher & ~LineEnd() 257 return ~Indent() & matcher & ~LineEnd()
258 else: 258 else:
259 return ~UncheckedIndent() & matcher & ~LineEnd() 259 return ~UncheckedIndent() & matcher & ~LineEnd()
260 260
261 261
262 def ContinuedBLineFactory(matcher): 262 def ContinuedBLineFactory(matcher):
263 ''' 263 '''
264 Create a `BLine` that extends over multiple lines. The "line wrap" 264 Create a `BLine` that extends over multiple lines. The "line wrap"
265 token is matched by the matcher supplied. 265 token is matched by the matcher supplied.
266 ''' 266 '''
267 matcher = coerce_(matcher, lambda regexp: Token(regexp)) 267 matcher = coerce_(matcher, lambda regexp: Token(regexp))
268 start = Indent() 268 start = Indent()
269 end = LineEnd() 269 end = LineEnd()
270 restricted = RestrictTokensBy(matcher, end, start) 270 restricted = RestrictTokensBy(matcher, end, start)
271 271
272 def factory(matcher): 272 def factory(matcher):
273 ''' 273 '''
274 The `BLine` replacement. 274 The `BLine` replacement.
275 ''' 275 '''
276 line = ~start & matcher & ~end 276 line = ~start & matcher & ~end
277 return restricted(line) 277 return restricted(line)
278 278
279 return factory 279 return factory
280 280
281 281
282 def Extend(matcher): 282 def Extend(matcher):
283 ''' 283 '''
284 Apply the give matcher to a token stream that ignores line endings and 284 Apply the give matcher to a token stream that ignores line endings and
285 starts (so it matches over multiple lines). 285 starts (so it matches over multiple lines).
286 ''' 286 '''
287 start = Indent() 287 start = Indent()
288 end = LineEnd() 288 end = LineEnd()
289 return RestrictTokensBy(end, start)(matcher) 289 return RestrictTokensBy(end, start)(matcher)
Powered by Google Project Hosting