1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334

import sys
import re
from lxml import etree # requires lxml 2.0
from copy import deepcopy

print "HTML5 Spec Splitter"

absolute_uris = False
use_html5lib_parser = False
use_html5lib_serialiser = False
file_args = []

for arg in sys.argv[1:]:
    if arg == '--absolute':
        absolute_uris = True
    elif arg == '--html5lib-parser':
        use_html5lib_parser = True
    elif arg == '--html5lib-serialiser':
        use_html5lib_serialiser = True
    else:
        file_args.append(arg)

if len(file_args) != 2:
    print 'Run like "python [options] spec-splitter.py index multipage"'
    print '(The directory "multipage" must already exist)'
    print
    print 'Options:'
    print '  --absolute ............. convert relative URLs to absolute (e.g. for images)'
    print '  --html5lib-parser ...... use html5lib parser instead of lxml'
    print '  --html5lib-serialiser .. use html5lib serialiser instead of lxml'
    sys.exit()

if use_html5lib_parser or use_html5lib_serialiser:
    import html5lib
    import html5lib.serializer
    import html5lib.treewalkers

index_page = 'index'

# The document is split on all <h2> elements, plus the following specific elements
# (which were chosen to split any pages that were larger than about 100-200KB, and
# may need to be adjusted as the spec changes):
split_exceptions = [
    'common-microsyntaxes', 'urls', 'fetching-resources', 'common-dom-interfaces', 'namespaces', # <-- infrastructure
    'elements', # <-- dom

    'scripting-1', 'sections', 'grouping-content', 'text-level-semantics', 'edits',
    'embedded-content-1', 'the-iframe-element', 'the-video-element', 'the-canvas-element', 'the-map-element', 'tabular-data',
    'forms', 'the-input-element', 'states-of-the-type-attribute', 'number-state', 'common-input-element-attributes', 'the-button-element', 'association-of-controls-and-forms',
    'interactive-elements', 'commands', 'common-idioms', 'selectors', # <-- semantics

    'origin-0', 'timers', 'offline', 'history', 'links', # <-- browsers
    'dnd', # <-- editing

    'workers', 'network', 'web-messaging', 'webstorage',
    'parsing', 'tokenization', 'tree-construction', 'the-end', 'named-character-references', # <-- syntax
]


print "Parsing..."

# Parse document
if use_html5lib_parser:
    parser = html5lib.html5parser.HTMLParser(tree = html5lib.treebuilders.getTreeBuilder('lxml'))
    doc = parser.parse(open(file_args[0]), encoding='utf-8')
else:
    parser = etree.HTMLParser(encoding='utf-8')
    doc = etree.parse(open(file_args[0]), parser)

print "Splitting..."

doctitle = doc.find('.//title').text

# Absolutise some references, so the spec can be hosted elsewhere
if absolute_uris:
    for a in ('href', 'src'):
        for t in ('link', 'script', 'img'):
            for e in doc.findall('//%s[@%s]' % (t, a)):
                if e.get(a)[0] == '/':
                    e.set(a, 'http://www.whatwg.org' + e.get(a))
                else:
                    e.set(a, 'http://www.whatwg.org/specs/web-apps/current-work/' + e.get(a))

# Extract the body from the source document
original_body = doc.find('body')

# Create an empty body, for the page content to be added into later
default_body = etree.Element('body')
if original_body.get('class'): default_body.set('class', original_body.get('class'))
if original_body.get('onload'): default_body.set('onload', 'fixBrokenLink(); %s' % original_body.get('onload'))
original_body.getparent().replace(original_body, default_body)

# Extract the header, so we can reuse it in every page
header = original_body.find('.//*[@class="head"]')

# Make a stripped-down version of it
short_header = deepcopy(header)
del short_header[2:]

# Extract the items in the TOC (remembering their nesting depth)
def extract_toc_items(items, ol, depth):
    for li in ol.iterchildren():
        for c in li.iterchildren():
            if c.tag == 'a':
                assert c.get('href')[0] == '#'
                items.append( (depth, c.get('href')[1:], c) )
            elif c.tag == 'ol':
                extract_toc_items(items, c, depth+1)
toc_items = []
extract_toc_items(toc_items, original_body.find('.//ol[@class="toc"]'), 0)

# Prepare the link-fixup script
link_fixup_script = etree.XML('<script src="link-fixup.js"/>')
doc.find('head')[-1].tail = '\n  '
doc.find('head').append(link_fixup_script)
link_fixup_script.tail = '\n  '

# Stuff for fixing up references:

def get_page_filename(name):
    return '%s.html' % name

# Finds all the ids and remembers which page they were on
id_pages = {}
def extract_ids(page, node):
    if node.get('id'):
        id_pages[node.get('id')] = page
    for e in node.findall('.//*[@id]'):
        id_pages[e.get('id')] = page

# Updates all the href="#id" to point to page#id
missing_warnings = set()
def fix_refs(page, node):
    for e in node.findall('.//a[@href]'):
        if e.get('href')[0] == '#':
            id = e.get('href')[1:]
            if id in id_pages:
                if id_pages[id] != page: # only do non-local links
                    e.set('href', '%s#%s' % (get_page_filename(id_pages[id]), id))
            else:
                missing_warnings.add(id)

def report_broken_refs():
    for id in sorted(missing_warnings):
        print "warning: can't find target for #%s" % id

pages = [] # for saving all the output, so fix_refs can be called in a second pass

# Iterator over the full spec's body contents
child_iter = original_body.iterchildren()

def add_class(e, cls):
    if e.get('class'):
        e.set('class', e.get('class') + ' ' + cls)
    else:
        e.set('class', cls)

# Contents/intro page:

page = deepcopy(doc)
add_class(page.getroot(), 'split index')
page_body = page.find('body')

# Keep copying stuff from the front of the source document into this
# page, until we find the first heading that isn't class="no-toc"
for e in child_iter:
    if e.getnext().tag == 'h2' and 'no-toc' not in (e.getnext().get('class') or '').split(' '):
        break
    page_body.append(e)

pages.append( (index_page, page, 'Front cover') )

# Section/subsection pages:

def first_elm(e):
    for c in e.iterchildren(tag=etree.Element):
        return c
    return None

def should_split(e):
    if e.tag == 'h2': return True
    if e.get('id') in split_exceptions: return True
    # handle wrapping <div>
    if e.tag == 'div':
        c = first_elm(e)
        if c:
            if c.tag == 'h2': return True
            if c.get('id') in split_exceptions: return True
    return False

def get_heading_text_and_id(e):
    if e.tag == 'div':
        node = first_elm(e)
    else:
        node = e
    title = re.sub('\s+', ' ', etree.tostring(node, method='text').strip())
    return title, node.get('id')

for heading in child_iter:
    # Handle the heading for this section
    title, name = get_heading_text_and_id(heading)
    if name == index_page: name = 'section-%s' % name
    print '  <%s> %s - %s' % (heading.tag, name, title)

    page = deepcopy(doc)
    add_class(page.getroot(), 'split chapter')
    page_body = page.find('body')

    page.find('//title').text = title + u' \u2014 ' + doctitle

    # Add the header
    page_body.append(deepcopy(short_header))

    # Add the page heading
    page_body.append(deepcopy(heading))
    extract_ids(name, heading)

    # Keep copying stuff from the source, until we reach the end of the
    # document or find a header to split on
    e = heading
    while e.getnext() is not None and not should_split(e.getnext()):
        e = child_iter.next()
        extract_ids(name, e)
        page_body.append(deepcopy(e))

    pages.append( (name, page, title) )

# Fix the links, and add some navigation:

for i in range(len(pages)):
    name, doc, title = pages[i]

    fix_refs(name, doc)

    if name == index_page: continue # don't add nav links to the TOC page

    head = doc.find('head')

    nav = etree.Element('nav')
    nav.text = '\n   '
    nav.tail = '\n\n  '

    if i > 1:
        href = get_page_filename(pages[i-1][0])
        title = pages[i-1][2]
        a = etree.XML(u'<a href="%s">\u2190 %s</a>' % (href, title))
        a.tail = u' \u2013\n   '
        nav.append(a)
        link = etree.XML('<link href="%s" title="%s" rel="prev"/>' % (href, title))
        link.tail = '\n  '
        head.append(link)

    a = etree.XML('<a href="%s.html#contents">Table of contents</a>' % index_page)
    a.tail = '\n  '
    nav.append(a)
    link = etree.XML('<link href="%s.html#contents" title="Table of contents" rel="index"/>' % index_page)
    link.tail = '\n  '
    head.append(link)

    if i != len(pages)-1:
        href = get_page_filename(pages[i+1][0])
        title = pages[i+1][2]
        a = etree.XML(u'<a href="%s">%s \u2192</a>' % (href, title))
        a.tail = '\n  '
        nav.append(a)
        a.getprevious().tail = u' \u2013\n   '
        link = etree.XML('<link href="%s" title="%s" rel="next"/>' % (href, title))
        link.tail = '\n  '
        head.append(link)

    # Add a subset of the TOC to each page:

    # Find the items that are on this page
    new_toc_items = [ (d, id, e) for (d, id, e) in toc_items if id_pages[id] == name ]
    if len(new_toc_items) > 1: # don't bother if there's only one item, since it looks silly
        # Construct the new toc <ol>
        new_toc = etree.XML(u'<ol class="toc"/>')
        cur_ol = new_toc
        cur_li = None
        cur_depth = 0
        # Add each item, reconstructing the nested <ol>s and <li>s to preserve
        # the nesting depth of each item
        for (d, id, e) in new_toc_items:
            while d > cur_depth:
                if cur_li is None:
                    cur_li = etree.XML(u'<li/>')
                    cur_ol.append(cur_li)
                cur_ol = etree.XML('<ol/>')
                cur_li.append(cur_ol)
                cur_li = None
                cur_depth += 1
            while d < cur_depth:
                cur_li = cur_ol.getparent()
                cur_ol = cur_li.getparent()
                cur_depth -= 1
            cur_li = etree.XML(u'<li/>')
            cur_li.append(deepcopy(e))
            cur_ol.append(cur_li)
        nav.append(new_toc)

    doc.find('body').insert(1, nav) # after the header

report_broken_refs()

print "Outputting..."

# Output all the pages
for name, doc, title in pages:
    f = open('%s/%s' % (file_args[1], get_page_filename(name)), 'w')
    if use_html5lib_serialiser:
        tokens = html5lib.treewalkers.getTreeWalker('lxml')(doc)
        serializer = html5lib.serializer.HTMLSerializer(quote_attr_values=True, inject_meta_charset=False)
        for text in serializer.serialize(tokens, encoding='us-ascii'):
            f.write(text)
    else:
        f.write(etree.tostring(doc, pretty_print=False, method="html"))

# Generate the script to fix broken links
f = open('%s/fragment-links.js' % (file_args[1]), 'w')
links = ','.join('"%s":"%s"' % (k.replace("\\", "\\\\").replace('"', '\\"'), v) for (k,v) in id_pages.items())
f.write('var fragment_links = { ' + re.sub(r"([^\x20-\x7f])", lambda m: "\\u%04x" % ord(m.group(1)), links) + ' };\n')
f.write("""
var fragid = window.location.hash.substr(1);
if (!fragid) { /* handle section-foo.html links from the old multipage version, and broken foo.html from the new version */
    var m = window.location.pathname.match(/\/(?:section-)?([\w\-]+)\.html/);
    if (m) fragid = m[1];
}
var page = fragment_links[fragid];
if (page) {
    window.location.replace(page+'.html#'+fragid);
}
""")

print "Done."