My favorites | Sign in
Project Home Downloads Wiki Issues Source
Checkout   Browse   Changes    
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
#!/usr/bin/env python
# coding: utf-8

# voter.py - vote reader for 2008 primaries
# Copyright (c) 2008 Michael Geary - http://mg.to/
# Free Beer and Free Speech License (MIT+GPL)
# http://freebeerfreespeech.org/
# http://www.opensource.org/licenses/mit-license.php
# http://www.opensource.org/licenses/gpl-2.0.php

import csv
import os
import re
import time
import urllib
import urllib2

from candidates import candidates
#from template import *
import private
import random
import simplejson as sj
import states

votespath = '../election-data/votes'

#def str( text ):
# strings = {
# 'county': 'town',
# 'counties': 'towns'
# }
# return strings[text] or text

def formatNumber( number ):
return str(number)

def json( obj ):
if 0:
# Pretty print
json = sj.dumps( obj, indent=4 )
else:
# Use compact format, but add some newlines in the hope of using less space for svn revisions
json = sj.dumps( obj, separators=( ',', ':' ) )
json = re.sub( '\],"', '],\n"', json )
json = re.sub( ':\[{', ':[\n{', json )
json = re.sub( '":{', '":\n{', json )
json = re.sub( '},{', '},\n{', json )
json = re.sub( '},"', '},\n"', json )
return json

def feedCSV( feed ):
return '%s/%s' %( votespath, feed['file'] )

def fetchData( feed ):
if 'url' not in feed: return
url = feed['url']
file = feedCSV(feed)
print 'Retrieving %s from:\n%s' %( file, url )
urllib.urlretrieve( url, file )

## Correct error in census data for Wentworth's Location
#if( name == "Wentworth" and number == '9' ):
# name = "Wentworth's Location"

#def clearParties( entity ):
# for party in 'dem', 'gop':
# ep = entity['parties'][party]
# if 'votes' in ep: del ep['votes']
# if 'precincts' in ep: del ep['precincts']
#
#def clearVotes( feed ):
# for state in states.array:
# clearParties( state )
# if 'counties' in state:
# for county in state['counties'].itervalues():
# clearParties( county )

def readVotes( feed ):
print 'Processing %s' % feed['file']
state = feed.get('state')
reader = csv.reader( open( feedCSV(feed), 'rb' ) )
header = []
while header == []:
header = reader.next()
if state: header.insert( 0, 'state' )
#print header
for row in reader:
if len(row) < 2: continue
if state: row.insert( 0, state )
# Maine hack
if row[2] == ' S.D.':
row[1] += ',' + row[2]
row.pop( 2 )
setData( feed, header, row )

def setData( feed, header, row ):
entity = state = states.byAbbr[ row[0] ]
if 'counties' not in state: state['counties'] = {}
setVotes( feed, state, header, row )

def getPrecincts( row ):
#print 'getPrecincts %s %s %s %s' %( row[0], row[1], row[2], row[3] )
return {
'reporting': int(row[3]),
'total': int(row[2])
}

fixcols = { 'trancredo': 'tancredo' }
ignorecols = { 'other':1, 'other-d':1, 'other-r':1, 'total-d':1, 'total-r':1, 'undecided-d':1, 'undecided-r':1, 'Uncommitted-D':1, 'Uncommitted-R':1, 'Uninstructed-D':1, 'Uninstructed-R':1, 'write-ins-d':1, 'write-ins-r':1 }

def fixCountyName( name ):
name = re.sub( ' County$', '', name )
fixNames = {
# NH
"Harts Location": "Hart's Location",
"Waterville": "Waterville Valley",
# PR - this is stupid but it works (should do it algorithmically)
"ADJUNTAS": "Adjuntas",
"AGUADA": "Aguada",
"AGUADILLA": "Aguadilla",
"AGUAS BUENAS": "Aguas Buenas",
"AIBONITO": "Aibonito",
"ARECIBO": "Arecibo",
"ARROYO": "Arroyo",
"AÑASCO": "Añasco",
"BARCELONETA": "Barceloneta",
"BARRANQUITAS": "Barranquitas",
"BAYAMÓN": "Bayamón",
"BAYAMON": "Bayamón",
"CABO ROJO": "Cabo Rojo",
"CAGUAS": "Caguas",
"CAMUY": "Camuy",
"CANÓVANAS": "Canóvanas",
"CANOVANAS": "Canóvanas",
"CAROLINA": "Carolina",
"CATAÑO": "Cataño",
"CAYEY": "Cayey",
"CEIBA": "Ceiba",
"CIALES": "Ciales",
"CIDRA": "Cidra",
"COAMO": "Coamo",
"COMERÍO": "Comerío",
"COMERIO": "Comerío",
"COROZAL": "Corozal",
"CULEBRA": "Culebra",
"DORADO": "Dorado",
"FAJARDO": "Fajardo",
"FLORIDA": "Florida",
"GUAYAMA": "Guayama",
"GUAYANILLA": "Guayanilla",
"GUAYNABO": "Guaynabo",
"GURABO": "Gurabo",
"GUÁNICA": "Guánica",
"GUANICA": "Guánica",
"HATILLO": "Hatillo",
"HORMIGUEROS": "Hormigueros",
"HUMACAO": "Humacao",
"ISABELA": "Isabela",
"JAYUYA": "Jayuya",
"JUANA DÍAZ": "Juana Díaz",
"JUANA DIAZ": "Juana Díaz",
"JUNCOS": "Juncos",
"LAJAS": "Lajas",
"LARES": "Lares",
"LAS MARÍAS": "Las Marías",
"LAS MARIAS": "Las Marías",
"LAS PIEDRAS": "Las Piedras",
"LOÍZA": "Loíza",
"LOIZA": "Loíza",
"LUQUILLO": "Luquillo",
"MANATÍ": "Manatí",
"MANATI": "Manatí",
"MARICAO": "Maricao",
"MAUNABO": "Maunabo",
"MAYAGÜEZ": "Mayagüez",
"MOCA": "Moca",
"MOROVIS": "Morovis",
"NAGUABO": "Naguabo",
"NARANJITO": "Naranjito",
"OROCOVIS": "Orocovis",
"PATILLAS": "Patillas",
"PEÑUELAS": "Peñuelas",
"PONCE": "Ponce",
"QUEBRADILLAS": "Quebradillas",
"RINCÓN": "Rincón",
"RINCON": "Rincón",
"RÍO GRANDE": "Río Grande",
"RIO GRANDE": "Río Grande",
"SABANA GRANDE": "Sabana Grande",
"SALINAS": "Salinas",
"SAN GERMÁN": "San Germán",
"SAN GERMAN": "San Germán",
"SAN JUAN": "San Juan",
"SAN LORENZO": "San Lorenzo",
"SAN SEBASTIÁN": "San Sebastián",
"SAN SEBASTIAN": "San Sebastián",
"SANTA ISABEL": "Santa Isabel",
"TOA ALTA": "Toa Alta",
"TOA BAJA": "Toa Baja",
"TRUJILLO ALTO": "Trujillo Alto",
"UTUADO": "Utuado",
"VEGA ALTA": "Vega Alta",
"VEGA BAJA": "Vega Baja",
"VIEQUES": "Vieques",
"VILLALBA": "Villalba",
"YABUCOA": "Yabucoa",
"YAUCO": "Yauco"
}
if( name in fixNames ):
name = fixNames[name]
#print 'County: %s' % name
return name

def setVotes( feed, entity, header, row ):
#print 'setVotes', row
counties = entity['counties']
countyname = fixCountyName( row[1] )
if countyname != '*':
if countyname not in counties:
counties[countyname] = { 'parties':{ 'dem':{}, 'gop':{} } }
entity = counties[countyname]
if ( row[0] == 'NE' or row[0] == 'WV' ) and row[4] != '':
if row[5] == '':
#print 'fixing 5'
row[5] = '0'
if row[6] == '':
#print 'fixing 6'
row[6] = '0'
#print row
first = 4
if feed['file'] == '0205.csv' and row[0] == 'ID': first = 14
for col in xrange( first, len(header) ):
if col >= len(row) or row[col] == '': continue
name = header[col]
name = fixcols.get( name, name )
if name in ignorecols: continue
candidate = candidates['byname'][name]
party = candidate['party']
p = entity['parties'][party]
if 'precincts' not in p: p['precincts'] = getPrecincts( row )
if 'votes' not in p: p['votes'] = {}
votes = int(row[col])
if votes: p['votes'][name] = votes

def percentage( n ):
pct = int( round( 100.0 * float(n) ) )
if pct == 100 and n < 1: pct = 99
return pct

def sortVotes( party ):
if not party.get('votes'): party['votes'] = {}
tally = []
for name, votes in party['votes'].iteritems():
delegates = 0
if 'delegatelist' in party:
if name in party['delegatelist']:
delegates = party['delegatelist'][name]
if delegates:
tally.append({ 'name':name, 'votes':votes, 'delegates':delegates })
else:
tally.append({ 'name':name, 'votes':votes })
tally.sort( lambda a, b: b['votes'] - a['votes'] )
party['votes'] = tally
if 'delegatehtml' in party: del party['delegatehtml']
if 'delegatelist' in party: del party['delegatelist']

#def setPins( locals ):
# least = most = None
# for local in locals.itervalues():
# votes = local['votes']
# if len(votes):
# n = votes[0]['votes']
# if n:
# if least == None or n < least: least = n
# if most == None or n > most: most = n
# for local in locals.itervalues():
# local['pinsize'] = 24
# votes = local['votes']
# if len(votes) and least and most:
# n = votes[0]['votes']
# if most == 1:
# if n == 1:
# local['pinsize'] = 40
# else:
# precincts = local['precincts']
# reporting = float(precincts['reporting']) / float(precincts['total'])
# fraction = float( n - least ) / float( most - least ) * reporting
# local['pinsize'] = int( 24 + fraction * 16 )

def cleanNum( n ):
return int( re.sub( '[^0-9]', '', n ) or 0 )

def addDelegates( usparty, partyname, party, state ):
if 'delegatehtml' not in party: return
row = party['delegatehtml']
party['delegates'] = cleanNum( row[1] )
party['delegatelist'] = {}
votes = party['votes']
def set( col, name ):
if len(row) + col < 2: return
n = cleanNum( row[col] )
if not n: return
print state['name'], 'delegates:', name, n
party['delegatelist'][name] = n
#if name in usparty['delegatelist']:
# usparty['delegatelist'][name] += n
#else:
# usparty['delegatelist'][name] = n
if partyname == 'dem':
usparty['delegatelist']['obama'] = 2191
usparty['delegatelist']['clinton'] = 1914
set( -2, 'obama' )
set( -1, 'clinton' )
else:
usparty['delegatelist']['mccain'] = 1563
usparty['delegatelist']['romney'] = 272
usparty['delegatelist']['huckabee'] = 282
usparty['delegatelist']['paul'] = 29
set( -4, 'mccain' )
set( -3, 'romney' )
set( -2, 'huckabee' )
set( -1, 'paul' )

def makeJson( party ):
ustotal = 0
usvotes = {}
usdelegatelist = {}
usprecincts = { 'total': 0, 'reporting': 0 }
usparty = { 'votes': usvotes, 'precincts': usprecincts, 'delegatelist': usdelegatelist }
statevotes = {}
leaders = {}
def addLeader( party ):
if len(party['votes']):
leaders[ party['votes'][0]['name'] ] = True
for state in states.array:
statetotal = 0
parties = state['parties']
if party not in parties: continue
stateparty = state['parties'][party]
stateparty['name'] = state['name']
if 'votes' not in stateparty: continue
addDelegates( usparty, party, stateparty, state )
sortVotes( stateparty )
statevotes[ state['name'] ] = stateparty
print 'Loading %s %s' %( state['name'], party )
for vote in stateparty['votes']:
name = vote['name']
count = vote['votes']
if name not in usvotes:
usvotes[name] = 0
usvotes[name] += count
ustotal += count
statetotal += count
countyvotes = {}
counties = state.get( 'counties', {} )
for countyname, county in counties.iteritems():
countyparty = county['parties'][party]
countyparty['name'] = countyname
sortVotes( countyparty )
addLeader( countyparty )
countytotal = 0
for vote in countyparty['votes']:
countytotal += vote['votes']
countyparty['total'] = countytotal
countyvotes[countyname] = countyparty
#setPins( countyvotes )
write(
'%s/%s_%s.js' %( votespath, state['abbr'].lower(), party ),
'GoogleElectionMap.votesReady(%s)' % json({
'status': 'ok',
'party': party,
'state': state['abbr'],
'total': statetotal,
'totals': stateparty,
'locals': countyvotes
}) )
sortVotes( usparty )
#setPins( statevotes )
write(
'%s/%s_%s.js' %( votespath, 'us', party ),
'GoogleElectionMap.votesReady(%s)' % json({
'status': 'ok',
'party': party,
'state': 'US',
'total': ustotal,
'totals': usparty,
'locals': statevotes
}) )
#print '%s of %s precincts reporting' %( state['precincts']['reporting'], state['precincts']['total'] )
print '%s leaders:' % party
for leader in leaders.iterkeys():
print leader

def getDelegates( party, urlparty ):
url = 'http://www.realclearpolitics.com/epolls/2008/president/%s_delegate_count.html' % urlparty
#print 'processing URL'
giRet = urllib2.urlopen(url).read()
iterator = re.finditer('<td bgcolor=""(.*?)</td>',giRet)
for match in iterator:
if re.search('href.*strong', match.group()) != None:
lastkey = cleankey(re.search('(href.*strong)', match.group()))
print lastkey
if lastkey in states.byName: states.byName[lastkey]['parties'][party]['delegatehtml'] = []
else:
if lastkey in states.byName: states.byName[lastkey]['parties'][party]['delegatehtml'].append(cleanvalue(match.group()))

def cleanvalue(value):
return re.search('>([^.]*?)(\.|</td>)', value).group(1)

def cleankey(key):
return re.search('<strong>(.*?)</strong',key.group()).group(1)

def write( name, text ):
print 'Writing %s' % name
f = open( name, 'wb' )
f.write( text )
f.close()

def update():
getDelegates( 'dem', 'democratic' );
getDelegates( 'gop', 'republican' );
for feed in private.feeds:
fetchData( feed )
readVotes( feed )
print 'Creating votes JSON...'
makeJson( 'dem' )
makeJson( 'gop' )
print 'Checking in votes JSON...'
os.system( 'svn ci -m "Vote update" %s' % votespath )
print 'Done!'

def main():
#while 1:
update()
#print 'Waiting 10 minutes...'
#time.sleep( 600 )

if __name__ == "__main__":
main()

Change log

r257 by election.maps.2008 on Jun 19, 2008   Diff
Add copyright/license info
Go to: 
Project members, sign in to write a code review

Older revisions

r215 by election.maps.2008 on Jun 6, 2008   Diff
Hack in hard coded USA total delegate
values - too much work to update
scraper to get these at this point
r213 by election.maps.2008 on Jun 4, 2008   Diff
Fix bug with missing Super Tuesday
results
r207 by election.maps.2008 on Jun 3, 2008   Diff
Get Puerto Rico working
All revisions of this file

File info

Size: 12518 bytes, 439 lines
Powered by Google Project Hosting