My favorites | Sign in
Project Home Downloads Wiki Issues Source
Checkout   Browse   Changes    
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
# = Test : Amazon SimpleDB Test
# A simple component designed to verify that
# the basic Sdb bindings are working properly and to do a littel benchmark
# to transfer a 1000 items back and forward to Amazon SimpleDb.
#
# This test uses SdbInterface directly, without using an ActiveSdb facade.
# Run with
#
# ruby runner.rb -c swarm_shoal_options.yml -t test/sdb
#
# Additional flags:
# --create: To empty and create a new Test sdb domain and load it with 1000
# items, one at a time.
# --batchcreate: To empty and create a new Test sdb domain and load it with
# 1000 items, using batch insertions.
#
# Make sure your Amazon access key and secret key are included in the configuration
# file you'll use to run the script.
#
# If no flag is provided, the items will be only read from the domain without
# creating and populating it beforehand.
#
# If everything is fine, it should connect to Amazon SimpleDB, optionally load
# 1000 items, read them back and produce some basic performance metrics.
#
# == Author
# Riccardo Govoni [battlehorse@gmail.com]
#
# == Copyright
# Copyright(c) 2009 - bayes-swarm project.
# Licensed under the GNU General Public License v2.

require 'right_aws'
require 'date'
require 'util/log'
require 'util/sdb_batchput'

include Pulsar::Log

# connect to SDB
log "Connecting to SDB..."
sdb_opts = Pulsar::Runner.opts['sdb']
sdb = RightAws::SdbInterface.new(sdb_opts[:access_key], sdb_opts[:secret_key])
log "Connected."

if flag?("--create")
# create domain
log "Refreshing domain..."
sdb.delete_domain "Test"
sdb.create_domain "Test"
log "Domain refreshed."

log "Creating 1000 elements..."
start = Time.now.to_i
10.times do |j|
100.times do |i|
attributes = {
'even' => ((j*100+i) % 2 == 0).to_s,
'when' => (Time.now - rand(365)*24*60*60).strftime('%Y-%m-%d')
}
sdb.put_attributes("Test", "test#{j*100+i}", attributes)
end
log "Created #{(j+1)*100} items."
end
stop = Time.now.to_i
log "Done. Total save time: #{stop-start}. Per item: #{(stop-start)/1000.0}"
end

if flag?("--batchcreate")
# create domain
log "Refreshing domain..."
sdb.delete_domain "Test"
sdb.create_domain "Test"
log "Domain refreshed."

log "Creating 1000 elements..."
start = Time.now.to_f
40.times do |j|
items = {}
25.times do |i|
attributes = {
'even' => ((j*25+i) % 2 == 0).to_s,
'when' => (Time.now - rand(365)*24*60*60).strftime('%Y-%m-%d')
}
items["test#{j*25+i}"] = attributes
end
sdb.batch_put_attributes("Test", items)
log "Created #{(j+1)*25} items."
end
stop = Time.now.to_f
log "Done. Total save time: #{stop-start}. Per item: #{(stop-start)/1000.0}"
end

log "Retrieving all items, 250 at a time..."
start = Time.now.to_f
fetched_items = []
next_token = nil
begin
res = sdb.select('select even, when from Test limit 250', next_token)
fetched_items.concat(res[:items])
next_token = res[:next_token]
end while next_token
odd_items = fetched_items.count { |t| t.values.first["even"].to_s == 'false'}
stop = Time.now.to_f
log "Done. Total odd items: #{odd_items}"
log "Total fetch time: #{stop-start}. " +
"Per item (#{fetched_items.size} items): " +
"#{(stop-start)/fetched_items.size.to_f}"

log "Fetching first 250 items selecting by date"
start = Time.now.to_f
res = sdb.select('select even, when from Test where when > "2008-06-01" limit 250')
fetched_items = res[:items]
fetched_items.each do |t|
parsed_date = Date.strptime(t.values.first["when"][0], '%Y-%m-%d')
check_date = Date.civil(2008, 6, 1)
puts "Retrieved date #{t} which shouldn't be here!" if parsed_date < check_date
end
stop = Time.now.to_f
log "Total query time: #{stop-start}. " +
"Per item (#{fetched_items.size} items): " +
"#{(stop-start)/fetched_items.size.to_f}"

Change log

r482 by battlehorse on Jun 5, 2009   Diff
Moving access_key and secret_key from
command line to config options
Go to: 
Project members, sign in to write a code review

Older revisions

r480 by battlehorse on May 30, 2009   Diff
Introducing initial support and tests
for Amazon SimpleDB (as a possible
alternative to MySQL). Extension to
right_aws gem to support
BatchPutAttributes for better
...
All revisions of this file

File info

Size: 3859 bytes, 122 lines
Powered by Google Project Hosting