My favorites | Sign in
Project Home Downloads Wiki Issues Source
Repository:
Checkout   Browse   Changes   Clones  
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;

namespace Google.ProtocolBuffers
{
/// <summary>
/// This class tries hard to allow us to generate strings directly from buffer outputs without having to
///
/// Note, non thread safe
/// </summary>
public class ByteStringStringInterning
{
private class ByteStringOrByteBuffer : IEquatable<ByteStringOrByteBuffer>
{
private readonly ByteString str;
private readonly ByteBuffer buffer;

public ByteStringOrByteBuffer(ByteString str)
{
this.str = str;
}

public ByteStringOrByteBuffer(ByteBuffer buffer)
{
this.buffer = buffer;
}

public bool Equals(ByteStringOrByteBuffer other)
{
if (ReferenceEquals(null, other)) return false;
if (ReferenceEquals(this, other)) return true;
if(other.str!=null && str != null)
return Equals(other.str, str);
if (other.buffer != null && buffer != null)
return Equals(other.buffer, buffer);
if (other.str != null && str == null)
return StringEqualsToBuffer(other.str, buffer);
return StringEqualsToBuffer(str, other.buffer);
}

private static bool StringEqualsToBuffer(ByteString byteString, ByteBuffer byteBuffer)
{
var strLen = byteString.Length;
if(strLen != byteBuffer.Length)
return false;
for (int i = 0; i < strLen; i++)
{
if(byteString.bytes[i] != byteBuffer.Buffer[byteBuffer.Offset+i])
return false;
}
return true;
}

public override bool Equals(object obj)
{
if (ReferenceEquals(null, obj)) return false;
if (ReferenceEquals(this, obj)) return true;
return Equals(obj as ByteStringOrByteBuffer);
}

public override int GetHashCode()
{
return str != null ? str.GetHashCode() : buffer.GetHashCode();
}
}

private readonly int limit;
private int timestamp;
private readonly IDictionary<ByteStringOrByteBuffer, Data> strings = new Dictionary<ByteStringOrByteBuffer, Data>();

public static ByteStringStringInterning CreateInstance()
{
return new ByteStringStringInterning(65536);
}

[Serializable]
private class Data
{
public string Value;
public int Timestamp;
}

private ByteStringStringInterning(int limit)
{
this.limit = limit;
}

public void Clear()
{
strings.Clear();
}

public string Intern(ByteBuffer str)
{
Data val;

int currentTimestamp = Interlocked.Increment(ref timestamp);
if (strings.TryGetValue(new ByteStringOrByteBuffer(str), out val))
{
Interlocked.Exchange(ref val.Timestamp, currentTimestamp);
return val.Value;
}

var byteString = str.ToByteString();
val = new Data { Timestamp = currentTimestamp, Value = byteString.ToStringUtf8() };

strings.Add(new ByteStringOrByteBuffer(byteString), val);

DoCleanupIfNeeded();
return val.Value;
}

private void DoCleanupIfNeeded()
{
if (strings.Count <= limit)
return;

// to avoid frequent thrashing, we will remove the bottom 10% of the current pool in one go
// that means that we will hit the limit fairly infrequently
var list = new List<KeyValuePair<ByteStringOrByteBuffer, Data>>(strings);
list.Sort((x, y) => x.Value.Timestamp - y.Value.Timestamp);

for (int i = 0; i < limit/10; i++)
{
strings.Remove(list[i].Key);
}
}
}
}

Change log

93cf393ba04d by Ayende Rahien <Aye...@ayende.com> on Feb 18, 2010   Diff
Implementing string interning
Go to: 
Project members, sign in to write a code review

Older revisions

All revisions of this file

File info

Size: 4393 bytes, 131 lines
Powered by Google Project Hosting