1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

#!/usr/bin/env python

import pymongo,sys

#
# Really lame script to learn the pymongo APIs
# mdfranz@gmail.com
#

logfile = "/var/log/squid/access.log"
m=('192.168.169.62','mongosquid')


def has_entry(collection,stamp):
    return False

def parse_line(collection,l):
    tags = {"source":2,"squidcode":3,"size":4,"method":5,"url":6,"format":7}

    f = l.split()    
    stamp = float(f[0])
    if not has_entry(coll,stamp):
        d = {}
        d['stamp'] = stamp
        for field  in tags.keys():
            if field in ['size']:
                d[field] = float(f[tags[field]])
            else:
                d[field] = f[tags[field]]
        return d
    else:
        return {}

# from http://stackoverflow.com/questions/136168/get-last-n-lines-of-a-file-with-python-similar-to-tail
def tail( f, window=1024 ):
    f.seek( 0, 2 )
    bytes= f.tell()
    size= window
    block= -1

    while size > 0 and bytes+block*1024  > 0:
        f.seek( block*1024, 2 ) # from the end!
        data= f.read( 1024 )
        linesFound= data.count('\n')
        size -= linesFound
        block -= 1

    f.seek( block*1024, 2 )
    f.readline() # find a newline
    lastBlocks= list( f.readlines() )
    return lastBlocks



###

if __name__ == "__main__":

    conn = pymongo.Connection(m[0])
    db = conn[m[1]]
    coll = db["raw"]

    if sys.argv[1] == "tail":
        lines = tail(open(logfile,"r"))
        for l in lines:
            d = parse_line(coll,l)
            if d != {}:
                coll.save(d)
    elif sys.argv[1] == "all":
        f = open(logfile)
        for l in f:
            coll.save(parse_line(coll,l))
    else:
        f = open(sys.argv[1])
        for l in f:
            coll.save(parse_line(coll,l))