bcolz/bench/iterblocks.py

import numpy as np
import numexpr as ne
import bcolz
import time

bcolz.defaults.cparams['shuffle'] = bcolz.SHUFFLE
#bcolz.defaults.cparams['shuffle'] = bcolz.BITSHUFFLE
#bcolz.defaults.cparams['cname'] = 'blosclz'
bcolz.defaults.cparams['cname'] = 'lz4'
bcolz.defaults.cparams['clevel'] = 5

N = 1e8
a = np.arange(N)
ca = bcolz.carray(a)



def timefunc(f):
    def f_timer(*args, **kwargs):
        start = time.time()
        result = f(*args, **kwargs)
        end = time.time()
        print f.__name__, 'took', round(end - start, 3), 'sec'
        return result
    return f_timer


@timefunc
def iterblocks0():
    return a.sum()

@timefunc
def iterblocks1(arr):
    return sum(i for i in arr)

@timefunc
def iterblocks2(arr):
    sum = 0.
    for b in bcolz.iterblocks(arr, blen=arr.chunklen):
        sum += b.sum()
    return sum


print repr(ca)

a0 = iterblocks0()
print "a0:", a0
# a1 = iterblocks1(ca)
# assert a0 == a1
a1 = iterblocks2(ca)
assert a0 == a1