Redis-py: pipeline

https://github.com/andymccurdy/redis-py/issues/451

Here's the code I used and the results I got testing against a Redis instance running on localhost.

import time
import redis
from redis.client import StrictPipeline



class StreamingPipeline(StrictPipeline):
    def reset(self):
        super(StreamingPipeline, self).reset()
        self.connection = self.connection_pool.get_connection('MULTI',
                                                              self.shard_hint)
        self.commands = []

    def execute_command(self, *args, **kwargs):
        self.connection.send_command(*args)
        self.commands.append(args[0])

    def execute(self, raise_on_error=True):
        connection = self.connection
        return [self.parse_response(connection, command_name)
                for command_name in self.commands]



def run(pipe, type):
    for num in (10, 50, 100, 1000, 10000):
        start = time.time()
        for _ in xrange(num):
            pipe.get('foo')
        result = pipe.execute()
        run_time = time.time() - start
        print "Processed %d %s GETs in %f seconds" % (num, type, run_time)
        pipe.reset()


client = redis.StrictRedis()

def streaming():
    pipe = StreamingPipeline(client.connection_pool,
                             client.response_callbacks,
                             False,
                             None)
    run(pipe, 'STREAMED')


def buffered():
    pipe = client.pipeline(transaction=False)
    run(pipe, 'BUFFERED')


buffered()
streaming()

Results:

Processed 10 BUFFERED GETs in 0.001150 seconds
Processed 50 BUFFERED GETs in 0.000862 seconds
Processed 100 BUFFERED GETs in 0.001425 seconds
Processed 1000 BUFFERED GETs in 0.010493 seconds
Processed 10000 BUFFERED GETs in 0.086214 seconds
Processed 10 STREAMED GETs in 0.000219 seconds
Processed 50 STREAMED GETs in 0.000801 seconds
Processed 100 STREAMED GETs in 0.001714 seconds
Processed 1000 STREAMED GETs in 0.016336 seconds
Processed 10000 STREAMED GETs in 0.168507 seconds