Applying multiple filters to list of tuples

I am looking for an efficient, pythonic way to apply multiple filters to a list of tuples.

As an example, assume filters like this:

def f1(t): return t[3]<10
def f2(t): return t[0]!=1
def f3(t): return t[1] in ("lisa","eric")
def f4(t): return t[3]>2

And n-tuples (i.e. db-records) like this:

tuples=[
(0,'tom','...',8),
(1,'john','...',17),
(2,'lisa','...',1),
(3,'eric','...',18)
]

The following works:

def nFilter(filters,tuples):
    if filters and tuples:
        return nFilter(filters,filter(filters.pop(),tuples))
    else: return tuples

With results like:

>>> nFilter([f1,f2,f3],tuples)
[(2, 'lisa', '...', 1)]

and

>>> nFilter([f1,f2,f3,f4],tuples)
[]

But I'm wondering if there is a more direct way; what I had in mind is something like function composition (i.e f1(f2(...fn(tuples)...))), for an arbitrary list of functions. There are references to a functional library containing a compose function in the docs, but the links are all dead.

Also, since I'm planning on using this on fairly large data sets, and possibly with a large number of filters in a production web service, it must be efficient, and I can't really say if this solution is.

Any suggestions or improvements are welcome.

python filter tuples

Edit Report

1 Answer

Well, no fancy itertools or the like here, just avoiding the overhead of recursion and generators using a simple loop:

def for_loop(filters, tuples):
    for f in filters:
        tuples = filter(f, tuples)
        if not tuples: 
            return tuples
    return tuples

Here's a little dirty benchmark:

import datetime
from itertools import ifilter
from timeit import Timer

def f1(t): return t[3]<10
def f2(t): return t[0]!=1
def f3(t): return t[1] in ("lisa","eric")
def f4(t): return t[3]>2

def original(filters,tuples):
    if filters and tuples:
        return original(filters,filter(filters.pop(),tuples))
    else: 
        return tuples

def filter_lambda_all(filters, tuples):
    return filter(lambda t: all(f(t) for f in filters), tuples)

def loop(filters, tuples):
    while filters and tuples:
        f = filters[0]
        del filters[0]
        tuples = filter(f, tuples)
    return tuples

def pop_loop(filters, tuples):
    while filters and tuples:
        tuples = filter(filters.pop(), tuples)
    return tuples

def for_loop(filters, tuples):
    for f in filters:
        tuples = filter(f, tuples)
        if not tuples: 
            return tuples
    return tuples


def with_ifilter(filters, tuples):
    for f in filters:
        tuples = ifilter(f, tuples)
    return tuples

_filters = [f1, f2, f3, f4]

def time(f):
    def t():
        return [    (0,'tom','...',8),
                    (1,'john','...',17),
                    (2,'lisa','...',1),
                    (3,'eric','...',18)
                ]*1000
    for i in xrange(4):
        list(f(_filters[i:] * 15,t()))

if __name__=='__main__':
    for f in (original,filter_lambda_all,loop,pop_loop,with_ifilter,for_loop):
        t = Timer(lambda: time(f))
        d = t.timeit(number=400)
        print f.__name__, d

Result:

original 7.23815271085
filter_lambda_all 14.1629812265
loop

answered 2012-09-12T11:11:58.233

Applying multiple filters to list of tuples

1 Answer

Your Answer