flowy/groupfilter_validator.py

import options
from copy import deepcopy
from validator_common import *
from groupfilter import Rule as RuleImpl
from groupfilter import GroupFilter as GroupFilterImpl
from groupfilter import AcceptGroupFilter as AcceptGroupFilterImpl
from operators import NOT
from timeindex import TimeIndex
import time



class GroupFilterValidator(object):
	# The initiation of the GroupFilterValidator happens only ones.
    def __init__(self, parser, grouper_validator):
        self.parser = parser
        self.grouper_validator = grouper_validator
        self.filters = deepcopy(parser.group_filters)
        self.branches_fields = self.get_branches_fields()
        self.br_name_to_grouper = grouper_validator.br_name_to_grouper
        self.br_name_to_gr_filter = {}
        self.impl = self.create_impl()

    def check_duplicate_filter_names(self):
            duplicates = {}
            for filter in self.filters:
                old_val = duplicates.setdefault(filter.name, 0)
                duplicates[filter.name] = old_val + 1
                
            duplicate_names = [k for k,v in duplicates.iteritems() if v > 1]
            if len(duplicate_names) > 0:
                msg = "Group filter(s) %s"%duplicate_names
                msg += " is/are all defined more than once."
                raise SyntaxError(msg)

    def check_field_refs(self):
        "Check record field references, for unknown fields"
        for filter in self.filters:
            for rule in iterate_rules(filter):
                for branch in filter.branches:
                    for arg in rule.args:
                        if type(arg) is Field:
                            if arg.name in self.branches_fields[branch]:
                                continue
                            else:
                                msg = 'There is no such field %s, '%arg.name
                                msg += 'referenced at line %s'%rule.line
                                raise SyntaxError(msg)

    

    def get_branches_fields(self):
        branches_fields = {}
        for grouper in self.grouper_validator.impl:
            branches_fields[grouper.branch_name] = grouper.group_record_fields
        return branches_fields

    def validate(self):
        self.check_for_unused_filters()
        self.check_field_refs()
        self.check_duplicate_filter_names()
    
    def check_for_unused_filters(self):
        for filter in self.filters:
            if len(filter.branches) == 0:
                msg = "Warning groupfilter %s "%filter.name
                msg += "defined on line %s"%filter.line
                msg += " is not used in any branch."
                print msg
                continue # skips unused filters
    
    def get_rule_impl(self, rule):
        op = find_op(rule)
        args = [self.get_rule_impl(arg) if type(arg) == Rule else arg
                for arg in rule.args]
        impl = RuleImpl(None, NOT(op) if rule.NOT else op, args)
        
        return impl
    
    def get_rules_impl(self, filter):
        replace_bound_rules(filter)
        replace_with_vals(filter)
        rules_impl = []
        for or_rule in filter.rules:
            or_rule_list = []
            for rule in or_rule:
                impl = self.get_rule_impl(rule)
                or_rule_list.append(impl)
            rules_impl.append(or_rule_list)
        
        return rules_impl

    def create_impl(self):
    
        #start = time.clock()
        #print "GF validation started at:", start
        self.validate()
        group_filters_impl = []
        
        for filter in self.filters:
            rules_impl = self.get_rules_impl(filter)
            for br_name in filter.branches:
                records = self.br_name_to_grouper[br_name]
                index = TimeIndex(5000)
                grouper = records
                # TODO: dont use pytables here
                #field_types = dict(zip(grouper.group_record_fields,
                #                       grouper.group_record_types))
#                print records
                #fname = options.temp_path + options.groups_file_prefix
                #fname += br_name+".h5"
                #if options.delete_temp_files: if_exists_delete(fname)
                #file = pytables.create_table_file(fname, field_types)
                #groups_table = pytables.FlowRecordsTable(fname) # Create separate table files for each of the branches
                filt_impl = GroupFilterImpl(rules_impl, records, br_name,
                                            groups_table, index)
                group_filters_impl.append(filt_impl)
        
        self.br_name_to_gr_filter = dict((filt.branch_name, filt) 
                                           for filt in group_filters_impl)
        
        
        # Check for branches that don't have group filters and and put accept
        # filters on them
        for br_name in self.br_name_to_grouper.keys():
            if br_name not in self.br_name_to_gr_filter.keys():
#                print "We get here if the group-filter is removed"
                records = self.br_name_to_grouper[br_name]
                index = TimeIndex(5000)
                grouper = records
                # TODO: dont use pytables here
                #field_types = dict(zip(grouper.group_record_fields,
                #                       grouper.group_record_types))
                #fname = options.temp_path + options.groups_file_prefix
                #fname += br_name+".h5"
                #if options.delete_temp_files: if_exists_delete(fname)
                #file = pytables.create_table_file(fname, field_types)
                #groups_table = pytables.FlowRecordsTable(fname)
                filt_impl = AcceptGroupFilterImpl(records, br_name,
                                            groups_table, index) 	# This class is called in case some branch is missing 
                                            						# the definition of a group-filter. Essentially a plain
                                            						# GroupFilter, but with no rules as an argument.
                self.br_name_to_gr_filter[br_name] = filt_impl
                group_filters_impl.append(filt_impl)
        
        #time_elapsed = (time.clock() - start)
        #print "GF Validation required:", time_elapsed
        return group_filters_impl