import options from copy import deepcopy from validator_common import * from groupfilter import Rule as RuleImpl from groupfilter import GroupFilter as GroupFilterImpl from groupfilter import AcceptGroupFilter as AcceptGroupFilterImpl from operators import NOT from timeindex import TimeIndex import time class GroupFilterValidator(object): # The initiation of the GroupFilterValidator happens only ones. def __init__(self, parser, grouper_validator): self.parser = parser self.grouper_validator = grouper_validator self.filters = deepcopy(parser.group_filters) self.branches_fields = self.get_branches_fields() self.br_name_to_grouper = grouper_validator.br_name_to_grouper self.br_name_to_gr_filter = {} self.impl = self.create_impl() def check_duplicate_filter_names(self): duplicates = {} for filter in self.filters: old_val = duplicates.setdefault(filter.name, 0) duplicates[filter.name] = old_val + 1 duplicate_names = [k for k,v in duplicates.iteritems() if v > 1] if len(duplicate_names) > 0: msg = "Group filter(s) %s"%duplicate_names msg += " is/are all defined more than once." raise SyntaxError(msg) def check_field_refs(self): "Check record field references, for unknown fields" for filter in self.filters: for rule in iterate_rules(filter): for branch in filter.branches: for arg in rule.args: if type(arg) is Field: if arg.name in self.branches_fields[branch]: continue else: msg = 'There is no such field %s, '%arg.name msg += 'referenced at line %s'%rule.line raise SyntaxError(msg) def get_branches_fields(self): branches_fields = {} for grouper in self.grouper_validator.impl: branches_fields[grouper.branch_name] = grouper.group_record_fields return branches_fields def validate(self): self.check_for_unused_filters() self.check_field_refs() self.check_duplicate_filter_names() def check_for_unused_filters(self): for filter in self.filters: if len(filter.branches) == 0: msg = "Warning groupfilter %s "%filter.name msg += "defined on line %s"%filter.line msg += " is not used in any branch." print msg continue # skips unused filters def get_rule_impl(self, rule): op = find_op(rule) args = [self.get_rule_impl(arg) if type(arg) == Rule else arg for arg in rule.args] impl = RuleImpl(None, NOT(op) if rule.NOT else op, args) return impl def get_rules_impl(self, filter): replace_bound_rules(filter) replace_with_vals(filter) rules_impl = [] for or_rule in filter.rules: or_rule_list = [] for rule in or_rule: impl = self.get_rule_impl(rule) or_rule_list.append(impl) rules_impl.append(or_rule_list) return rules_impl def create_impl(self): #start = time.clock() #print "GF validation started at:", start self.validate() group_filters_impl = [] for filter in self.filters: rules_impl = self.get_rules_impl(filter) for br_name in filter.branches: records = self.br_name_to_grouper[br_name] index = TimeIndex(5000) grouper = records # TODO: dont use pytables here #field_types = dict(zip(grouper.group_record_fields, # grouper.group_record_types)) # print records #fname = options.temp_path + options.groups_file_prefix #fname += br_name+".h5" #if options.delete_temp_files: if_exists_delete(fname) #file = pytables.create_table_file(fname, field_types) #groups_table = pytables.FlowRecordsTable(fname) # Create separate table files for each of the branches filt_impl = GroupFilterImpl(rules_impl, records, br_name, groups_table, index) group_filters_impl.append(filt_impl) self.br_name_to_gr_filter = dict((filt.branch_name, filt) for filt in group_filters_impl) # Check for branches that don't have group filters and and put accept # filters on them for br_name in self.br_name_to_grouper.keys(): if br_name not in self.br_name_to_gr_filter.keys(): # print "We get here if the group-filter is removed" records = self.br_name_to_grouper[br_name] index = TimeIndex(5000) grouper = records # TODO: dont use pytables here #field_types = dict(zip(grouper.group_record_fields, # grouper.group_record_types)) #fname = options.temp_path + options.groups_file_prefix #fname += br_name+".h5" #if options.delete_temp_files: if_exists_delete(fname) #file = pytables.create_table_file(fname, field_types) #groups_table = pytables.FlowRecordsTable(fname) filt_impl = AcceptGroupFilterImpl(records, br_name, groups_table, index) # This class is called in case some branch is missing # the definition of a group-filter. Essentially a plain # GroupFilter, but with no rules as an argument. self.br_name_to_gr_filter[br_name] = filt_impl group_filters_impl.append(filt_impl) #time_elapsed = (time.clock() - start) #print "GF Validation required:", time_elapsed return group_filters_impl