flowy/groupfilter_validator.py
2010-11-05 18:57:01 +01:00

141 lines
5.8 KiB
Python

import options
from copy import deepcopy
from validator_common import *
from groupfilter import Rule as RuleImpl
from groupfilter import GroupFilter as GroupFilterImpl
from groupfilter import AcceptGroupFilter as AcceptGroupFilterImpl
from operators import NOT
import pytables
from timeindex import TimeIndex
import time
class GroupFilterValidator(object):
# The initiation of the GroupFilterValidator happens only ones.
def __init__(self, parser, grouper_validator):
self.parser = parser
self.grouper_validator = grouper_validator
self.filters = deepcopy(parser.group_filters)
self.branches_fields = self.get_branches_fields()
self.br_name_to_grouper = grouper_validator.br_name_to_grouper
self.br_name_to_gr_filter = {}
self.impl = self.create_impl()
def check_duplicate_filter_names(self):
duplicates = {}
for filter in self.filters:
old_val = duplicates.setdefault(filter.name, 0)
duplicates[filter.name] = old_val + 1
duplicate_names = [k for k,v in duplicates.iteritems() if v > 1]
if len(duplicate_names) > 0:
msg = "Group filter(s) %s"%duplicate_names
msg += " is/are all defined more than once."
raise SyntaxError(msg)
def check_field_refs(self):
"Check record field references, for unknown fields"
for filter in self.filters:
for rule in iterate_rules(filter):
for branch in filter.branches:
check_rule_fields(rule, self.branches_fields[branch])
def get_branches_fields(self):
branches_fields = {}
for grouper in self.grouper_validator.impl:
branches_fields[grouper.branch_name] = grouper.group_record_fields
return branches_fields
def validate(self):
self.check_for_unused_filters()
self.check_field_refs()
self.check_duplicate_filter_names()
def check_for_unused_filters(self):
for filter in self.filters:
if len(filter.branches) == 0:
msg = "Warning groupfilter %s "%filter.name
msg += "defined on line %s"%filter.line
msg += " is not used in any branch."
print msg
continue # skips unused filters
def get_rule_impl(self, rule):
op = find_op(rule)
args = [self.get_rule_impl(arg) if type(arg) == Rule else arg
for arg in rule.args]
impl = RuleImpl(None, NOT(op) if rule.NOT else op, args)
return impl
def get_rules_impl(self, filter):
replace_bound_rules(filter)
replace_with_vals(filter)
rules_impl = []
for or_rule in filter.rules:
or_rule_list = []
for rule in or_rule:
impl = self.get_rule_impl(rule)
or_rule_list.append(impl)
rules_impl.append(or_rule_list)
return rules_impl
def create_impl(self):
#start = time.clock()
#print "GF validation started at:", start
self.validate()
group_filters_impl = []
for filter in self.filters:
rules_impl = self.get_rules_impl(filter)
for br_name in filter.branches:
records = self.br_name_to_grouper[br_name]
index = TimeIndex(5000)
grouper = records
field_types = dict(zip(grouper.group_record_fields,
grouper.group_record_types))
# print records
fname = options.temp_path + options.groups_file_prefix
fname += br_name+".h5"
if options.delete_temp_files: if_exists_delete(fname)
file = pytables.create_table_file(fname, field_types)
groups_table = pytables.FlowRecordsTable(fname) # Create separate table files for each of the branches
filt_impl = GroupFilterImpl(rules_impl, records, br_name,
groups_table, index)
group_filters_impl.append(filt_impl)
self.br_name_to_gr_filter = dict((filt.branch_name, filt)
for filt in group_filters_impl)
# Check for branches that don't have group filters and and put accept
# filters on them
for br_name in self.br_name_to_grouper.keys():
if br_name not in self.br_name_to_gr_filter.keys():
# print "We get here if the group-filter is removed"
records = self.br_name_to_grouper[br_name]
index = TimeIndex(5000)
grouper = records
field_types = dict(zip(grouper.group_record_fields,
grouper.group_record_types))
fname = options.temp_path + options.groups_file_prefix
fname += br_name+".h5"
if options.delete_temp_files: if_exists_delete(fname)
file = pytables.create_table_file(fname, field_types)
groups_table = pytables.FlowRecordsTable(fname)
filt_impl = AcceptGroupFilterImpl(records, br_name,
groups_table, index) # This class is called in case some branch is missing
# the definition of a group-filter. Essentially a plain
# GroupFilter, but with no rules as an argument.
self.br_name_to_gr_filter[br_name] = filt_impl
group_filters_impl.append(filt_impl)
#time_elapsed = (time.clock() - start)
#print "GF Validation required:", time_elapsed
return group_filters_impl