You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
142 lines
5.8 KiB
Python
142 lines
5.8 KiB
Python
import options
|
|
from copy import deepcopy
|
|
from validator_common import *
|
|
from groupfilter import Rule as RuleImpl
|
|
from groupfilter import GroupFilter as GroupFilterImpl
|
|
from groupfilter import AcceptGroupFilter as AcceptGroupFilterImpl
|
|
from operators import NOT
|
|
import pytables
|
|
from timeindex import TimeIndex
|
|
import time
|
|
|
|
|
|
|
|
class GroupFilterValidator(object):
|
|
# The initiation of the GroupFilterValidator happens only ones.
|
|
def __init__(self, parser, grouper_validator):
|
|
self.parser = parser
|
|
self.grouper_validator = grouper_validator
|
|
self.filters = deepcopy(parser.group_filters)
|
|
self.branches_fields = self.get_branches_fields()
|
|
self.br_name_to_grouper = grouper_validator.br_name_to_grouper
|
|
self.br_name_to_gr_filter = {}
|
|
self.impl = self.create_impl()
|
|
|
|
def check_duplicate_filter_names(self):
|
|
duplicates = {}
|
|
for filter in self.filters:
|
|
old_val = duplicates.setdefault(filter.name, 0)
|
|
duplicates[filter.name] = old_val + 1
|
|
|
|
duplicate_names = [k for k,v in duplicates.iteritems() if v > 1]
|
|
if len(duplicate_names) > 0:
|
|
msg = "Group filter(s) %s"%duplicate_names
|
|
msg += " is/are all defined more than once."
|
|
raise SyntaxError(msg)
|
|
|
|
def check_field_refs(self):
|
|
"Check record field references, for unknown fields"
|
|
for filter in self.filters:
|
|
for rule in iterate_rules(filter):
|
|
for branch in filter.branches:
|
|
check_rule_fields(rule, self.branches_fields[branch])
|
|
|
|
|
|
def get_branches_fields(self):
|
|
branches_fields = {}
|
|
for grouper in self.grouper_validator.impl:
|
|
branches_fields[grouper.branch_name] = grouper.group_record_fields
|
|
return branches_fields
|
|
|
|
def validate(self):
|
|
self.check_for_unused_filters()
|
|
self.check_field_refs()
|
|
self.check_duplicate_filter_names()
|
|
|
|
def check_for_unused_filters(self):
|
|
for filter in self.filters:
|
|
if len(filter.branches) == 0:
|
|
msg = "Warning groupfilter %s "%filter.name
|
|
msg += "defined on line %s"%filter.line
|
|
msg += " is not used in any branch."
|
|
print msg
|
|
continue # skips unused filters
|
|
|
|
def get_rule_impl(self, rule):
|
|
op = find_op(rule)
|
|
args = [self.get_rule_impl(arg) if type(arg) == Rule else arg
|
|
for arg in rule.args]
|
|
impl = RuleImpl(None, NOT(op) if rule.NOT else op, args)
|
|
|
|
return impl
|
|
|
|
def get_rules_impl(self, filter):
|
|
replace_bound_rules(filter)
|
|
replace_with_vals(filter)
|
|
rules_impl = []
|
|
for or_rule in filter.rules:
|
|
or_rule_list = []
|
|
for rule in or_rule:
|
|
impl = self.get_rule_impl(rule)
|
|
or_rule_list.append(impl)
|
|
rules_impl.append(or_rule_list)
|
|
|
|
return rules_impl
|
|
|
|
def create_impl(self):
|
|
|
|
#start = time.clock()
|
|
#print "GF validation started at:", start
|
|
self.validate()
|
|
group_filters_impl = []
|
|
|
|
for filter in self.filters:
|
|
rules_impl = self.get_rules_impl(filter)
|
|
for br_name in filter.branches:
|
|
records = self.br_name_to_grouper[br_name]
|
|
index = TimeIndex(5000)
|
|
grouper = records
|
|
field_types = dict(zip(grouper.group_record_fields,
|
|
grouper.group_record_types))
|
|
# print records
|
|
fname = options.temp_path + options.groups_file_prefix
|
|
fname += br_name+".h5"
|
|
if options.delete_temp_files: if_exists_delete(fname)
|
|
file = pytables.create_table_file(fname, field_types)
|
|
groups_table = pytables.FlowRecordsTable(fname) # Create separate table files for each of the branches
|
|
filt_impl = GroupFilterImpl(rules_impl, records, br_name,
|
|
groups_table, index)
|
|
group_filters_impl.append(filt_impl)
|
|
|
|
self.br_name_to_gr_filter = dict((filt.branch_name, filt)
|
|
for filt in group_filters_impl)
|
|
|
|
|
|
# Check for branches that don't have group filters and and put accept
|
|
# filters on them
|
|
for br_name in self.br_name_to_grouper.keys():
|
|
if br_name not in self.br_name_to_gr_filter.keys():
|
|
# print "We get here if the group-filter is removed"
|
|
records = self.br_name_to_grouper[br_name]
|
|
index = TimeIndex(5000)
|
|
grouper = records
|
|
field_types = dict(zip(grouper.group_record_fields,
|
|
grouper.group_record_types))
|
|
fname = options.temp_path + options.groups_file_prefix
|
|
fname += br_name+".h5"
|
|
if options.delete_temp_files: if_exists_delete(fname)
|
|
file = pytables.create_table_file(fname, field_types)
|
|
groups_table = pytables.FlowRecordsTable(fname)
|
|
filt_impl = AcceptGroupFilterImpl(records, br_name,
|
|
groups_table, index) # This class is called in case some branch is missing
|
|
# the definition of a group-filter. Essentially a plain
|
|
# GroupFilter, but with no rules as an argument.
|
|
self.br_name_to_gr_filter[br_name] = filt_impl
|
|
group_filters_impl.append(filt_impl)
|
|
|
|
#time_elapsed = (time.clock() - start)
|
|
#print "GF Validation required:", time_elapsed
|
|
return group_filters_impl
|
|
|
|
|