removed unnecessary stuff

gnuplot-http.dat

@@ -1,7 +0,0 @@
-# Records #	 	Splitter(s)		Grouper(s)		Merger(s)		Branch A	Branch B	Records Match
-3811				0.6				0.74			2.81			146			143			68
-26521				24.8			34.95			144.75			1800		1816		1683
-56992				53.06			57.68			443.36			1985		2004		2438
-99925				100.03			136.09			960.34			3644		3684		4038
-298063				475.83			1415.34			11485			16412		16666		15131
-916633				1706.32											50141		50338

gnuplot-https.dat

@@ -1,6 +0,0 @@
-# Records  	Splitter(s)		Grouper(s)		Merger(s)		Branch A	Branch B	Records Match
-26521	6.1	6.17	6.23	243	243	486
-56992	13.2	13.2	13.23	158	61	219	
-99925			
-298063		
-916633	

gnuplot.dat

@@ -1,5 +0,0 @@
-# Records #	 	Splitter	Grouper		Merger
-3811				
-26521
-56992
-298063

profiling-heavy-functions.txt

@@ -1,885 +0,0 @@
-/var/netflow/ft-data-fall09/sne-ft-data/2009/sneze/2009-11-29/
- 26521 records
- 1683 records matched the http request
-
-
-
-deepcopy
-
-Splitter initiated
-Parsing and validation finished: 0.31
-Started filtering
-Finished filtering
-Filters ready
-Splitter time elapsed: 27.19
-Finished grouping branch A
-Finished grouping branch B
-Finished filtering groups for branch A
-Finished filtering groups for branch B
-Group filter time elapsed: 45.0
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: 162.3
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 168.99
-Closing remaining open files: ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done
-
-real	2m49.129s
-user	2m44.070s
-sys	0m5.824s
-
-Splitter initiated
-Parsing and validation finished: 0.33
-Started filtering
-Finished filtering
-Filters ready
-Splitter time elapsed: 30.16
-Finished grouping branch B
-Finished grouping branch A
-Finished filtering groups for branch B
-Finished filtering groups for branch A
-Group filter time elapsed: 34.2
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: 138.3
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 143.71
-Closing remaining open files: ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done
-
-real	2m24.193s
-user	2m19.957s
-sys	0m4.608s
-
-
-
-deep_copy
-
-Splitter initiated
-Parsing and validation finished: 0.36
-Started filtering
-Finished filtering
-Filters ready
-Splitter time elapsed: 24.02
-Finished grouping branch B
-Finished grouping branch A
-Finished filtering groups for branch B
-Finished filtering groups for branch A
-Group filter time elapsed: 32.74
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: 155.7
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 162.56
-Closing remaining open files: ./flowy-run/GroupsA.h5... done ./flowy-run/GroupsB.h5... done
-
-real	2m43.294s
-user	2m38.782s
-sys	0m4.628s
-
-Splitter initiated
-Parsing and validation finished: 0.26
-Started filtering
-Finished filtering
-Filters ready
-Splitter time elapsed: 24.8
-Finished grouping branch B
-Finished grouping branch A
-Finished filtering groups for branch A
-Finished filtering groups for branch B
-Group filter time elapsed: 34.95
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: 144.75
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 149.73
-Closing remaining open files: ./flowy-run/GroupsA.h5... done ./flowy-run/GroupsB.h5... done
-
-real	2m36.640s
-user	2m27.385s
-sys	0m3.508s
-
-Splitter initiated
-Parsing and validation finished: 0.3
-Started filtering
-Finished filtering
-Filters ready
-Splitter time elapsed: 24.2
-Finished grouping branch B
- Finished grouping branch A
-Finished filtering groups for branch A
-Finished filtering groups for branch B
-Group filter time elapsed: 31.15
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: 145.9
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 151.19
-
-real	2m31.325s
-user	2m26.629s
-sys	0m5.412s
-
-modified reset/deepcopy
-(('reset', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 64), (26521, 11.015153884887695, 11.560714000001838))
-
-
-
- 
-(('new_group', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 21), (1466, 6.5672850608825684, 5.3123339999998507))
-(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 25), (1468, 775.12532043457031, 766.78390699999591))
-(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 15), (3228, 155.0828640460968, 160.51002500000152))
-(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/splitter.py', 37), (3229, 87.616034030914307, 89.193573000000356))
-(('append', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 118), (3490, 35.743690967559814, 30.529941999999664))
-(('notify', '/usr/lib/python2.6/threading.py', 270), (6570, 10.859287977218628, 10.72066600000062))
-(('_is_owned', '/usr/lib/python2.6/threading.py', 219), (6695, 9.4564809799194336, 9.1245670000004111))
-(('final_result', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 57), (26521, 5.4859673976898193, 5.0482840000003648))
-
-(('reset', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 45), (26521, 85.135001659393311, 88.205508000023968))
-
-(('_deepcopy_dict', '/usr/lib/python2.6/copy.py', 251), (26712, 73.298033714294434, 75.524687000011454))
-(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/record.py', 129), (27270, 27.118208885192871, 27.781735000003209))
-(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/merger.py', 72), (97059, 33.632721662521362, 30.013754000007793))
-(('read_row', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 99), (99360, 518.74268817901611, 468.40537100055235))
-(('iterrows', '/usr/local/lib/python2.6/dist-packages/tables/table.py', 1441), (99377, 118.15105223655701, 106.11463399998161))
-(('read_rows_list', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 105), (99797, 522.83437442779541, 472.12965100054475))
-(('read_rows_list', '/home/melnikovkolya/classes/semester-3-project/flowy/record.py', 133), (99797, 550.52120852470398, 497.50723100058826))
-(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/merger.py', 23), (147484, 24.74915337562561, 21.317261000004237))
-(('deepcopy', '/usr/lib/python2.6/copy.py', 144), (187567, 161.90160441398621, 165.33823200019515))
-(('read', '/usr/local/lib/python2.6/dist-packages/tables/vlarray.py', 700), (195364, 274.85678458213806, 246.25141199899576))
-(('<genexpr>', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 102), (294714, 294.22120332717896, 264.55258099813909))
-(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 178), (856942, 596.70967555046082, 576.32406800022113))
-(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 101), (861729, 430.92800951004028, 418.1861820004529))
-(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 126), (989422, 290.51547265052795, 272.90903400041935))
-(('idx2long', '/usr/local/lib/python2.6/dist-packages/tables/utils.py', 66), (784846, 75.146798133850098, 69.772329999996373))
-(('is_idx', '/usr/local/lib/python2.6/dist-packages/tables/utils.py', 44), (784846, 26.284930467605591, 21.873351000002572))
-
-
-
-
-
-/var/netflow/ft-data-fall09/kur-ft-data/2009-11-17/
- 56992 records
- 2438 records matched the http request
- 
-With profiler off:
-real	8m8.700s
-user	7m47.945s
-sys	0m12.909s
-
-
-Splitter initiated
-Parsing and validation finished: 1.29
-Started filtering
-Finished filtering
-Filters ready
-Splitter time elapsed: 58.21
-Finished grouping branch B
-Finished grouping branch A
-Finished filtering groups for branch A
-Finished filtering groups for branch B
-Group filter time elapsed: 59.8
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: 471.27
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 480.68
-
-56992
-root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# time python flowy.py http-download.flw
-0.72
-Splitter initiated
-Started filtering
- Group filter time started: 0.29
-3955
-Filters ready
-Splitter finished
-Splitter time elapsed: 53.06
-Number of records in branch A 1985
-Number of records in branch B 2004
-Finished grouping branch A
-Finished group-filtering for branch A
-Finished grouping branch B
-Finished group-filtering for branch B
-Group filter threads joined: 57.68
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: 443.36
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 452.1
-Closing remaining open files: ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done
-
-real	7m46.456s
-user	7m21.036s
-sys	0m11.921s
-
- 
-(('new_group', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 21), (1824, 9.5541517734527588, 9.8766150000006974))
-(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 25), (1828, 1249.1410629749298, 1300.497268999989))
-(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/splitter.py', 37), (3955, 59.615991353988647, 62.479928999999061))
-(('split', '/home/melnikovkolya/classes/semester-3-project/flowy/splitter.py', 17), (3955, 30.423548460006714, 32.126016000000902))
-(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 15), (3956, 456.31715869903564, 475.12168400000229))
-(('get', '/usr/lib/python2.6/Queue.py', 150), (3957, 35.274902582168579, 37.742364999999495))
-(('append', '/usr/local/lib/python2.6/dist-packages/tables/vlarray.py', 452), (5486, 76.012235879898071, 76.39678599999479))
-(('append', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 118), (5785, 81.44921350479126, 81.341101000000435))
-(('notify', '/usr/lib/python2.6/threading.py', 270), (8002, 17.408251523971558, 17.825101000000359))
-(('_is_owned', '/usr/lib/python2.6/threading.py', 219), (8101, 14.244855642318726, 15.092936000000464))
-(('final_result', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 57), (56992, 15.892577886581421, 15.040958000006583))
-
-(('reset', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 45), (56992, 255.76119065284729, 262.48040000008808))
-
-(('_deepcopy_dict', '/usr/lib/python2.6/copy.py', 251), (57183, 218.50618243217468, 224.26205200008098))
-(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 89), (58365, 30.709211587905884, 31.189945000012358))
-(('iterate_fixed_fields', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 93), (58365, 19.963983297348022, 19.749231000007512))
-(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/record.py', 129), (58365, 86.714945554733276, 88.23755700004449))
-(('_deepcopy_list', '/usr/lib/python2.6/copy.py', 224), (114144, 72.901082038879395, 73.184596000045076))
-(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 90), (117636, 47.137009859085083, 43.210651000023745))
-(('_deepcopy_atomic', '/usr/lib/python2.6/copy.py', 197), (171331, 14.566928386688232, 13.152824000005694))
-(('_keep_alive', '/usr/lib/python2.6/copy.py', 261), (343098, 47.557926893234253, 39.274455000023863))
-(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/merger.py', 72), (343759, 89.168351411819458, 86.809352999718158))
-(('read_row', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 99), (347405, 1355.7759656906128, 1345.6080879980259))
-(('iterrows', '/usr/local/lib/python2.6/dist-packages/tables/table.py', 1441), (347422, 306.37827634811401, 304.82301899932509))
-(('read_rows_list', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 105), (348171, 1369.7901601791382, 1360.4090329980108))
-
-(('deepcopy', '/usr/lib/python2.6/copy.py', 144), (400864, 485.14781737327576, 489.78665900019996))
-
-(('__init__', '/home/melnikovkolya/classes/semester-3-project/flowy/record.py', 82), (408197, 79.613070487976074, 80.693067999662162))
-(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/merger.py', 23), (527995, 64.410658597946167, 62.123842999773387))
-(('read', '/usr/local/lib/python2.6/dist-packages/tables/vlarray.py', 700), (689950, 714.14480590820312, 706.58424299669286))
-(('<genexpr>', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 102), (1037339, 765.8496515750885, 758.55947299578656))
-(('_processRangeRead', '/usr/local/lib/python2.6/dist-packages/tables/leaf.py', 449), (1037372, 470.43238306045532, 463.84111299771757))
-(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 101), (1380363, 747.47748589515686, 753.67501099601259))
-(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 178), (1380426, 1028.9652721881866, 1053.8537989941979))
-(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 126), (1740570, 498.78313732147217, 495.35881499854258))
-(('EQ', '/home/melnikovkolya/classes/semester-3-project/flowy/operators.py', 63), (2370745, 182.36606240272522, 156.70575899921459))
-(('idx2long', '/usr/local/lib/python2.6/dist-packages/tables/utils.py', 66), (2764694, 214.65504741668701, 203.63286399914659))
-(('is_idx', '/usr/local/lib/python2.6/dist-packages/tables/utils.py', 44), (2764694, 75.347645044326782, 63.899976999761293))
-
-
-(('reset', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 64), (56992, 31.726502895355225, 31.213908000036554)) 
-(('deep_copy', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 41), (56994, 15.406083345413208, 16.889049000018872))
-
-
-7 days of data
-python ft2hdf.py /var/netflow/ft-data-fall09/sne-ft-data/2009/sneze/2009-12-0* netflow-trace.h5
-
-246350 records in total
-12394 records match the query
-
-profiling:
-Splitter initiated
-Parsing and validation finished: 2.22
-Started filtering
-Finished filtering
-Filters ready
-Splitter time elapsed: 1130.1
-Finished grouping branch B
-Finished filtering groups for branch B
-Finished grouping branch A
-Finished filtering groups for branch A
-Group filter time elapsed: 2123.665408
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: -185.553776
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: -114.543776
-
-no profiling:
-Splitter initiated
-Parsing and validation finished: 0.26
-Started filtering
-Finished filtering
-Filters ready
-Splitter time elapsed: 320.43
-Finished grouping branch B
-Finished grouping branch A
-Finished filtering groups for branch B
-Finished filtering groups for branch A
-Group filter time elapsed: 922.42
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: 1039.122704
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 1074.252704
-Closing remaining open files: ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done
-
-
-start()real	90m16.511s
-user	86m23.020s
-sys	3m7.356s
-
-Splitter initiated
-Parsing and validation finished: 0.31
-Started filtering
-Finished filtering
-Filters ready
-Splitter time elapsed: 346.66
-Finished grouping branch B
-Finished grouping branch A
-Finished filtering groups for branch B
-Finished filtering groups for branch A
-Group filter time elapsed: 916.19
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: 1037.532704
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 1073.552704
-
-(('reset', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 64), (246349, 940.52704691886902, 994.15005099796895))
-
-
-(('reset', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 64), (246349, 111.18868279457092, 105.20649999988791))
-(('deep_copy', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 41), (246351, 61.105264902114868, 52.447237999959725))
-(('read_rows_list', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 105), (3155228, 13582.554839611053, 13318.368595361764))
-(('read_rows_list', '/home/melnikovkolya/classes/semester-3-project/flowy/record.py', 133), (3155228, 14223.106386899948, 13948.747855334786))
-(('read', '/usr/local/lib/python2.6/dist-packages/tables/vlarray.py', 700), (6280932, 6694.1691343784332, 6541.9808274548959))
-(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 178), (30651429, 17337.516788959503, 17566.637794171394))
-(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 101), (30677828, 12477.594463348389, 12583.8665639143)) 
-(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 126), (35383022, 8230.0888061523438, 8037.6824171527333)) 
-(('EQ', '/home/melnikovkolya/classes/semester-3-project/flowy/operators.py', 63), (40143460, 2728.9575715065002, 2304.1001345953482))
-
-
-root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# time flow-cat /var/netflow/ft-data-fall09/kur-ft-data/2009-11-16/ | flow-print | wc -l
-99925
-
-root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# time python flowy.py http-download.flw0.77
-Splitter initiated
-Started filtering
-Group filter time started: 0.33
-7222
-Filters ready
-Splitter finished
-Splitter time elapsed: 100.03
-Number of records in branch B 3684
-Number of records in branch A 3644
-Finished grouping branch A
-Finished group-filtering for branch A
-Finished grouping branch B
-Finished group-filtering for branch B
-Group filter threads joined: 136.09
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: 960.34
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 974.11
-Closing remaining open files: ./flowy-run/GroupsA.h5... done ./flowy-run/GroupsB.h5... done
-
-real	16m39.728s
-user	15m49.067s
-sys	0m26.002s
-
-
-
-
-
-
-
-
-
-root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# python ft2hdf.py /var/netflow/ft-data-fall09/sne-ft-data/2009/sneze/2009-11-* netflow-trace.h5
-
-298063
-
-root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# time python flowy.py http-download.flw
-0.84
-Splitter initiated
-Started filtering
-29448
-Filters ready
-Splitter finished
-Splitter time elapsed: 475.83
-Number of records in branch B 16666
-Number of records in branch A 16412
-Finished grouping branch B
-Finished group-filtering for branch B
-Finished grouping branch A
-Finished group-filtering for branch A
-Group filter threads joined: 1415.34
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: -1347.101888 = 11485
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: -1301.531888 = 11531
-Closing remaining open files: ./flowy-run/GroupsA.h5... done ./flowy-run/GroupsB.h5... done
-
-real	192m11.507s = 11531
-user	185m39.648s
-sys	7m25.104s
-
-
-
-The following example shows how heavy is the influence of performing the match operations (and the consequences in case of a match being found) by having a different load for 2 different branches.
-We can see that processing of Branch A, which has been matching records against a port number 443, has a running time of around 17 seconds, while the branch, which checked whether a record entry has a TCP protocol has taken around 90 seconds. The reason for a larger running time is that many more entries that match the prot=TCP requirement have been fround, and it takes an additional processing to index and to append the record to a group.
-
-Though each of the executions of the reset function takes on average 9-10 times more time than the match function, the number of the executions of the match function is at least 30 times greater than that of the reset function.
-
-After spot-profiling (running multi-thread profiler on certain sections of the code only), I could verify that the time spent in executing the match calls of the grouper module was causing most significant slow-down in all of the code, up to the merger module. Depending on the complexity of each of the match() calls, the execution time varied for the same number of function calls. The three match() calls from different classes form a nested chain, where one match() function, relies on another match(). The heaviest (in terms of time per execution) of all three match functions is the top-most match(), that comes from the Group class of the grouper module. Besides relying on a double-nested match call from two other classes, it also performs a
-and calculates the first and the last records of newly-formed group, which is necessary for relative comparisons.
-
-with an average time spent per cycle (including the profiler overhead) being:
-(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 193), (280246, 151.20158743858337, 154.237679000132))
-
-
-A very simple GrouperModule class match, with a single 
-
-
-    def match(self, record, group):
-        for rule in self.rules:
-            if not rule.match(record, group):
-                return False
-        return True
-
-This was followed by the filtering operation
-
-
-Splitter initiated
-GF validation started at: 0.89
-GF Validation required: 0.09
-Parsing and validation finished: 0.32
-Started filtering
- Grouping started at:Fitlering time started at: Grouping started at:  1.0
-1.0
-1.0
-Finished filtering
-Filtering required: 16.87
-Filters ready
-Splitter time elapsed: 17.11
-Finished grouping branch A
-Grouping time required branch A 17.34
-Current time is:  18.34
-Finished filtering groups for branch A
-Finished grouping branch B
-Grouping time required branch B 90.08
-Current time is:  91.08
-Finished filtering groups for branch B
-Group filter time elapsed: 90.41
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: 111.58
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 111.75
-Closing remaining open files: ./flowy-run/GroupsA.h5... done ./flowy-run/GroupsB.h5... done
-
-real	1m52.516s
-user	1m50.411s
-sys	0m2.136s
-
-
-The performance of the group-filters differs significantly, depending on how many matching records have been found in the filtering stage.
-I.e., a filter for the port 443 results in relatively few records, while the filter for a port number >30000 results in many record matching it. The matching records need to be processed and stored for further group-filters, where the group filters try to form groups from the matched records. An example of running a query, which identifies flows with a destination port 443 and a source port > 30000, is shown next. It can be seen, that group-filtering of branch B, which is responsible for filtering out the srcport > 30000 request has a much larger running time than that of branch A, which looks only for those few records with a destination port of 443.
-Splitter initiated
-Parsing and validation finished: 0.28
-Started filtering
-Fitlering time started at: 0.98
-Finished filtering
-Filtering required: 33.49
-Filters ready
-Splitter time elapsed: 33.61
-Finished grouping branch A
-Grouping time finished for branch A 40.49
-Finished filtering groups for branch A
-Finished grouping branch B
-Grouping time finished for branch B 228.46
-Finished filtering groups for branch B
-Group filter time elapsed: 227.86
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: 252.77
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 253.31
-Closing remaining open files: ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done
-
-real	4m11.671s
-user	4m9.296s
-sys	0m4.880s
-
-
-
-==================================HTTPS PROFILE===========================================
-
-
-
-
-A query that selects source and destination ports is defined as follows. We used that query to compare simple running times of different tools:
-
-splitter S {}
-
-filter www_tcp {
-	dstport = 443
-}
-
-filter www_port {
-	srcport = 443
-
-}
-
-grouper g_www_tcp {
-   module g1 {
-   }
-   aggregate srcip, dstip, sum(bytes) as bytes, count(rec_id) as n, 
-   bitOR(tcp_flags) as flags, union(srcport) as srcports
-}
-
-grouper g_www_port {
-   module g1 {
-   }
-   aggregate srcip, dstip, sum(bytes) as bytes, count(rec_id) as n, 
-   bitOR(tcp_flags) as flags, union(dstport) as dstports
-}
-
-groupfilter ggf {
-  bitAND(flags, 0x13) = 0x13
-}
-
-merger M {
-   module m1 {
-	branches B, A 
-	A m B delta 1440min
-   }
-   export m1
-}
-
-ungrouper U {}
-
-"./netflow-trace.h5" -> S
-S branch A -> www_tcp -> g_www_tcp -> ggf -> M
-S branch B -> www_port -> g_www_port -> ggf -> M
-M->U->"./ungroped.h5"
-
-The same number of 
-
-/var/netflow/ft-data-fall09/sne-ft-data/2009/sneze/2009-11-29/
-
-26521 records in total
-486 records match
-
-root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# time python flowy.py https-flows.flw
-0.73
-Splitter initiated
-Started filtering
-486
-Filters ready
-Splitter finished
-Number of records in branch A 243 Number of records in branch B 243
-
-Finished grouping branch A
-Finished group-filtering for branch A
-Finished grouping branch B
-Finished group-filtering for branch B
-Group filter threads joined
-Finished merging branches:  ['B', 'A']
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 6.61
-Closing remaining open files: ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done ./netflow-trace.h5... done ./netflow-trace.h5... done ./flowy-run/GroupsB-merged.h5... done ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA-merged.h5... done ./flowy-run/MergedM.h5... done ./flowy-run/MergedM.h5... done ./ungroped.h5... done ./flowy-run/GroupsA.h5... done
-
-real	0m14.245s
-user	0m7.168s
-sys	0m0.280s
-root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# time python flowy.py https-flows.flw
-0.81
-Splitter initiated
-Started filtering
-486
-Filters ready
-Splitter finished
-Number of records in branch B 243Number of records in branch A 243
-
-Finished grouping branch A
-Finished group-filtering for branch A
-Finished grouping branch B
-Finished group-filtering for branch B
-Group filter threads joined
-Finished merging branches:  ['B', 'A']
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 6.31
-Closing remaining open files: ./netflow-trace.h5... done ./flowy-run/GroupsA.h5... done ./flowy-run/MergedM.h5... done ./flowy-run/GroupsB-merged.h5... done ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done ./ungroped.h5... done ./netflow-trace.h5... done ./flowy-run/MergedM.h5... done ./flowy-run/GroupsA-merged.h5... done ./flowy-run/GroupsB.h5... done
-
-real	0m9.051s
-user	0m7.072s
-sys	0m0.160s
-
-root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# time python flowy.py https-flows.flw
-0.83
-Splitter initiated
-Started filtering
-Group filter time started: 0.23
-486
-Filters ready
-Number of records in branch A 243
- Splitter finished
-Splitter time elapsed: 6.1
-Finished grouping branch A
-Finished group-filtering for branch A
-Number of records in branch B 243
-Finished grouping branch B
-Finished group-filtering for branch B
-Group filter threads joined: 6.17
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: 6.23
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 7.36
-Closing remaining open files: ./flowy-run/GroupsB.h5... done ./netflow-trace.h5... done ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done ./netflow-trace.h5... done ./flowy-run/GroupsB-merged.h5... done ./flowy-run/GroupsA.h5... done ./flowy-run/MergedM.h5... done ./flowy-run/GroupsA-merged.h5... done ./ungroped.h5... done ./flowy-run/MergedM.h5... done
-
-real	0m15.893s
-user	0m7.440s
-sys	0m0.868s
-
-
-
-Most frequent:
-
-(('final_result', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 99), (26521, 1.8366894721984863, 1.7001189999999156))
- 
-(('reset', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 83), (26521, 3.138737678527832, 3.0042079999998066))
- 
-(('deep_copy', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 58), (26523, 1.7581963539123535, 1.6681159999999338))
- 
-(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 92), (26526, 3.3419792652130127, 3.0921969999998495))
- 
-(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/record.py', 130), (26526, 9.8621282577514648, 9.6565820000015421))
- 
-(('iterate_fixed_fields', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 96), (26526, 1.9721605777740479, 1.7561189999999769))
- 
-(('__init__', '/home/melnikovkolya/classes/semester-3-project/flowy/record.py', 82), (27015, 4.6438140869140625, 4.6482780000005732))
- 
-(('mask', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 90), (53042, 1.6173598766326904, 1.5800989999999153))
- 
-(('EQ', '/home/melnikovkolya/classes/semester-3-project/flowy/operators.py', 63), (53044, 1.4263303279876709, 1.1120729999999632))
- 
-(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 134), (53046, 5.1699655055999756, 4.6562810000002663))
-
-
-Heaviest:
-
-(('__init__', '/home/melnikovkolya/classes/semester-3-project/flowy/groupfilter_validator.py', 16), (1, 0.18725490570068359, 0.18801199999999962))
- 
-(('get_interval_records', '/home/melnikovkolya/classes/semester-3-project/flowy/timeindex.py', 57), (1, 0.2019498348236084, 0.20001300000000199))
- 
-(('pass_allen_indices_down', '/home/melnikovkolya/classes/semester-3-project/flowy/merger.py', 79), (1, 0.20258498191833496, 0.20001300000000199))
- 
-(('go', '/home/melnikovkolya/classes/semester-3-project/flowy/merger.py', 108), (1, 0.2162168025970459, 0.21201300000000245))
- 
-(('__init__', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper_validator.py', 11), (1, 0.22698211669921875, 0.22401300000000002))
- 
-(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 36), (4, 1.1266498565673828, 1.1920739999999945))
- 
-(('_form_master_re', '/usr/lib/pymodules/python2.6/ply/lex.py', 482), (1, 0.30334997177124023, 0.22801499999999986))
- 
-(('validate_rules', '/usr/lib/pymodules/python2.6/ply/lex.py', 723), (1, 0.33556008338928223, 0.31602000000000008))
- 
-(('validate_all', '/usr/lib/pymodules/python2.6/ply/lex.py', 567), (1, 0.33656787872314453, 0.31602000000000008))
- 
-(('__init__', '/home/melnikovkolya/classes/semester-3-project/flowy/filter_validator.py', 10), (1, 0.37907099723815918, 0.3560230000000002))
- 
-(('go', '/home/melnikovkolya/classes/semester-3-project/flowy/groupfilter.py', 14), (2, 1.1871206760406494, 1.248076999999995))
- 
-(('create_impl', '/home/melnikovkolya/classes/semester-3-project/flowy/ungrouper_validator.py', 76), (1, 0.60985612869262695, 0.60803800000000052))
- 
-(('lex', '/usr/lib/pymodules/python2.6/ply/lex.py', 865), (1, 0.65552186965942383, 0.56003499999999995))
- 
-(('__init__', '/home/melnikovkolya/classes/semester-3-project/flowy/parser.py', 9), (1, 0.6572871208190918, 0.56403499999999995))
- 
-(('__init__', '/home/melnikovkolya/classes/semester-3-project/flowy/ungrouper_validator.py', 10), (1, 0.67348289489746094, 0.67204200000000114))
- 
-(('__init__', '/home/melnikovkolya/classes/semester-3-project/flowy/parser.py', 182), (1, 0.71254801750183105, 0.6200389999999999))
- 
-(('go', '/home/melnikovkolya/classes/semester-3-project/flowy/ungrouper.py', 29), (1, 1.85223388671875, 1.8081130000000023))
-
-
-
-/var/netflow/ft-data-fall09/kur-ft-data/2009-11-17/
- 56992 records
-
-root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# python ft2hdf.py /var/netflow/ft-data-fall09/kur-ft-data/2009-11-17/ netflow-trace.h5
-root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# time python flowy.py https-flows.flw
-0.7
-Splitter initiated
-Started filtering
- Group filter time started: 0.27
-219
-Filters ready
-Splitter finished
-Splitter time elapsed: 13.2
-Number of records in branch A 158
-Finished grouping branch A
-Finished group-filtering for branch A
-Number of records in branch B 61
-Finished grouping branch B
-Finished group-filtering for branch B
-Group filter threads joined: 13.18
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: 13.23
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 13.83
-Closing remaining open files: ./netflow-trace.h5... done ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done ./netflow-trace.h5... done ./flowy-run/GroupsB-merged.h5... done ./flowy-run/GroupsA.h5... done ./flowy-run/GroupsB.h5... done ./flowy-run/MergedM.h5... done ./flowy-run/GroupsA-merged.h5... done ./ungroped.h5... done ./flowy-run/MergedM.h5... done
-
-real	0m15.696s
-user	0m13.653s
-sys	0m1.004s
-root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# python printhdf.py ungroped.h5 | wc -l
-Closing remaining open files: ungroped.h5... done
-219
-
-
-
-root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# python ft2hdf.py /var/netflow/ft-data-fall09/kur-ft-data/2009-11-16/ netflow-trace.h5
-
-99924
-
-root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# time python flowy.py https-flows.flw
-0.71
-Splitter initiated
-Started filtering
- Group filter time started: 0.27
-1434
-Filters ready
-Splitter finished
-Splitter time elapsed: 23.19
-Number of records in branch A 748
-Finished grouping branch A
-Finished group-filtering for branch A
-Number of records in branch B 686
-Finished grouping branch B
-Finished group-filtering for branch B
-Group filter threads joined: 23.23
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: 23.31
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 26.48
-Closing remaining open files: ./netflow-trace.h5... done ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done ./netflow-trace.h5... done ./flowy-run/GroupsB-merged.h5... done ./flowy-run/GroupsA.h5... done ./flowy-run/MergedM.h5... done ./flowy-run/GroupsA-merged.h5... done ./ungroped.h5... done ./flowy-run/MergedM.h5... done
-
-real	0m28.767s
-user	0m24.486s
-sys	0m2.840s
-root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# python printhdf.py ungroped.h5 | wc -l
-Closing remaining open files: ungroped.h5... done
-1434
-
-
-
-root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# python ft2hdf.py /var/netflow/ft-data-fall09/sne-ft-data/2009/sneze/2009-11-* netflow-trace.h5
-
-298063
-
-root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# python printhdf.py ungroped.h5 | wc -l
-Closing remaining open files: ungroped.h5... done
-4087
-
-
-0.76
-Splitter initiated
-Started filtering
-4087
-Filters ready
-Splitter finished
-Group filter time started: 53.73
-Splitter time elapsed: 53.73
-Number of records in branch A 2041
-Finished grouping branch A
-Finished group-filtering for branch A
-Number of records in branch B 2046
-Finished grouping branch B
-Finished group-filtering for branch B
-Group filter threads joined: 54.37
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: 54.47
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 63.47
-Closing remaining open files: ./flowy-run/GroupsB-merged.h5... done ./netflow-trace.h5... done ./ungroped.h5... done ./flowy-run/MergedM.h5... done ./flowy-run/MergedM.h5... done ./netflow-trace.h5... done ./flowy-run/GroupsA.h5... done ./flowy-run/GroupsA.h5... done ./flowy-run/GroupsA-merged.h5... done ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsB.h5... done
-
-real	1m8.146s
-user	1m3.576s
-sys	0m0.776s
-
-
-
-
-
-
-
-
-
-\begin{mytinylisting}
-\begin{verbatim}
-splitter S {}
-
-filter www_req {
-   dstport = 443 OR dstport = 80 OR dstport = 8080
-   unix_secs > 1259413200
-   unix_secs < 1259445600
-}
-
-filter www_res {
-   srcport = 443 OR srcport = 80 OR srcport = 8080
-   unix_secs < 1259445600
-   unix_secs > 1259413200
-}
-
-grouper g_www_req {
-   module g1 {
-   }
-   aggregate bitOR(tcp_flags) as flags
-}
-
-grouper g_www_res {
-   module g1 {
-   }
-   aggregate bitOR(tcp_flags) as flags
-}
-
-groupfilter ggf {
-   bitAND(flags, 0x13) = 0x13
-}
-
-merger M {
-   module m1 {
-      branches B, A 
-      A d B OR B d A
-   }
-   export m1
-}
-
-ungrouper U {}
-
-"./h5ports.h5" -> S
-S branch A -> www_req -> g_www_req -> ggf -> M
-S branch B -> www_res -> g_www_res -> ggf -> M
-M->U->"./portsungroped.h5"
-\end{verbatim}
-\end{mytinylisting}
-%
-Execution of that query
-\begin{verbatim}
-flowy# time python flowy.py ports.flw
-0.83
-Splitter initiated
-Started filtering
-Group filter time started: 0.3
-1463
-Filters ready
-Splitter finished
-Splitter time elapsed: 7.12
-Number of records in branch B 1463
-Finished grouping branch B
-Finished group-filtering for branch B
- Number of records in branch A 1463
-Finished grouping branch A
-Finished group-filtering for branch A
-Group filter threads joined: 7.26
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: 7.26
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 13.92
-
-real	0m14.788s
-user	0m13.969s
-sys	    0m0.900s
-\end{verbatim}
-
-

run-output2.txt

@@ -1,25 +0,0 @@
-0.82
-[Input('./netflow-trace.h5', 50, set([]), set([]), set([])), BranchNode('S', 50, set([]), set([]))]
-
-[BranchNode('S', 51, set([]), set([])), Branch('A', 51, None, set([]), set([])), BranchNode('www_req', 51, set([]), set([])), BranchNode('g_www_req', 51, set([]), set([])), BranchNode('ggf', 51, set([]), set([])), BranchNode('M', 51, set([]), set([]))]
-
-[BranchNode('S', 52, set([]), set([])), Branch('B', 52, None, set([]), set([])), BranchNode('www_res', 52, set([]), set([])), BranchNode('g_www_res', 52, set([]), set([])), BranchNode('ggf', 52, set([]), set([])), BranchNode('M', 52, set([]), set([]))]
-
-[BranchNode('M', 53, set([]), set([])), BranchNode('U', 53, set([]), set([])), Output('./ungroped.h5', 53, set([]), set([]), set([]))]
-
-Splitter initiated
-Parsing and validation finished: 0.31
-Started filtering
-Finished filtering
-Filters ready
-Splitter time elapsed: 346.66
-Finished grouping branch B
-Finished grouping branch A
-Finished filtering groups for branch B
-Finished filtering groups for branch A
-Group filter time elapsed: 916.19
-Finished merging branches:  ['B', 'A']
-Merger time elapsed: 1037.532704
-Ungrouper U finished exectution
-FINISHED!
-Overall time elapsed: 1073.552704