removed unnecessary stuff
This commit is contained in:
josch
parent
d6fe38272e
commit
fd3e6ae694
47 changed files with 0 additions and 6613 deletions
Binary file not shown.
BIN
allen_index.pyc
BIN
allen_index.pyc
Binary file not shown.
BIN
allen_ops.pyc
BIN
allen_ops.pyc
Binary file not shown.
BIN
custops.pyc
BIN
custops.pyc
Binary file not shown.
BIN
filter.pyc
BIN
filter.pyc
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
flowy_exec.pyc
BIN
flowy_exec.pyc
Binary file not shown.
BIN
ft2hdf.pyc
BIN
ft2hdf.pyc
Binary file not shown.
BIN
ftreader.pyc
BIN
ftreader.pyc
Binary file not shown.
|
|
@ -1,7 +0,0 @@
|
|||
# Records # Splitter(s) Grouper(s) Merger(s) Branch A Branch B Records Match
|
||||
3811 0.6 0.74 2.81 146 143 68
|
||||
26521 24.8 34.95 144.75 1800 1816 1683
|
||||
56992 53.06 57.68 443.36 1985 2004 2438
|
||||
99925 100.03 136.09 960.34 3644 3684 4038
|
||||
298063 475.83 1415.34 11485 16412 16666 15131
|
||||
916633 1706.32 50141 50338
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
# Records Splitter(s) Grouper(s) Merger(s) Branch A Branch B Records Match
|
||||
26521 6.1 6.17 6.23 243 243 486
|
||||
56992 13.2 13.2 13.23 158 61 219
|
||||
99925
|
||||
298063
|
||||
916633
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
# Records # Splitter Grouper Merger
|
||||
3811
|
||||
26521
|
||||
56992
|
||||
298063
|
||||
BIN
grouper.pyc
BIN
grouper.pyc
Binary file not shown.
Binary file not shown.
BIN
groupfilter.pyc
BIN
groupfilter.pyc
Binary file not shown.
Binary file not shown.
BIN
h5ports.h5
BIN
h5ports.h5
Binary file not shown.
BIN
merger.pyc
BIN
merger.pyc
Binary file not shown.
Binary file not shown.
BIN
netflow-trace.h5
BIN
netflow-trace.h5
Binary file not shown.
BIN
operators.pyc
BIN
operators.pyc
Binary file not shown.
BIN
options.pyc
BIN
options.pyc
Binary file not shown.
4298
parser.out
4298
parser.out
File diff suppressed because it is too large
Load diff
BIN
parser.pyc
BIN
parser.pyc
Binary file not shown.
BIN
parsetab.pyc
BIN
parsetab.pyc
Binary file not shown.
BIN
portsungroped.h5
BIN
portsungroped.h5
Binary file not shown.
BIN
profiler.pyc
BIN
profiler.pyc
Binary file not shown.
|
|
@ -1,885 +0,0 @@
|
|||
/var/netflow/ft-data-fall09/sne-ft-data/2009/sneze/2009-11-29/
|
||||
26521 records
|
||||
1683 records matched the http request
|
||||
|
||||
|
||||
|
||||
deepcopy
|
||||
|
||||
Splitter initiated
|
||||
Parsing and validation finished: 0.31
|
||||
Started filtering
|
||||
Finished filtering
|
||||
Filters ready
|
||||
Splitter time elapsed: 27.19
|
||||
Finished grouping branch A
|
||||
Finished grouping branch B
|
||||
Finished filtering groups for branch A
|
||||
Finished filtering groups for branch B
|
||||
Group filter time elapsed: 45.0
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: 162.3
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 168.99
|
||||
Closing remaining open files: ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done
|
||||
|
||||
real 2m49.129s
|
||||
user 2m44.070s
|
||||
sys 0m5.824s
|
||||
|
||||
Splitter initiated
|
||||
Parsing and validation finished: 0.33
|
||||
Started filtering
|
||||
Finished filtering
|
||||
Filters ready
|
||||
Splitter time elapsed: 30.16
|
||||
Finished grouping branch B
|
||||
Finished grouping branch A
|
||||
Finished filtering groups for branch B
|
||||
Finished filtering groups for branch A
|
||||
Group filter time elapsed: 34.2
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: 138.3
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 143.71
|
||||
Closing remaining open files: ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done
|
||||
|
||||
real 2m24.193s
|
||||
user 2m19.957s
|
||||
sys 0m4.608s
|
||||
|
||||
|
||||
|
||||
deep_copy
|
||||
|
||||
Splitter initiated
|
||||
Parsing and validation finished: 0.36
|
||||
Started filtering
|
||||
Finished filtering
|
||||
Filters ready
|
||||
Splitter time elapsed: 24.02
|
||||
Finished grouping branch B
|
||||
Finished grouping branch A
|
||||
Finished filtering groups for branch B
|
||||
Finished filtering groups for branch A
|
||||
Group filter time elapsed: 32.74
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: 155.7
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 162.56
|
||||
Closing remaining open files: ./flowy-run/GroupsA.h5... done ./flowy-run/GroupsB.h5... done
|
||||
|
||||
real 2m43.294s
|
||||
user 2m38.782s
|
||||
sys 0m4.628s
|
||||
|
||||
Splitter initiated
|
||||
Parsing and validation finished: 0.26
|
||||
Started filtering
|
||||
Finished filtering
|
||||
Filters ready
|
||||
Splitter time elapsed: 24.8
|
||||
Finished grouping branch B
|
||||
Finished grouping branch A
|
||||
Finished filtering groups for branch A
|
||||
Finished filtering groups for branch B
|
||||
Group filter time elapsed: 34.95
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: 144.75
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 149.73
|
||||
Closing remaining open files: ./flowy-run/GroupsA.h5... done ./flowy-run/GroupsB.h5... done
|
||||
|
||||
real 2m36.640s
|
||||
user 2m27.385s
|
||||
sys 0m3.508s
|
||||
|
||||
Splitter initiated
|
||||
Parsing and validation finished: 0.3
|
||||
Started filtering
|
||||
Finished filtering
|
||||
Filters ready
|
||||
Splitter time elapsed: 24.2
|
||||
Finished grouping branch B
|
||||
Finished grouping branch A
|
||||
Finished filtering groups for branch A
|
||||
Finished filtering groups for branch B
|
||||
Group filter time elapsed: 31.15
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: 145.9
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 151.19
|
||||
|
||||
real 2m31.325s
|
||||
user 2m26.629s
|
||||
sys 0m5.412s
|
||||
|
||||
modified reset/deepcopy
|
||||
(('reset', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 64), (26521, 11.015153884887695, 11.560714000001838))
|
||||
|
||||
|
||||
|
||||
|
||||
(('new_group', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 21), (1466, 6.5672850608825684, 5.3123339999998507))
|
||||
(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 25), (1468, 775.12532043457031, 766.78390699999591))
|
||||
(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 15), (3228, 155.0828640460968, 160.51002500000152))
|
||||
(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/splitter.py', 37), (3229, 87.616034030914307, 89.193573000000356))
|
||||
(('append', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 118), (3490, 35.743690967559814, 30.529941999999664))
|
||||
(('notify', '/usr/lib/python2.6/threading.py', 270), (6570, 10.859287977218628, 10.72066600000062))
|
||||
(('_is_owned', '/usr/lib/python2.6/threading.py', 219), (6695, 9.4564809799194336, 9.1245670000004111))
|
||||
(('final_result', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 57), (26521, 5.4859673976898193, 5.0482840000003648))
|
||||
|
||||
(('reset', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 45), (26521, 85.135001659393311, 88.205508000023968))
|
||||
|
||||
(('_deepcopy_dict', '/usr/lib/python2.6/copy.py', 251), (26712, 73.298033714294434, 75.524687000011454))
|
||||
(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/record.py', 129), (27270, 27.118208885192871, 27.781735000003209))
|
||||
(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/merger.py', 72), (97059, 33.632721662521362, 30.013754000007793))
|
||||
(('read_row', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 99), (99360, 518.74268817901611, 468.40537100055235))
|
||||
(('iterrows', '/usr/local/lib/python2.6/dist-packages/tables/table.py', 1441), (99377, 118.15105223655701, 106.11463399998161))
|
||||
(('read_rows_list', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 105), (99797, 522.83437442779541, 472.12965100054475))
|
||||
(('read_rows_list', '/home/melnikovkolya/classes/semester-3-project/flowy/record.py', 133), (99797, 550.52120852470398, 497.50723100058826))
|
||||
(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/merger.py', 23), (147484, 24.74915337562561, 21.317261000004237))
|
||||
(('deepcopy', '/usr/lib/python2.6/copy.py', 144), (187567, 161.90160441398621, 165.33823200019515))
|
||||
(('read', '/usr/local/lib/python2.6/dist-packages/tables/vlarray.py', 700), (195364, 274.85678458213806, 246.25141199899576))
|
||||
(('<genexpr>', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 102), (294714, 294.22120332717896, 264.55258099813909))
|
||||
(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 178), (856942, 596.70967555046082, 576.32406800022113))
|
||||
(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 101), (861729, 430.92800951004028, 418.1861820004529))
|
||||
(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 126), (989422, 290.51547265052795, 272.90903400041935))
|
||||
(('idx2long', '/usr/local/lib/python2.6/dist-packages/tables/utils.py', 66), (784846, 75.146798133850098, 69.772329999996373))
|
||||
(('is_idx', '/usr/local/lib/python2.6/dist-packages/tables/utils.py', 44), (784846, 26.284930467605591, 21.873351000002572))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/var/netflow/ft-data-fall09/kur-ft-data/2009-11-17/
|
||||
56992 records
|
||||
2438 records matched the http request
|
||||
|
||||
With profiler off:
|
||||
real 8m8.700s
|
||||
user 7m47.945s
|
||||
sys 0m12.909s
|
||||
|
||||
|
||||
Splitter initiated
|
||||
Parsing and validation finished: 1.29
|
||||
Started filtering
|
||||
Finished filtering
|
||||
Filters ready
|
||||
Splitter time elapsed: 58.21
|
||||
Finished grouping branch B
|
||||
Finished grouping branch A
|
||||
Finished filtering groups for branch A
|
||||
Finished filtering groups for branch B
|
||||
Group filter time elapsed: 59.8
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: 471.27
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 480.68
|
||||
|
||||
56992
|
||||
root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# time python flowy.py http-download.flw
|
||||
0.72
|
||||
Splitter initiated
|
||||
Started filtering
|
||||
Group filter time started: 0.29
|
||||
3955
|
||||
Filters ready
|
||||
Splitter finished
|
||||
Splitter time elapsed: 53.06
|
||||
Number of records in branch A 1985
|
||||
Number of records in branch B 2004
|
||||
Finished grouping branch A
|
||||
Finished group-filtering for branch A
|
||||
Finished grouping branch B
|
||||
Finished group-filtering for branch B
|
||||
Group filter threads joined: 57.68
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: 443.36
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 452.1
|
||||
Closing remaining open files: ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done
|
||||
|
||||
real 7m46.456s
|
||||
user 7m21.036s
|
||||
sys 0m11.921s
|
||||
|
||||
|
||||
(('new_group', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 21), (1824, 9.5541517734527588, 9.8766150000006974))
|
||||
(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 25), (1828, 1249.1410629749298, 1300.497268999989))
|
||||
(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/splitter.py', 37), (3955, 59.615991353988647, 62.479928999999061))
|
||||
(('split', '/home/melnikovkolya/classes/semester-3-project/flowy/splitter.py', 17), (3955, 30.423548460006714, 32.126016000000902))
|
||||
(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 15), (3956, 456.31715869903564, 475.12168400000229))
|
||||
(('get', '/usr/lib/python2.6/Queue.py', 150), (3957, 35.274902582168579, 37.742364999999495))
|
||||
(('append', '/usr/local/lib/python2.6/dist-packages/tables/vlarray.py', 452), (5486, 76.012235879898071, 76.39678599999479))
|
||||
(('append', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 118), (5785, 81.44921350479126, 81.341101000000435))
|
||||
(('notify', '/usr/lib/python2.6/threading.py', 270), (8002, 17.408251523971558, 17.825101000000359))
|
||||
(('_is_owned', '/usr/lib/python2.6/threading.py', 219), (8101, 14.244855642318726, 15.092936000000464))
|
||||
(('final_result', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 57), (56992, 15.892577886581421, 15.040958000006583))
|
||||
|
||||
(('reset', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 45), (56992, 255.76119065284729, 262.48040000008808))
|
||||
|
||||
(('_deepcopy_dict', '/usr/lib/python2.6/copy.py', 251), (57183, 218.50618243217468, 224.26205200008098))
|
||||
(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 89), (58365, 30.709211587905884, 31.189945000012358))
|
||||
(('iterate_fixed_fields', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 93), (58365, 19.963983297348022, 19.749231000007512))
|
||||
(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/record.py', 129), (58365, 86.714945554733276, 88.23755700004449))
|
||||
(('_deepcopy_list', '/usr/lib/python2.6/copy.py', 224), (114144, 72.901082038879395, 73.184596000045076))
|
||||
(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 90), (117636, 47.137009859085083, 43.210651000023745))
|
||||
(('_deepcopy_atomic', '/usr/lib/python2.6/copy.py', 197), (171331, 14.566928386688232, 13.152824000005694))
|
||||
(('_keep_alive', '/usr/lib/python2.6/copy.py', 261), (343098, 47.557926893234253, 39.274455000023863))
|
||||
(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/merger.py', 72), (343759, 89.168351411819458, 86.809352999718158))
|
||||
(('read_row', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 99), (347405, 1355.7759656906128, 1345.6080879980259))
|
||||
(('iterrows', '/usr/local/lib/python2.6/dist-packages/tables/table.py', 1441), (347422, 306.37827634811401, 304.82301899932509))
|
||||
(('read_rows_list', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 105), (348171, 1369.7901601791382, 1360.4090329980108))
|
||||
|
||||
(('deepcopy', '/usr/lib/python2.6/copy.py', 144), (400864, 485.14781737327576, 489.78665900019996))
|
||||
|
||||
(('__init__', '/home/melnikovkolya/classes/semester-3-project/flowy/record.py', 82), (408197, 79.613070487976074, 80.693067999662162))
|
||||
(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/merger.py', 23), (527995, 64.410658597946167, 62.123842999773387))
|
||||
(('read', '/usr/local/lib/python2.6/dist-packages/tables/vlarray.py', 700), (689950, 714.14480590820312, 706.58424299669286))
|
||||
(('<genexpr>', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 102), (1037339, 765.8496515750885, 758.55947299578656))
|
||||
(('_processRangeRead', '/usr/local/lib/python2.6/dist-packages/tables/leaf.py', 449), (1037372, 470.43238306045532, 463.84111299771757))
|
||||
(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 101), (1380363, 747.47748589515686, 753.67501099601259))
|
||||
(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 178), (1380426, 1028.9652721881866, 1053.8537989941979))
|
||||
(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 126), (1740570, 498.78313732147217, 495.35881499854258))
|
||||
(('EQ', '/home/melnikovkolya/classes/semester-3-project/flowy/operators.py', 63), (2370745, 182.36606240272522, 156.70575899921459))
|
||||
(('idx2long', '/usr/local/lib/python2.6/dist-packages/tables/utils.py', 66), (2764694, 214.65504741668701, 203.63286399914659))
|
||||
(('is_idx', '/usr/local/lib/python2.6/dist-packages/tables/utils.py', 44), (2764694, 75.347645044326782, 63.899976999761293))
|
||||
|
||||
|
||||
(('reset', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 64), (56992, 31.726502895355225, 31.213908000036554))
|
||||
(('deep_copy', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 41), (56994, 15.406083345413208, 16.889049000018872))
|
||||
|
||||
|
||||
7 days of data
|
||||
python ft2hdf.py /var/netflow/ft-data-fall09/sne-ft-data/2009/sneze/2009-12-0* netflow-trace.h5
|
||||
|
||||
246350 records in total
|
||||
12394 records match the query
|
||||
|
||||
profiling:
|
||||
Splitter initiated
|
||||
Parsing and validation finished: 2.22
|
||||
Started filtering
|
||||
Finished filtering
|
||||
Filters ready
|
||||
Splitter time elapsed: 1130.1
|
||||
Finished grouping branch B
|
||||
Finished filtering groups for branch B
|
||||
Finished grouping branch A
|
||||
Finished filtering groups for branch A
|
||||
Group filter time elapsed: 2123.665408
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: -185.553776
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: -114.543776
|
||||
|
||||
no profiling:
|
||||
Splitter initiated
|
||||
Parsing and validation finished: 0.26
|
||||
Started filtering
|
||||
Finished filtering
|
||||
Filters ready
|
||||
Splitter time elapsed: 320.43
|
||||
Finished grouping branch B
|
||||
Finished grouping branch A
|
||||
Finished filtering groups for branch B
|
||||
Finished filtering groups for branch A
|
||||
Group filter time elapsed: 922.42
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: 1039.122704
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 1074.252704
|
||||
Closing remaining open files: ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done
|
||||
|
||||
|
||||
start()real 90m16.511s
|
||||
user 86m23.020s
|
||||
sys 3m7.356s
|
||||
|
||||
Splitter initiated
|
||||
Parsing and validation finished: 0.31
|
||||
Started filtering
|
||||
Finished filtering
|
||||
Filters ready
|
||||
Splitter time elapsed: 346.66
|
||||
Finished grouping branch B
|
||||
Finished grouping branch A
|
||||
Finished filtering groups for branch B
|
||||
Finished filtering groups for branch A
|
||||
Group filter time elapsed: 916.19
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: 1037.532704
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 1073.552704
|
||||
|
||||
(('reset', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 64), (246349, 940.52704691886902, 994.15005099796895))
|
||||
|
||||
|
||||
(('reset', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 64), (246349, 111.18868279457092, 105.20649999988791))
|
||||
(('deep_copy', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 41), (246351, 61.105264902114868, 52.447237999959725))
|
||||
(('read_rows_list', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 105), (3155228, 13582.554839611053, 13318.368595361764))
|
||||
(('read_rows_list', '/home/melnikovkolya/classes/semester-3-project/flowy/record.py', 133), (3155228, 14223.106386899948, 13948.747855334786))
|
||||
(('read', '/usr/local/lib/python2.6/dist-packages/tables/vlarray.py', 700), (6280932, 6694.1691343784332, 6541.9808274548959))
|
||||
(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 178), (30651429, 17337.516788959503, 17566.637794171394))
|
||||
(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 101), (30677828, 12477.594463348389, 12583.8665639143))
|
||||
(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 126), (35383022, 8230.0888061523438, 8037.6824171527333))
|
||||
(('EQ', '/home/melnikovkolya/classes/semester-3-project/flowy/operators.py', 63), (40143460, 2728.9575715065002, 2304.1001345953482))
|
||||
|
||||
|
||||
root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# time flow-cat /var/netflow/ft-data-fall09/kur-ft-data/2009-11-16/ | flow-print | wc -l
|
||||
99925
|
||||
|
||||
root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# time python flowy.py http-download.flw0.77
|
||||
Splitter initiated
|
||||
Started filtering
|
||||
Group filter time started: 0.33
|
||||
7222
|
||||
Filters ready
|
||||
Splitter finished
|
||||
Splitter time elapsed: 100.03
|
||||
Number of records in branch B 3684
|
||||
Number of records in branch A 3644
|
||||
Finished grouping branch A
|
||||
Finished group-filtering for branch A
|
||||
Finished grouping branch B
|
||||
Finished group-filtering for branch B
|
||||
Group filter threads joined: 136.09
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: 960.34
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 974.11
|
||||
Closing remaining open files: ./flowy-run/GroupsA.h5... done ./flowy-run/GroupsB.h5... done
|
||||
|
||||
real 16m39.728s
|
||||
user 15m49.067s
|
||||
sys 0m26.002s
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# python ft2hdf.py /var/netflow/ft-data-fall09/sne-ft-data/2009/sneze/2009-11-* netflow-trace.h5
|
||||
|
||||
298063
|
||||
|
||||
root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# time python flowy.py http-download.flw
|
||||
0.84
|
||||
Splitter initiated
|
||||
Started filtering
|
||||
29448
|
||||
Filters ready
|
||||
Splitter finished
|
||||
Splitter time elapsed: 475.83
|
||||
Number of records in branch B 16666
|
||||
Number of records in branch A 16412
|
||||
Finished grouping branch B
|
||||
Finished group-filtering for branch B
|
||||
Finished grouping branch A
|
||||
Finished group-filtering for branch A
|
||||
Group filter threads joined: 1415.34
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: -1347.101888 = 11485
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: -1301.531888 = 11531
|
||||
Closing remaining open files: ./flowy-run/GroupsA.h5... done ./flowy-run/GroupsB.h5... done
|
||||
|
||||
real 192m11.507s = 11531
|
||||
user 185m39.648s
|
||||
sys 7m25.104s
|
||||
|
||||
|
||||
|
||||
The following example shows how heavy is the influence of performing the match operations (and the consequences in case of a match being found) by having a different load for 2 different branches.
|
||||
We can see that processing of Branch A, which has been matching records against a port number 443, has a running time of around 17 seconds, while the branch, which checked whether a record entry has a TCP protocol has taken around 90 seconds. The reason for a larger running time is that many more entries that match the prot=TCP requirement have been fround, and it takes an additional processing to index and to append the record to a group.
|
||||
|
||||
Though each of the executions of the reset function takes on average 9-10 times more time than the match function, the number of the executions of the match function is at least 30 times greater than that of the reset function.
|
||||
|
||||
After spot-profiling (running multi-thread profiler on certain sections of the code only), I could verify that the time spent in executing the match calls of the grouper module was causing most significant slow-down in all of the code, up to the merger module. Depending on the complexity of each of the match() calls, the execution time varied for the same number of function calls. The three match() calls from different classes form a nested chain, where one match() function, relies on another match(). The heaviest (in terms of time per execution) of all three match functions is the top-most match(), that comes from the Group class of the grouper module. Besides relying on a double-nested match call from two other classes, it also performs a
|
||||
and calculates the first and the last records of newly-formed group, which is necessary for relative comparisons.
|
||||
|
||||
with an average time spent per cycle (including the profiler overhead) being:
|
||||
(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 193), (280246, 151.20158743858337, 154.237679000132))
|
||||
|
||||
|
||||
A very simple GrouperModule class match, with a single
|
||||
|
||||
|
||||
def match(self, record, group):
|
||||
for rule in self.rules:
|
||||
if not rule.match(record, group):
|
||||
return False
|
||||
return True
|
||||
|
||||
This was followed by the filtering operation
|
||||
|
||||
|
||||
Splitter initiated
|
||||
GF validation started at: 0.89
|
||||
GF Validation required: 0.09
|
||||
Parsing and validation finished: 0.32
|
||||
Started filtering
|
||||
Grouping started at:Fitlering time started at: Grouping started at: 1.0
|
||||
1.0
|
||||
1.0
|
||||
Finished filtering
|
||||
Filtering required: 16.87
|
||||
Filters ready
|
||||
Splitter time elapsed: 17.11
|
||||
Finished grouping branch A
|
||||
Grouping time required branch A 17.34
|
||||
Current time is: 18.34
|
||||
Finished filtering groups for branch A
|
||||
Finished grouping branch B
|
||||
Grouping time required branch B 90.08
|
||||
Current time is: 91.08
|
||||
Finished filtering groups for branch B
|
||||
Group filter time elapsed: 90.41
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: 111.58
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 111.75
|
||||
Closing remaining open files: ./flowy-run/GroupsA.h5... done ./flowy-run/GroupsB.h5... done
|
||||
|
||||
real 1m52.516s
|
||||
user 1m50.411s
|
||||
sys 0m2.136s
|
||||
|
||||
|
||||
The performance of the group-filters differs significantly, depending on how many matching records have been found in the filtering stage.
|
||||
I.e., a filter for the port 443 results in relatively few records, while the filter for a port number >30000 results in many record matching it. The matching records need to be processed and stored for further group-filters, where the group filters try to form groups from the matched records. An example of running a query, which identifies flows with a destination port 443 and a source port > 30000, is shown next. It can be seen, that group-filtering of branch B, which is responsible for filtering out the srcport > 30000 request has a much larger running time than that of branch A, which looks only for those few records with a destination port of 443.
|
||||
Splitter initiated
|
||||
Parsing and validation finished: 0.28
|
||||
Started filtering
|
||||
Fitlering time started at: 0.98
|
||||
Finished filtering
|
||||
Filtering required: 33.49
|
||||
Filters ready
|
||||
Splitter time elapsed: 33.61
|
||||
Finished grouping branch A
|
||||
Grouping time finished for branch A 40.49
|
||||
Finished filtering groups for branch A
|
||||
Finished grouping branch B
|
||||
Grouping time finished for branch B 228.46
|
||||
Finished filtering groups for branch B
|
||||
Group filter time elapsed: 227.86
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: 252.77
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 253.31
|
||||
Closing remaining open files: ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done
|
||||
|
||||
real 4m11.671s
|
||||
user 4m9.296s
|
||||
sys 0m4.880s
|
||||
|
||||
|
||||
|
||||
==================================HTTPS PROFILE===========================================
|
||||
|
||||
|
||||
|
||||
|
||||
A query that selects source and destination ports is defined as follows. We used that query to compare simple running times of different tools:
|
||||
|
||||
splitter S {}
|
||||
|
||||
filter www_tcp {
|
||||
dstport = 443
|
||||
}
|
||||
|
||||
filter www_port {
|
||||
srcport = 443
|
||||
|
||||
}
|
||||
|
||||
grouper g_www_tcp {
|
||||
module g1 {
|
||||
}
|
||||
aggregate srcip, dstip, sum(bytes) as bytes, count(rec_id) as n,
|
||||
bitOR(tcp_flags) as flags, union(srcport) as srcports
|
||||
}
|
||||
|
||||
grouper g_www_port {
|
||||
module g1 {
|
||||
}
|
||||
aggregate srcip, dstip, sum(bytes) as bytes, count(rec_id) as n,
|
||||
bitOR(tcp_flags) as flags, union(dstport) as dstports
|
||||
}
|
||||
|
||||
groupfilter ggf {
|
||||
bitAND(flags, 0x13) = 0x13
|
||||
}
|
||||
|
||||
merger M {
|
||||
module m1 {
|
||||
branches B, A
|
||||
A m B delta 1440min
|
||||
}
|
||||
export m1
|
||||
}
|
||||
|
||||
ungrouper U {}
|
||||
|
||||
"./netflow-trace.h5" -> S
|
||||
S branch A -> www_tcp -> g_www_tcp -> ggf -> M
|
||||
S branch B -> www_port -> g_www_port -> ggf -> M
|
||||
M->U->"./ungroped.h5"
|
||||
|
||||
The same number of
|
||||
|
||||
/var/netflow/ft-data-fall09/sne-ft-data/2009/sneze/2009-11-29/
|
||||
|
||||
26521 records in total
|
||||
486 records match
|
||||
|
||||
root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# time python flowy.py https-flows.flw
|
||||
0.73
|
||||
Splitter initiated
|
||||
Started filtering
|
||||
486
|
||||
Filters ready
|
||||
Splitter finished
|
||||
Number of records in branch A 243 Number of records in branch B 243
|
||||
|
||||
Finished grouping branch A
|
||||
Finished group-filtering for branch A
|
||||
Finished grouping branch B
|
||||
Finished group-filtering for branch B
|
||||
Group filter threads joined
|
||||
Finished merging branches: ['B', 'A']
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 6.61
|
||||
Closing remaining open files: ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done ./netflow-trace.h5... done ./netflow-trace.h5... done ./flowy-run/GroupsB-merged.h5... done ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA-merged.h5... done ./flowy-run/MergedM.h5... done ./flowy-run/MergedM.h5... done ./ungroped.h5... done ./flowy-run/GroupsA.h5... done
|
||||
|
||||
real 0m14.245s
|
||||
user 0m7.168s
|
||||
sys 0m0.280s
|
||||
root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# time python flowy.py https-flows.flw
|
||||
0.81
|
||||
Splitter initiated
|
||||
Started filtering
|
||||
486
|
||||
Filters ready
|
||||
Splitter finished
|
||||
Number of records in branch B 243Number of records in branch A 243
|
||||
|
||||
Finished grouping branch A
|
||||
Finished group-filtering for branch A
|
||||
Finished grouping branch B
|
||||
Finished group-filtering for branch B
|
||||
Group filter threads joined
|
||||
Finished merging branches: ['B', 'A']
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 6.31
|
||||
Closing remaining open files: ./netflow-trace.h5... done ./flowy-run/GroupsA.h5... done ./flowy-run/MergedM.h5... done ./flowy-run/GroupsB-merged.h5... done ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done ./ungroped.h5... done ./netflow-trace.h5... done ./flowy-run/MergedM.h5... done ./flowy-run/GroupsA-merged.h5... done ./flowy-run/GroupsB.h5... done
|
||||
|
||||
real 0m9.051s
|
||||
user 0m7.072s
|
||||
sys 0m0.160s
|
||||
|
||||
root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# time python flowy.py https-flows.flw
|
||||
0.83
|
||||
Splitter initiated
|
||||
Started filtering
|
||||
Group filter time started: 0.23
|
||||
486
|
||||
Filters ready
|
||||
Number of records in branch A 243
|
||||
Splitter finished
|
||||
Splitter time elapsed: 6.1
|
||||
Finished grouping branch A
|
||||
Finished group-filtering for branch A
|
||||
Number of records in branch B 243
|
||||
Finished grouping branch B
|
||||
Finished group-filtering for branch B
|
||||
Group filter threads joined: 6.17
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: 6.23
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 7.36
|
||||
Closing remaining open files: ./flowy-run/GroupsB.h5... done ./netflow-trace.h5... done ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done ./netflow-trace.h5... done ./flowy-run/GroupsB-merged.h5... done ./flowy-run/GroupsA.h5... done ./flowy-run/MergedM.h5... done ./flowy-run/GroupsA-merged.h5... done ./ungroped.h5... done ./flowy-run/MergedM.h5... done
|
||||
|
||||
real 0m15.893s
|
||||
user 0m7.440s
|
||||
sys 0m0.868s
|
||||
|
||||
|
||||
|
||||
Most frequent:
|
||||
|
||||
(('final_result', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 99), (26521, 1.8366894721984863, 1.7001189999999156))
|
||||
|
||||
(('reset', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 83), (26521, 3.138737678527832, 3.0042079999998066))
|
||||
|
||||
(('deep_copy', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 58), (26523, 1.7581963539123535, 1.6681159999999338))
|
||||
|
||||
(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 92), (26526, 3.3419792652130127, 3.0921969999998495))
|
||||
|
||||
(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/record.py', 130), (26526, 9.8621282577514648, 9.6565820000015421))
|
||||
|
||||
(('iterate_fixed_fields', '/home/melnikovkolya/classes/semester-3-project/flowy/pytables.py', 96), (26526, 1.9721605777740479, 1.7561189999999769))
|
||||
|
||||
(('__init__', '/home/melnikovkolya/classes/semester-3-project/flowy/record.py', 82), (27015, 4.6438140869140625, 4.6482780000005732))
|
||||
|
||||
(('mask', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 90), (53042, 1.6173598766326904, 1.5800989999999153))
|
||||
|
||||
(('EQ', '/home/melnikovkolya/classes/semester-3-project/flowy/operators.py', 63), (53044, 1.4263303279876709, 1.1120729999999632))
|
||||
|
||||
(('match', '/home/melnikovkolya/classes/semester-3-project/flowy/filter.py', 134), (53046, 5.1699655055999756, 4.6562810000002663))
|
||||
|
||||
|
||||
Heaviest:
|
||||
|
||||
(('__init__', '/home/melnikovkolya/classes/semester-3-project/flowy/groupfilter_validator.py', 16), (1, 0.18725490570068359, 0.18801199999999962))
|
||||
|
||||
(('get_interval_records', '/home/melnikovkolya/classes/semester-3-project/flowy/timeindex.py', 57), (1, 0.2019498348236084, 0.20001300000000199))
|
||||
|
||||
(('pass_allen_indices_down', '/home/melnikovkolya/classes/semester-3-project/flowy/merger.py', 79), (1, 0.20258498191833496, 0.20001300000000199))
|
||||
|
||||
(('go', '/home/melnikovkolya/classes/semester-3-project/flowy/merger.py', 108), (1, 0.2162168025970459, 0.21201300000000245))
|
||||
|
||||
(('__init__', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper_validator.py', 11), (1, 0.22698211669921875, 0.22401300000000002))
|
||||
|
||||
(('__iter__', '/home/melnikovkolya/classes/semester-3-project/flowy/grouper.py', 36), (4, 1.1266498565673828, 1.1920739999999945))
|
||||
|
||||
(('_form_master_re', '/usr/lib/pymodules/python2.6/ply/lex.py', 482), (1, 0.30334997177124023, 0.22801499999999986))
|
||||
|
||||
(('validate_rules', '/usr/lib/pymodules/python2.6/ply/lex.py', 723), (1, 0.33556008338928223, 0.31602000000000008))
|
||||
|
||||
(('validate_all', '/usr/lib/pymodules/python2.6/ply/lex.py', 567), (1, 0.33656787872314453, 0.31602000000000008))
|
||||
|
||||
(('__init__', '/home/melnikovkolya/classes/semester-3-project/flowy/filter_validator.py', 10), (1, 0.37907099723815918, 0.3560230000000002))
|
||||
|
||||
(('go', '/home/melnikovkolya/classes/semester-3-project/flowy/groupfilter.py', 14), (2, 1.1871206760406494, 1.248076999999995))
|
||||
|
||||
(('create_impl', '/home/melnikovkolya/classes/semester-3-project/flowy/ungrouper_validator.py', 76), (1, 0.60985612869262695, 0.60803800000000052))
|
||||
|
||||
(('lex', '/usr/lib/pymodules/python2.6/ply/lex.py', 865), (1, 0.65552186965942383, 0.56003499999999995))
|
||||
|
||||
(('__init__', '/home/melnikovkolya/classes/semester-3-project/flowy/parser.py', 9), (1, 0.6572871208190918, 0.56403499999999995))
|
||||
|
||||
(('__init__', '/home/melnikovkolya/classes/semester-3-project/flowy/ungrouper_validator.py', 10), (1, 0.67348289489746094, 0.67204200000000114))
|
||||
|
||||
(('__init__', '/home/melnikovkolya/classes/semester-3-project/flowy/parser.py', 182), (1, 0.71254801750183105, 0.6200389999999999))
|
||||
|
||||
(('go', '/home/melnikovkolya/classes/semester-3-project/flowy/ungrouper.py', 29), (1, 1.85223388671875, 1.8081130000000023))
|
||||
|
||||
|
||||
|
||||
/var/netflow/ft-data-fall09/kur-ft-data/2009-11-17/
|
||||
56992 records
|
||||
|
||||
root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# python ft2hdf.py /var/netflow/ft-data-fall09/kur-ft-data/2009-11-17/ netflow-trace.h5
|
||||
root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# time python flowy.py https-flows.flw
|
||||
0.7
|
||||
Splitter initiated
|
||||
Started filtering
|
||||
Group filter time started: 0.27
|
||||
219
|
||||
Filters ready
|
||||
Splitter finished
|
||||
Splitter time elapsed: 13.2
|
||||
Number of records in branch A 158
|
||||
Finished grouping branch A
|
||||
Finished group-filtering for branch A
|
||||
Number of records in branch B 61
|
||||
Finished grouping branch B
|
||||
Finished group-filtering for branch B
|
||||
Group filter threads joined: 13.18
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: 13.23
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 13.83
|
||||
Closing remaining open files: ./netflow-trace.h5... done ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done ./netflow-trace.h5... done ./flowy-run/GroupsB-merged.h5... done ./flowy-run/GroupsA.h5... done ./flowy-run/GroupsB.h5... done ./flowy-run/MergedM.h5... done ./flowy-run/GroupsA-merged.h5... done ./ungroped.h5... done ./flowy-run/MergedM.h5... done
|
||||
|
||||
real 0m15.696s
|
||||
user 0m13.653s
|
||||
sys 0m1.004s
|
||||
root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# python printhdf.py ungroped.h5 | wc -l
|
||||
Closing remaining open files: ungroped.h5... done
|
||||
219
|
||||
|
||||
|
||||
|
||||
root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# python ft2hdf.py /var/netflow/ft-data-fall09/kur-ft-data/2009-11-16/ netflow-trace.h5
|
||||
|
||||
99924
|
||||
|
||||
root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# time python flowy.py https-flows.flw
|
||||
0.71
|
||||
Splitter initiated
|
||||
Started filtering
|
||||
Group filter time started: 0.27
|
||||
1434
|
||||
Filters ready
|
||||
Splitter finished
|
||||
Splitter time elapsed: 23.19
|
||||
Number of records in branch A 748
|
||||
Finished grouping branch A
|
||||
Finished group-filtering for branch A
|
||||
Number of records in branch B 686
|
||||
Finished grouping branch B
|
||||
Finished group-filtering for branch B
|
||||
Group filter threads joined: 23.23
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: 23.31
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 26.48
|
||||
Closing remaining open files: ./netflow-trace.h5... done ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsA.h5... done ./netflow-trace.h5... done ./flowy-run/GroupsB-merged.h5... done ./flowy-run/GroupsA.h5... done ./flowy-run/MergedM.h5... done ./flowy-run/GroupsA-merged.h5... done ./ungroped.h5... done ./flowy-run/MergedM.h5... done
|
||||
|
||||
real 0m28.767s
|
||||
user 0m24.486s
|
||||
sys 0m2.840s
|
||||
root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# python printhdf.py ungroped.h5 | wc -l
|
||||
Closing remaining open files: ungroped.h5... done
|
||||
1434
|
||||
|
||||
|
||||
|
||||
root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# python ft2hdf.py /var/netflow/ft-data-fall09/sne-ft-data/2009/sneze/2009-11-* netflow-trace.h5
|
||||
|
||||
298063
|
||||
|
||||
root@melnikovkolya-laptop:/home/melnikovkolya/classes/semester-3-project/flowy# python printhdf.py ungroped.h5 | wc -l
|
||||
Closing remaining open files: ungroped.h5... done
|
||||
4087
|
||||
|
||||
|
||||
0.76
|
||||
Splitter initiated
|
||||
Started filtering
|
||||
4087
|
||||
Filters ready
|
||||
Splitter finished
|
||||
Group filter time started: 53.73
|
||||
Splitter time elapsed: 53.73
|
||||
Number of records in branch A 2041
|
||||
Finished grouping branch A
|
||||
Finished group-filtering for branch A
|
||||
Number of records in branch B 2046
|
||||
Finished grouping branch B
|
||||
Finished group-filtering for branch B
|
||||
Group filter threads joined: 54.37
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: 54.47
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 63.47
|
||||
Closing remaining open files: ./flowy-run/GroupsB-merged.h5... done ./netflow-trace.h5... done ./ungroped.h5... done ./flowy-run/MergedM.h5... done ./flowy-run/MergedM.h5... done ./netflow-trace.h5... done ./flowy-run/GroupsA.h5... done ./flowy-run/GroupsA.h5... done ./flowy-run/GroupsA-merged.h5... done ./flowy-run/GroupsB.h5... done ./flowy-run/GroupsB.h5... done
|
||||
|
||||
real 1m8.146s
|
||||
user 1m3.576s
|
||||
sys 0m0.776s
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
\begin{mytinylisting}
|
||||
\begin{verbatim}
|
||||
splitter S {}
|
||||
|
||||
filter www_req {
|
||||
dstport = 443 OR dstport = 80 OR dstport = 8080
|
||||
unix_secs > 1259413200
|
||||
unix_secs < 1259445600
|
||||
}
|
||||
|
||||
filter www_res {
|
||||
srcport = 443 OR srcport = 80 OR srcport = 8080
|
||||
unix_secs < 1259445600
|
||||
unix_secs > 1259413200
|
||||
}
|
||||
|
||||
grouper g_www_req {
|
||||
module g1 {
|
||||
}
|
||||
aggregate bitOR(tcp_flags) as flags
|
||||
}
|
||||
|
||||
grouper g_www_res {
|
||||
module g1 {
|
||||
}
|
||||
aggregate bitOR(tcp_flags) as flags
|
||||
}
|
||||
|
||||
groupfilter ggf {
|
||||
bitAND(flags, 0x13) = 0x13
|
||||
}
|
||||
|
||||
merger M {
|
||||
module m1 {
|
||||
branches B, A
|
||||
A d B OR B d A
|
||||
}
|
||||
export m1
|
||||
}
|
||||
|
||||
ungrouper U {}
|
||||
|
||||
"./h5ports.h5" -> S
|
||||
S branch A -> www_req -> g_www_req -> ggf -> M
|
||||
S branch B -> www_res -> g_www_res -> ggf -> M
|
||||
M->U->"./portsungroped.h5"
|
||||
\end{verbatim}
|
||||
\end{mytinylisting}
|
||||
%
|
||||
Execution of that query
|
||||
\begin{verbatim}
|
||||
flowy# time python flowy.py ports.flw
|
||||
0.83
|
||||
Splitter initiated
|
||||
Started filtering
|
||||
Group filter time started: 0.3
|
||||
1463
|
||||
Filters ready
|
||||
Splitter finished
|
||||
Splitter time elapsed: 7.12
|
||||
Number of records in branch B 1463
|
||||
Finished grouping branch B
|
||||
Finished group-filtering for branch B
|
||||
Number of records in branch A 1463
|
||||
Finished grouping branch A
|
||||
Finished group-filtering for branch A
|
||||
Group filter threads joined: 7.26
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: 7.26
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 13.92
|
||||
|
||||
real 0m14.788s
|
||||
user 0m13.969s
|
||||
sys 0m0.900s
|
||||
\end{verbatim}
|
||||
|
||||
|
||||
BIN
pytables.pyc
BIN
pytables.pyc
Binary file not shown.
BIN
record.pyc
BIN
record.pyc
Binary file not shown.
1387
run-output.txt
1387
run-output.txt
File diff suppressed because it is too large
Load diff
|
|
@ -1,25 +0,0 @@
|
|||
0.82
|
||||
[Input('./netflow-trace.h5', 50, set([]), set([]), set([])), BranchNode('S', 50, set([]), set([]))]
|
||||
|
||||
[BranchNode('S', 51, set([]), set([])), Branch('A', 51, None, set([]), set([])), BranchNode('www_req', 51, set([]), set([])), BranchNode('g_www_req', 51, set([]), set([])), BranchNode('ggf', 51, set([]), set([])), BranchNode('M', 51, set([]), set([]))]
|
||||
|
||||
[BranchNode('S', 52, set([]), set([])), Branch('B', 52, None, set([]), set([])), BranchNode('www_res', 52, set([]), set([])), BranchNode('g_www_res', 52, set([]), set([])), BranchNode('ggf', 52, set([]), set([])), BranchNode('M', 52, set([]), set([]))]
|
||||
|
||||
[BranchNode('M', 53, set([]), set([])), BranchNode('U', 53, set([]), set([])), Output('./ungroped.h5', 53, set([]), set([]), set([]))]
|
||||
|
||||
Splitter initiated
|
||||
Parsing and validation finished: 0.31
|
||||
Started filtering
|
||||
Finished filtering
|
||||
Filters ready
|
||||
Splitter time elapsed: 346.66
|
||||
Finished grouping branch B
|
||||
Finished grouping branch A
|
||||
Finished filtering groups for branch B
|
||||
Finished filtering groups for branch A
|
||||
Group filter time elapsed: 916.19
|
||||
Finished merging branches: ['B', 'A']
|
||||
Merger time elapsed: 1037.532704
|
||||
Ungrouper U finished exectution
|
||||
FINISHED!
|
||||
Overall time elapsed: 1073.552704
|
||||
BIN
splitter.pyc
BIN
splitter.pyc
Binary file not shown.
Binary file not shown.
BIN
statement.pyc
BIN
statement.pyc
Binary file not shown.
BIN
timeindex.pyc
BIN
timeindex.pyc
Binary file not shown.
BIN
ungroped.h5
BIN
ungroped.h5
Binary file not shown.
BIN
ungroped.h5.300k
BIN
ungroped.h5.300k
Binary file not shown.
BIN
ungrouper.pyc
BIN
ungrouper.pyc
Binary file not shown.
Binary file not shown.
Binary file not shown.
Loading…
Reference in a new issue