From 4a46aa673ab55c2bce882743c519d675946f4c52 Mon Sep 17 00:00:00 2001 From: siddharthvipul Date: Mon, 4 Jun 2018 15:53:56 +0530 Subject: [PATCH 1/3] add smallfile parser with smaple data --- parsers/smallFileParser/README.md | 0 parsers/smallFileParser/parse.py | 84 +++++++++++++++++++ parsers/smallFileParser/sampleOutput.csv | 3 + parsers/smallFileParser/smallFileDataRead.out | 43 ++++++++++ .../smallFileParser/smallFileDataWrite.out | 43 ++++++++++ 5 files changed, 173 insertions(+) create mode 100644 parsers/smallFileParser/README.md create mode 100755 parsers/smallFileParser/parse.py create mode 100644 parsers/smallFileParser/sampleOutput.csv create mode 100644 parsers/smallFileParser/smallFileDataRead.out create mode 100644 parsers/smallFileParser/smallFileDataWrite.out diff --git a/parsers/smallFileParser/README.md b/parsers/smallFileParser/README.md new file mode 100644 index 0000000..e69de29 diff --git a/parsers/smallFileParser/parse.py b/parsers/smallFileParser/parse.py new file mode 100755 index 0000000..fa07c5a --- /dev/null +++ b/parsers/smallFileParser/parse.py @@ -0,0 +1,84 @@ +#!/usr/bin/python3 +''' +This program is for smallFile, parses the output and puts important numbers in a +csv file. + +usage: parse.py [-h] [-i INPUT] [-o OUTPUT] + +SmallFile Parser + +optional arguments: + -h, --help : show this help message and exit + -i INPUT, --input INPUT : smallfile output [path] file name + -o OUTPUT, --output OUTPUT : export parsed result in a file (csv) of the name +''' +import re +import argparse +import csv +import os.path + +def shred_digit(line): + ''' + This function fetches numbers from a line + ''' + return re.findall(r'\d+.\d+', line) + +def find_pattern(file_name): + ''' + This function is to get numbers by filtering pattern + ''' + attr = [] + with open(file_name) as fobj: + data_file = fobj.read().split("\n") + for line in data_file: + if re.match('^total data', line): + attr.append(shred_digit(line)[0]) + + elif re.match('^files/sec', line): + attr.append(shred_digit(line)[0]) + + elif re.match('^IOPS', line): + attr.append(shred_digit(line)[0]) + + elif re.match('^MiB/sec', line): + attr.append(shred_digit(line)[0]) + return attr +def write_to_csv(numbers, fname): + ''' + this function writes a given list in csv file + ''' + fname = fname + ".csv" + with open(fname, 'a+') as csv_file: + writer = csv.writer(csv_file) + writer.writerow(numbers) + +def main(): + ''' + Main function that checks the argument and performs accordingly + ''' + input_help = 'smallfile output file name' + output_help = 'export parsed result in a file (csv) of this name.' + + header = ['Total Data', 'Files/sec', 'IOPS', 'MiB/sec'] + + parser = argparse.ArgumentParser(description='SmallFile Parser') + parser.add_argument('-i', '--input', type=str, help=input_help) + parser.add_argument('-o', '--output', type=str, help=output_help) + args = parser.parse_args() + + if args.input and os.path.isfile(args.input): + result = find_pattern(args.input) + print(result) + if args.output: + if os.path.isfile(args.output+".csv"): + write_to_csv(result, args.output) + else: + write_to_csv(header, args.output) + write_to_csv(result, args.output) + else: + print(result) + else: + parser.print_help() + +if __name__ == '__main__': + main() diff --git a/parsers/smallFileParser/sampleOutput.csv b/parsers/smallFileParser/sampleOutput.csv new file mode 100644 index 0000000..50f6982 --- /dev/null +++ b/parsers/smallFileParser/sampleOutput.csv @@ -0,0 +1,3 @@ +Total Data,Files/sec,IOPS,MiB/sec +0.488,48.639892,48.639892,3.039993 +0.488,355.916851,355.916851,22.244803 diff --git a/parsers/smallFileParser/smallFileDataRead.out b/parsers/smallFileParser/smallFileDataRead.out new file mode 100644 index 0000000..2b90e5e --- /dev/null +++ b/parsers/smallFileParser/smallFileDataRead.out @@ -0,0 +1,43 @@ + version : 3.1 + hosts in test : None + top test directory(s) : ['/gluster-mount'] + operation : read + files/thread : 1000 + threads : 8 + record size (KB, 0 = maximum) : 0 + file size (KB) : 64 + file size distribution : fixed + files per dir : 100 + dirs per dir : 10 + threads share directories? : N + filename prefix : + filename suffix : + hash file number into dir.? : N + fsync after modify? : N + pause between files (microsec) : 1000 + minimum directories per sec : 50 + finish all requests? : Y + stonewall? : Y + measure response times? : N + verify read? : Y + verbose? : False + log to stderr? : False + ext.attr.size : 0 + ext.attr.count : 0 +host = dhcp37-55.lab.eng.blr.redhat.com,thr = 00,elapsed = 22.428904,files = 1000,records = 1000,status = ok +host = dhcp37-55.lab.eng.blr.redhat.com,thr = 01,elapsed = 22.353161,files = 1000,records = 1000,status = ok +host = dhcp37-55.lab.eng.blr.redhat.com,thr = 02,elapsed = 22.381413,files = 1000,records = 1000,status = ok +host = dhcp37-55.lab.eng.blr.redhat.com,thr = 03,elapsed = 22.409773,files = 1000,records = 1000,status = ok +host = dhcp37-55.lab.eng.blr.redhat.com,thr = 04,elapsed = 22.477160,files = 1000,records = 1000,status = ok +host = dhcp37-55.lab.eng.blr.redhat.com,thr = 05,elapsed = 22.396067,files = 1000,records = 1000,status = ok +host = dhcp37-55.lab.eng.blr.redhat.com,thr = 06,elapsed = 22.458280,files = 1000,records = 1000,status = ok +host = dhcp37-55.lab.eng.blr.redhat.com,thr = 07,elapsed = 22.449188,files = 1000,records = 1000,status = ok +total threads = 8 +total files = 8000 +total IOPS = 8000 +total data = 0.488 GiB +100.00% of requested files processed, minimum is 90.00 +elapsed time = 22.477 +files/sec = 355.916851 +IOPS = 355.916851 +MiB/sec = 22.244803 diff --git a/parsers/smallFileParser/smallFileDataWrite.out b/parsers/smallFileParser/smallFileDataWrite.out new file mode 100644 index 0000000..c03e6ae --- /dev/null +++ b/parsers/smallFileParser/smallFileDataWrite.out @@ -0,0 +1,43 @@ + version : 3.1 + hosts in test : None + top test directory(s) : ['/gluster-mount'] + operation : create + files/thread : 1000 + threads : 8 + record size (KB, 0 = maximum) : 0 + file size (KB) : 64 + file size distribution : fixed + files per dir : 100 + dirs per dir : 10 + threads share directories? : N + filename prefix : + filename suffix : + hash file number into dir.? : N + fsync after modify? : N + pause between files (microsec) : 1000 + minimum directories per sec : 50 + finish all requests? : Y + stonewall? : Y + measure response times? : N + verify read? : Y + verbose? : False + log to stderr? : False + ext.attr.size : 0 + ext.attr.count : 0 +host = dhcp37-55.lab.eng.blr.redhat.com,thr = 00,elapsed = 164.443753,files = 1000,records = 1000,status = ok +host = dhcp37-55.lab.eng.blr.redhat.com,thr = 01,elapsed = 164.234573,files = 1000,records = 1000,status = ok +host = dhcp37-55.lab.eng.blr.redhat.com,thr = 02,elapsed = 163.609050,files = 1000,records = 1000,status = ok +host = dhcp37-55.lab.eng.blr.redhat.com,thr = 03,elapsed = 163.997008,files = 1000,records = 1000,status = ok +host = dhcp37-55.lab.eng.blr.redhat.com,thr = 04,elapsed = 164.337409,files = 1000,records = 1000,status = ok +host = dhcp37-55.lab.eng.blr.redhat.com,thr = 05,elapsed = 164.416341,files = 1000,records = 1000,status = ok +host = dhcp37-55.lab.eng.blr.redhat.com,thr = 06,elapsed = 163.464908,files = 1000,records = 1000,status = ok +host = dhcp37-55.lab.eng.blr.redhat.com,thr = 07,elapsed = 164.474050,files = 1000,records = 1000,status = ok +total threads = 8 +total files = 8000 +total IOPS = 8000 +total data = 0.488 GiB +100.00% of requested files processed, minimum is 90.00 +elapsed time = 164.474 +files/sec = 48.639892 +IOPS = 48.639892 +MiB/sec = 3.039993 From 2cc8c0d12875b933e267c88c9162f3ba89a756de Mon Sep 17 00:00:00 2001 From: siddharthvipul Date: Mon, 4 Jun 2018 15:54:27 +0530 Subject: [PATCH 2/3] add readme --- parsers/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 parsers/README.md diff --git a/parsers/README.md b/parsers/README.md new file mode 100644 index 0000000..55e0fb9 --- /dev/null +++ b/parsers/README.md @@ -0,0 +1,8 @@ +# Parsers +This directory contains parsers that take output of tools like fio, +smallfile, ioZone etc and stores them in a csv file that can be used to compare +performances amongst releases. + +## Note: +* This is WIP code and far from done.. but it works! + From c5a0cafe391a0164da5b30a1cf5c4e187b3b5567 Mon Sep 17 00:00:00 2001 From: siddharthvipul Date: Mon, 4 Jun 2018 18:10:40 +0530 Subject: [PATCH 3/3] remove ips --- parsers/smallFileParser/smallFileDataRead.out | 16 ++++++++-------- parsers/smallFileParser/smallFileDataWrite.out | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/parsers/smallFileParser/smallFileDataRead.out b/parsers/smallFileParser/smallFileDataRead.out index 2b90e5e..ed110eb 100644 --- a/parsers/smallFileParser/smallFileDataRead.out +++ b/parsers/smallFileParser/smallFileDataRead.out @@ -24,14 +24,14 @@ log to stderr? : False ext.attr.size : 0 ext.attr.count : 0 -host = dhcp37-55.lab.eng.blr.redhat.com,thr = 00,elapsed = 22.428904,files = 1000,records = 1000,status = ok -host = dhcp37-55.lab.eng.blr.redhat.com,thr = 01,elapsed = 22.353161,files = 1000,records = 1000,status = ok -host = dhcp37-55.lab.eng.blr.redhat.com,thr = 02,elapsed = 22.381413,files = 1000,records = 1000,status = ok -host = dhcp37-55.lab.eng.blr.redhat.com,thr = 03,elapsed = 22.409773,files = 1000,records = 1000,status = ok -host = dhcp37-55.lab.eng.blr.redhat.com,thr = 04,elapsed = 22.477160,files = 1000,records = 1000,status = ok -host = dhcp37-55.lab.eng.blr.redhat.com,thr = 05,elapsed = 22.396067,files = 1000,records = 1000,status = ok -host = dhcp37-55.lab.eng.blr.redhat.com,thr = 06,elapsed = 22.458280,files = 1000,records = 1000,status = ok -host = dhcp37-55.lab.eng.blr.redhat.com,thr = 07,elapsed = 22.449188,files = 1000,records = 1000,status = ok +host = x,thr = 00,elapsed = 22.428904,files = 1000,records = 1000,status = ok +host = x,thr = 01,elapsed = 22.353161,files = 1000,records = 1000,status = ok +host = x,thr = 02,elapsed = 22.381413,files = 1000,records = 1000,status = ok +host = x,thr = 03,elapsed = 22.409773,files = 1000,records = 1000,status = ok +host = x,thr = 04,elapsed = 22.477160,files = 1000,records = 1000,status = ok +host = x,thr = 05,elapsed = 22.396067,files = 1000,records = 1000,status = ok +host = x,thr = 06,elapsed = 22.458280,files = 1000,records = 1000,status = ok +host = x,thr = 07,elapsed = 22.449188,files = 1000,records = 1000,status = ok total threads = 8 total files = 8000 total IOPS = 8000 diff --git a/parsers/smallFileParser/smallFileDataWrite.out b/parsers/smallFileParser/smallFileDataWrite.out index c03e6ae..0a9d3f7 100644 --- a/parsers/smallFileParser/smallFileDataWrite.out +++ b/parsers/smallFileParser/smallFileDataWrite.out @@ -24,14 +24,14 @@ log to stderr? : False ext.attr.size : 0 ext.attr.count : 0 -host = dhcp37-55.lab.eng.blr.redhat.com,thr = 00,elapsed = 164.443753,files = 1000,records = 1000,status = ok -host = dhcp37-55.lab.eng.blr.redhat.com,thr = 01,elapsed = 164.234573,files = 1000,records = 1000,status = ok -host = dhcp37-55.lab.eng.blr.redhat.com,thr = 02,elapsed = 163.609050,files = 1000,records = 1000,status = ok -host = dhcp37-55.lab.eng.blr.redhat.com,thr = 03,elapsed = 163.997008,files = 1000,records = 1000,status = ok -host = dhcp37-55.lab.eng.blr.redhat.com,thr = 04,elapsed = 164.337409,files = 1000,records = 1000,status = ok -host = dhcp37-55.lab.eng.blr.redhat.com,thr = 05,elapsed = 164.416341,files = 1000,records = 1000,status = ok -host = dhcp37-55.lab.eng.blr.redhat.com,thr = 06,elapsed = 163.464908,files = 1000,records = 1000,status = ok -host = dhcp37-55.lab.eng.blr.redhat.com,thr = 07,elapsed = 164.474050,files = 1000,records = 1000,status = ok +host =x ,thr = 00,elapsed = 164.443753,files = 1000,records = 1000,status = ok +host =x ,thr = 01,elapsed = 164.234573,files = 1000,records = 1000,status = ok +host =x ,thr = 02,elapsed = 163.609050,files = 1000,records = 1000,status = ok +host =x ,thr = 03,elapsed = 163.997008,files = 1000,records = 1000,status = ok +host =x ,thr = 04,elapsed = 164.337409,files = 1000,records = 1000,status = ok +host =x ,thr = 05,elapsed = 164.416341,files = 1000,records = 1000,status = ok +host =x ,thr = 06,elapsed = 163.464908,files = 1000,records = 1000,status = ok +host =x ,thr = 07,elapsed = 164.474050,files = 1000,records = 1000,status = ok total threads = 8 total files = 8000 total IOPS = 8000