Siddharthvipul/parsers

parsers/README.md

@@ -0,0 +1,8 @@
+# Parsers
+This directory contains parsers that take output of tools like fio,
+smallfile, ioZone etc and stores them in a csv file that can be used to compare
+performances amongst releases.
+
+## Note:
+* This is WIP code and far from done.. but it works!
+

parsers/smallFileParser/parse.py

@@ -0,0 +1,84 @@
+#!/usr/bin/python3
+'''
+This program is for smallFile, parses the output and puts important numbers in a
+csv file.
+
+usage: parse.py [-h] [-i INPUT] [-o OUTPUT]
+
+SmallFile Parser
+
+optional arguments:
+  -h, --help                 :  show this help message and exit
+  -i INPUT, --input INPUT    :  smallfile output [path] file name
+  -o OUTPUT, --output OUTPUT :  export parsed result in a file (csv) of the name
+'''
+import re
+import argparse
+import csv
+import os.path
+
+def shred_digit(line):
+    '''
+    This function fetches numbers from a line
+    '''
+    return re.findall(r'\d+.\d+', line)
+
+def find_pattern(file_name):
+    '''
+    This function is to get numbers by filtering pattern
+    '''
+    attr = []
+    with open(file_name) as fobj:
+        data_file = fobj.read().split("\n")
+        for line in data_file:
+            if re.match('^total data', line):
+                attr.append(shred_digit(line)[0])
+
+            elif re.match('^files/sec', line):
+                attr.append(shred_digit(line)[0])
+
+            elif re.match('^IOPS', line):
+                attr.append(shred_digit(line)[0])
+
+            elif re.match('^MiB/sec', line):
+                attr.append(shred_digit(line)[0])
+    return attr
+def write_to_csv(numbers, fname):
+    '''
+    this function writes a given list in csv file
+    '''
+    fname = fname + ".csv"
+    with open(fname, 'a+') as csv_file:
+        writer = csv.writer(csv_file)
+        writer.writerow(numbers)
+
+def main():
+    '''
+    Main function that checks the argument and performs accordingly
+    '''
+    input_help = 'smallfile output file name'
+    output_help = 'export parsed result in a file (csv) of this name.'
+
+    header = ['Total Data', 'Files/sec', 'IOPS', 'MiB/sec']
+
+    parser = argparse.ArgumentParser(description='SmallFile Parser')
+    parser.add_argument('-i', '--input', type=str, help=input_help)
+    parser.add_argument('-o', '--output', type=str, help=output_help)
+    args = parser.parse_args()
+
+    if args.input and os.path.isfile(args.input):
+        result = find_pattern(args.input)
+        print(result)
+        if args.output:
+            if os.path.isfile(args.output+".csv"):
+                write_to_csv(result, args.output)
+            else:
+                write_to_csv(header, args.output)
+                write_to_csv(result, args.output)
+        else:
+            print(result)
+    else:
+        parser.print_help()
+
+if __name__ == '__main__':
+    main()

parsers/smallFileParser/sampleOutput.csv

@@ -0,0 +1,3 @@
+Total Data,Files/sec,IOPS,MiB/sec
+0.488,48.639892,48.639892,3.039993
+0.488,355.916851,355.916851,22.244803

parsers/smallFileParser/smallFileDataRead.out

@@ -0,0 +1,43 @@
+                                 version : 3.1
+                           hosts in test : None
+                   top test directory(s) : ['/gluster-mount']
+                               operation : read
+                            files/thread : 1000
+                                 threads : 8
+           record size (KB, 0 = maximum) : 0
+                          file size (KB) : 64
+                  file size distribution : fixed
+                           files per dir : 100
+                            dirs per dir : 10
+              threads share directories? : N
+                         filename prefix : 
+                         filename suffix : 
+             hash file number into dir.? : N
+                     fsync after modify? : N
+          pause between files (microsec) : 1000
+             minimum directories per sec : 50
+                    finish all requests? : Y
+                              stonewall? : Y
+                 measure response times? : N
+                            verify read? : Y
+                                verbose? : False
+                          log to stderr? : False
+                           ext.attr.size : 0
+                          ext.attr.count : 0
+host = x,thr = 00,elapsed = 22.428904,files = 1000,records = 1000,status = ok
+host = x,thr = 01,elapsed = 22.353161,files = 1000,records = 1000,status = ok
+host = x,thr = 02,elapsed = 22.381413,files = 1000,records = 1000,status = ok
+host = x,thr = 03,elapsed = 22.409773,files = 1000,records = 1000,status = ok
+host = x,thr = 04,elapsed = 22.477160,files = 1000,records = 1000,status = ok
+host = x,thr = 05,elapsed = 22.396067,files = 1000,records = 1000,status = ok
+host = x,thr = 06,elapsed = 22.458280,files = 1000,records = 1000,status = ok
+host = x,thr = 07,elapsed = 22.449188,files = 1000,records = 1000,status = ok
+total threads = 8
+total files = 8000
+total IOPS = 8000
+total data =     0.488 GiB
+100.00% of requested files processed, minimum is  90.00
+elapsed time =    22.477
+files/sec = 355.916851
+IOPS = 355.916851
+MiB/sec = 22.244803

parsers/smallFileParser/smallFileDataWrite.out

@@ -0,0 +1,43 @@
+                                 version : 3.1
+                           hosts in test : None
+                   top test directory(s) : ['/gluster-mount']
+                               operation : create
+                            files/thread : 1000
+                                 threads : 8
+           record size (KB, 0 = maximum) : 0
+                          file size (KB) : 64
+                  file size distribution : fixed
+                           files per dir : 100
+                            dirs per dir : 10
+              threads share directories? : N
+                         filename prefix : 
+                         filename suffix : 
+             hash file number into dir.? : N
+                     fsync after modify? : N
+          pause between files (microsec) : 1000
+             minimum directories per sec : 50
+                    finish all requests? : Y
+                              stonewall? : Y
+                 measure response times? : N
+                            verify read? : Y
+                                verbose? : False
+                          log to stderr? : False
+                           ext.attr.size : 0
+                          ext.attr.count : 0
+host =x ,thr = 00,elapsed = 164.443753,files = 1000,records = 1000,status = ok
+host =x ,thr = 01,elapsed = 164.234573,files = 1000,records = 1000,status = ok
+host =x ,thr = 02,elapsed = 163.609050,files = 1000,records = 1000,status = ok
+host =x ,thr = 03,elapsed = 163.997008,files = 1000,records = 1000,status = ok
+host =x ,thr = 04,elapsed = 164.337409,files = 1000,records = 1000,status = ok
+host =x ,thr = 05,elapsed = 164.416341,files = 1000,records = 1000,status = ok
+host =x ,thr = 06,elapsed = 163.464908,files = 1000,records = 1000,status = ok
+host =x ,thr = 07,elapsed = 164.474050,files = 1000,records = 1000,status = ok
+total threads = 8
+total files = 8000
+total IOPS = 8000
+total data =     0.488 GiB
+100.00% of requested files processed, minimum is  90.00
+elapsed time =   164.474
+files/sec = 48.639892
+IOPS = 48.639892
+MiB/sec = 3.039993