Skip to content
This repository was archived by the owner on Mar 23, 2023. It is now read-only.

Commit 2d39b68

Browse files
KFilipekKFilipek
committed
[utils] add benchmark runner for YCSB
This tools allows to put multiple suites and run them one-by-one and parse the output to easy to use form as CSV files.
1 parent df8221c commit 2d39b68

3 files changed

Lines changed: 327 additions & 0 deletions

File tree

‎utils/parser.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# SPDX-License-Identifier: BSD-3-Clause
2+
# Copyright 2017-2022, Intel Corporation
3+
4+
import os
5+
from os.path import join, getsize
6+
7+
for root, dirs, filenames in os.walk('results'):
8+
if len(dirs) == 0:
9+
parsed_results = []
10+
for filename in filenames:
11+
if filename.split('_')[0] == 'run':
12+
with open(root + '/' + filename) as file_object:
13+
file_object.readline()
14+
trimmed_lines = []
15+
for line in file_object.readlines():
16+
record = tuple(line.replace(',','').split(' '))
17+
if record[0] != '[CLEANUP]' or record[0] != '[READ-FAILED]':
18+
if record[0] == '[READ]' or record[0] == '[INSERT]' or record[0] == '[UPDATE]' or record[0] == '[OVERALL]': #in case of READ
19+
try:
20+
int(record[1])
21+
except ValueError: #if cannot cast it's fine
22+
trimmed_lines.append(record)
23+
parsed_results.append([int(filename.split('_')[1].split('.')[0]), trimmed_lines])
24+
25+
parsed_results = sorted(parsed_results, key=lambda x: x[0], reverse=False)
26+
csv = []
27+
print root
28+
threads = 'Threads;#;'
29+
if len(parsed_results) <= 0:
30+
continue
31+
print '------CSV------'
32+
for i in range(0, len(parsed_results[0][1])):
33+
csv.append(parsed_results[0][1][i][0] + ';' + parsed_results[0][1][i][1] + ';')
34+
for test_result in parsed_results:
35+
threads += str(test_result[0]) + ';'
36+
for i, line in enumerate(test_result[1]):
37+
csv[i] += line[2].replace('\n','').replace('.',',') + ';'
38+
csv.insert(0, threads)
39+
with open(root + '/results.csv','w') as csv_file:
40+
for x in csv:
41+
csv_file.write(x + '\n')
42+
print x
43+
csv_file.close()

‎utils/run_suite.py‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
#!/usr/bin/python2
2+
3+
# SPDX-License-Identifier: BSD-3-Clause
4+
# Copyright 2017-2022, Intel Corporation
5+
6+
import json
7+
import os
8+
import subprocess
9+
10+
#comment
11+
# SUITE write_workload
12+
# THREADS 1 2 4 8 16 32 48 64 96
13+
# JOURNALING enabled/disabled
14+
# RECORDS 1000
15+
# OPERATIONS 100
16+
# READ_PROPORTION 0.0
17+
# UPDATE_PROPORTION 0.0
18+
# INSERT_PROPORTION 1.0
19+
# YCSB_NUMA 1
20+
# DROP_BEFORE
21+
# ENDSUITE
22+
23+
#GET PATHS FROM CONFIG FILE
24+
PATH_TO_YCSB = ''
25+
26+
path_configuration = open("path_configuration.txt", "r")
27+
for line in path_configuration:
28+
if line.startswith('YCSB_PATH='):
29+
arg = line.split("=")
30+
if len(arg) > 1:
31+
PATH_TO_YCSB = arg[1].replace('\n','')
32+
else:
33+
raise NameError('No path in YCSB_PATH!')
34+
35+
if not os.path.isdir(PATH_TO_YCSB):
36+
raise NameError('Wrong path to YCSB!')
37+
38+
class Test:
39+
def __init__(self):
40+
self.pmemkv_engine = "cmap"
41+
self.pmemkv_dbsize = 0
42+
self.pmemkv_dbpath = "/dev/shm/"
43+
self.workload_type = "workloada"
44+
self.testName = ""
45+
self.threads = []
46+
# self.journaling = ""
47+
self.records = 0
48+
self.operations = 0
49+
self.read_proportion = -1.0
50+
self.update_proportion = -1.0
51+
self.insert_proportion = -1.0
52+
self.ycsb_numa = -1
53+
# Actually we don't need creation
54+
# self.drop_before = -1
55+
# self.create_after_drop = -1
56+
self.is_load = -1
57+
def toJSON(self):
58+
return json.dumps(self, default=lambda o: o.__dict__,
59+
sort_keys=True, indent=4)
60+
61+
def getArgs(str):
62+
arguments = []
63+
for i in range(1, len(str)):
64+
arguments.append(str[i])
65+
return arguments
66+
67+
KEYWORDS = set(["THREADS", "JOURNALING", "RECORDS", "OPERATIONS",
68+
"READ_PROPORTION", "LOAD", "UPDATE_PROPORTION",
69+
"INSERT_PROPORTION", "YCSB_NUMA", "SUITE", "ENDSUITE",
70+
"DROP_BEFORE", "CREATE_AFTER_DROP", "PMEMKV_ENGINE",
71+
"PMEMKV_DBSIZE", "PMEMKV_DBPATH", "WORKLOAD_TYPE"]) #Add keyword if you need to extend implementation
72+
73+
# open meta file
74+
with open("test_suite.txt", "r") as configfile:
75+
configurations = []
76+
for line in configfile:
77+
splittedLine = line.split()
78+
if line == '\n' or line.startswith('#'):
79+
continue
80+
if len(set.intersection(KEYWORDS, splittedLine)) != 1:
81+
print(splittedLine)
82+
raise NameError('Too many keywords in single line!')
83+
84+
#get args if exists
85+
args = getArgs(splittedLine)
86+
87+
#if line starts from keyword we must read arguments
88+
if splittedLine[0] == "SUITE":
89+
configurations.append(Test())
90+
configurations[len(configurations)-1].testName = args[0]
91+
elif splittedLine[0] == "THREADS":
92+
configurations[len(configurations)-1].threads = args
93+
elif splittedLine[0] == "LOAD":
94+
configurations[len(configurations)-1].is_load = 1
95+
elif splittedLine[0] == "RECORDS":
96+
configurations[len(configurations)-1].records = args[0]
97+
elif splittedLine[0] == "OPERATIONS":
98+
configurations[len(configurations)-1].operations = args[0]
99+
elif splittedLine[0] == "READ_PROPORTION":
100+
configurations[len(configurations)-1].read_proportion = args[0]
101+
elif splittedLine[0] == "UPDATE_PROPORTION":
102+
configurations[len(configurations)-1].update_proportion = args[0]
103+
elif splittedLine[0] == "INSERT_PROPORTION":
104+
configurations[len(configurations)-1].insert_proportion = args[0]
105+
elif splittedLine[0] == "YCSB_NUMA":
106+
configurations[len(configurations)-1].ycsb_numa = args[0]
107+
elif splittedLine[0] == "PMEMKV_ENGINE":
108+
configurations[len(configurations)-1].pmemkv_engine = args[0]
109+
elif splittedLine[0] == "PMEMKV_DBSIZE":
110+
configurations[len(configurations)-1].pmemkv_dbsize = args[0]
111+
elif splittedLine[0] == "PMEMKV_DBPATH":
112+
configurations[len(configurations)-1].pmemkv_dbpath = args[0]
113+
elif splittedLine[0] == "WORKLOAD_TYPE":
114+
configurations[len(configurations)-1].workload_type = args[0]
115+
elif splittedLine[0] == "ENDSUITE":
116+
continue
117+
else:
118+
raise NameError('Unrecognized keyword')
119+
configfile.close()
120+
121+
print('Script read those tests:')
122+
i = 1
123+
for conf in configurations:
124+
print('{:>20} {:<12}'.format('Test#: ', str(i)))
125+
print('{:>20} {:<12}'.format("Name: ", conf.testName))
126+
print('{:>20} {:<12}'.format("Threads: " ,str(conf.threads)))
127+
print('{:>20} {:<12}'.format("Records: ", conf.records))
128+
print('{:>20} {:<12}'.format("Operation: ", conf.operations))
129+
print('{:>20} {:<12}'.format("Read proportion: ", str(conf.read_proportion)))
130+
print('{:>20} {:<12}'.format("Update proportion: ", str(conf.update_proportion)))
131+
print('{:>20} {:<12}'.format("Insert proportion: ", str(conf.insert_proportion)))
132+
print('{:>20} {:<12}'.format("Is load: ", str(conf.is_load)))
133+
print('{:>20} {:<12}'.format("NUMA for YCSB: ", conf.ycsb_numa))
134+
print('{:>20} {:<12}'.format("Workload type: ", conf.workload_type))
135+
print('{:>20} {:<12}'.format("Pmemkv engine: ", conf.pmemkv_engine))
136+
print('{:>20} {:<12}'.format("Pmemkv size: ", conf.pmemkv_dbsize))
137+
print('{:>20} {:<12}'.format("Pmemkv path: ", conf.pmemkv_dbpath))
138+
print("")
139+
i = i + 1
140+
141+
# PUT CONFIGURATION TO FILE IN PROPER PATH
142+
results_directory = "results/"
143+
if not os.path.exists(results_directory):
144+
os.makedirs(results_directory)
145+
i = 1
146+
with open(results_directory + '/configurations.json', 'w') as jsonconfig:
147+
for conf in configurations:
148+
jsonconfig.write(conf.toJSON() + '\n')
149+
if not os.path.exists(results_directory + conf.testName + '/'):
150+
os.makedirs(results_directory + conf.testName + '/')
151+
with open(results_directory + conf.testName + '/test_description.txt', 'a') as test_description:
152+
test_description.write('{:>20} {:<12}'.format('Test#: ', str(i)) + '\n') # 'Test #' + str(i)
153+
test_description.write('{:>20} {:<12}'.format("Name: ", conf.testName) + '\n')
154+
test_description.write('{:>20} {:<12}'.format("Threads: " ,str(conf.threads)) + '\n')
155+
test_description.write('{:>20} {:<12}'.format("Records: ", conf.records) + '\n')
156+
test_description.write('{:>20} {:<12}'.format("Operation: ", conf.operations) + '\n')
157+
test_description.write('{:>20} {:<12}'.format("Read proportion: ", str(conf.read_proportion)) + '\n')
158+
test_description.write('{:>20} {:<12}'.format("Update proportion: ", str(conf.update_proportion)) + '\n')
159+
test_description.write('{:>20} {:<12}'.format("Insert proportion: ", str(conf.insert_proportion)) + '\n')
160+
test_description.write('{:>20} {:<12}'.format("NUMA for YCSB: ", conf.ycsb_numa) + '\n')
161+
test_description.write('{:>20} {:<12}'.format("Workload type: ", conf.workload_type) + '\n')
162+
test_description.write('{:>20} {:<12}'.format("Pmemkv engine: ", conf.pmemkv_engine) + '\n')
163+
test_description.write('{:>20} {:<12}'.format("Pmemkv size: ", conf.pmemkv_dbsize) + '\n')
164+
test_description.write('{:>20} {:<12}'.format("Pmemkv path: ", conf.pmemkv_dbpath) + '\n')
165+
test_description.write('\n')
166+
i = i + 1
167+
168+
# run specified configurations
169+
generated_commands = []
170+
for test in configurations:
171+
command_prefix = ''
172+
command_suffix = ''
173+
174+
command_prefix = './run_workload.sh ' + test.testName
175+
176+
if not test.is_load == 1:
177+
command_prefix += ' run '
178+
else:
179+
command_prefix += ' load '
180+
181+
182+
# Put path to YCSB main directory
183+
command_suffix += PATH_TO_YCSB + ' '
184+
# Put operation numbers
185+
command_suffix += test.records + ' ' + test.operations + ' '
186+
# Put workload ratios
187+
command_suffix += test.read_proportion + ' ' + test.update_proportion + ' ' + test.insert_proportion + ' '
188+
# Put NUMA node
189+
if test.ycsb_numa == -1:
190+
print('NUMA node is not set for test: ' + test.testName + '.')
191+
command_suffix += test.ycsb_numa + ' '
192+
# Put workload type
193+
command_suffix += test.workload_type + ' '
194+
# Put engine specific fields
195+
command_suffix += test.pmemkv_engine + ' ' + test.pmemkv_dbsize + ' ' + test.pmemkv_dbpath + ' '
196+
197+
for thread_no in test.threads:
198+
# DROP&CREATE BEFORE NEXT INSERTS
199+
generated_commands.append(command_prefix + thread_no + ' ' + command_suffix)
200+
201+
# Generate script
202+
with open('testplan.sh','w') as testplan:
203+
testplan.write('#!/bin/bash\n')
204+
for x in generated_commands:
205+
testplan.write(x + '\n')
206+
print(generated_commands)

‎utils/run_workload.sh‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
#!/bin/bash
2+
3+
# SPDX-License-Identifier: BSD-3-Clause
4+
# Copyright 2017-2022, Intel Corporation
5+
6+
# Run workload from command line
7+
#
8+
# e.g. ./run_workload.sh run_cmap run 12 PATH_TO_YCSB 1000000 1000000
9+
# {0} {1} {2} {3} {4} {5} {6}
10+
# -1.0 -1.0 -1.0 1 workloadb csmap 80000000 DBPATH
11+
# {7} {8} {9} {10} {11} {12} {13} {14}
12+
# 1 - suite name
13+
# 2 - ycsb phase: load/run
14+
# 3 - thread count
15+
# 4 - path to YCSB
16+
# 5 - record count
17+
# 6 - operation count
18+
# 7 - read proportion
19+
# 8 - insert proportion
20+
# 9 - update proportion
21+
# 10 - NUMA node for YCSB
22+
# 11 - workload scenario (workload[a-f])
23+
####### Engine related args
24+
# 12 - pmemkv: engine name
25+
# 13 - pmemkv: pool size
26+
# 14 - pmemkv: path to pool
27+
28+
YCSB_PATH=/home/kfilipek/Development/YCSB/ # TODO(kfilipek): remove hardcoding
29+
echo $YCSB_PATH
30+
OLD_PATH=$(pwd)
31+
32+
echo $@
33+
echo "Passed $# argumets to script"
34+
35+
if [ "$#" -ne "14" ];
36+
then
37+
echo "Illegal number of parameters, should be 11. Check script documentation."
38+
exit 0
39+
fi
40+
41+
mkdir -p "results/$1/" # Create results directory: results/{test_suite_name}/
42+
# Prepare future arguments for YCSB
43+
NUMA_ARG=""
44+
READ_RATIO=""
45+
INSERT_RATIO=""
46+
UPDATE_RATIO=""
47+
if [ "$7" != "-1.0" ];
48+
then
49+
READ_RATIO=" -p readproportion=$7 "
50+
fi
51+
if [ "$8" != "-1.0" ];
52+
then
53+
INSERT_RATIO=" -p insertproportion=$8 "
54+
fi
55+
if [ "$9" != "-1.0" ];
56+
then
57+
UPDATE_RATIO=" -p updateproportion=$9 "
58+
fi
59+
if [ "${10}" != "-1" ];
60+
then
61+
NUMA_ARG=" numactl -N ${10} "
62+
fi
63+
# echo "READ_RATIO param: $READ_RATIO"
64+
# echo "INSERT_RATIO param: $INSERT_RATIO"
65+
# echo "UPDATE_RATIO param: $UPDATE_RATIO"
66+
# echo "NUMA NODE param: $NUMA_ARG"
67+
#exit
68+
69+
# TODOD(kfilipek): Implement splitting threads into processes
70+
cd $YCSB_PATH
71+
if [ "${2}" == "load" ];
72+
then
73+
# Remove old DB before new load phase
74+
rm -rf ${14}
75+
fi
76+
echo "PMEM_IS_PMEM_FORCE=1 $NUMA_ARG bin/ycsb.sh $2 pmemkv -P workloads/${11} -p hdrhistogram.percentiles=95,99,99.9,99.99 -p recordcount=$5 -p operationcount=$6 -p pmemkv.engine=${12} -p pmemkv.dbsize=${13} -p pmemkv.dbpath=${14} > $OLD_PATH/results/$1/${2}_${3}.log" >> $OLD_PATH/results/$1/cmds_executed.log
77+
PMEM_IS_PMEM_FORCE=1 $NUMA_ARG bin/ycsb.sh $2 pmemkv -P workloads/${11} -p hdrhistogram.percentiles=95,99,99.9,99.99 -p recordcount=$5 -p operationcount=$6 -p pmemkv.engine=${12} -p pmemkv.dbsize=${13} -p pmemkv.dbpath=${14} > $OLD_PATH/results/$1/${2}_${3}.log
78+
cd $OLD_PATH

0 commit comments

Comments
 (0)