Github_Repos/cvw
1
0
Fork 1
mirror of https://github.com/openhwgroup/cvw synced 2025-01-23 21:14:37 +00:00
Code Issues Packages Projects Releases Wiki Activity
33a26fb78c
cvw/bin/parseHPMC.py
Ross Thompson 33a26fb78c Updates to branch predictor collection.
2024-03-29 13:52:28 -05:00

503 lines
24 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/python3
###########################################
## Written: Rose Thompson ross1728@gmail.com
## Created: 20 September 2023
## Modified:
##
## Purpose: Parses the performance counters from a modelsim trace.
##
## A component of the CORE-V-WALLY configurable RISC-V project.
## https://github.com/openhwgroup/cvw
##
## Copyright (C) 2021-23 Harvey Mudd College & Oklahoma State University
##
## SPDX-License-Identifier: Apache-2.0 WITH SHL-2.1
##
## Licensed under the Solderpad Hardware License v 2.1 (the “License”); you may not use this file
## except in compliance with the License, or, at your option, the Apache License version 2.0. You
## may obtain a copy of the License at
##
## https:##solderpad.org/licenses/SHL-2.1/
##
## Unless required by applicable law or agreed to in writing, any work distributed under the
## License is distributed on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
## either express or implied. See the License for the specific language governing permissions
## and limitations under the License.
################################################################################################
import os
import sys
import matplotlib.pyplot as plt
import math
import numpy as np
import argparse
RefDataBP = [('twobitCModel6', 'twobitCModel', 64, 128, 10.0060297551637), ('twobitCModel8', 'twobitCModel', 256, 512, 8.4320392215602), ('twobitCModel10', 'twobitCModel', 1024, 2048, 7.29493318805151),
('twobitCModel12', 'twobitCModel', 4096, 8192, 6.84739616147794), ('twobitCModel14', 'twobitCModel', 16384, 32768, 5.68432926870082), ('twobitCModel16', 'twobitCModel', 65536, 131072, 5.68432926870082),
('gshareCModel6', 'gshareCModel', 64, 128, 11.4737703417701), ('gshareCModel8', 'gshareCModel', 256, 512, 8.52341470761974), ('gshareCModel10', 'gshareCModel', 1024, 2048, 6.32975690693015),
('gshareCModel12', 'gshareCModel', 4096, 8192, 4.55424632377659), ('gshareCModel14', 'gshareCModel', 16384, 32768, 3.54251547725509), ('gshareCModel16', 'gshareCModel', 65536, 131072, 1.90424999467293)]
RefDataBTB = [('BTBCModel6', 'BTBCModel', 64, 128, 1.51480272475844), ('BTBCModel8', 'BTBCModel', 256, 512, 0.209057900418965), ('BTBCModel10', 'BTBCModel', 1024, 2048, 0.0117345454469572),
('BTBCModel12', 'BTBCModel', 4096, 8192, 0.00125540990359826), ('BTBCModel14', 'BTBCModel', 16384, 32768, 0.000732471628510962), ('BTBCModel16', 'BTBCModel', 65536, 131072, 0.000732471628510962)]
def ParseBranchListFile(path):
'''Take the path to the list of Questa Sim log files containing the performance counters outputs. File
is formated in row columns. Each row is a trace with the file, branch predictor type, and the parameters.
parameters can be any number and depend on the predictor type. Returns a list of lists.'''
lst = []
BranchList = open(path, 'r')
for line in BranchList:
tokens = line.split()
predictorLog = os.path.dirname(path) + '/' + tokens[0]
predictorType = tokens[1]
predictorParams = tokens[2::]
lst.append([predictorLog, predictorType, predictorParams])
#print(predictorLog, predictorType, predictorParams)
return lst
def ProcessFile(fileName):
'''Extract preformance counters from a modelsim log. Outputs a list of tuples for each test/benchmark.
The tuple contains the test name, optimization characteristics, and dictionary of performance counters.'''
# 1 find lines with Read memfile and extract test name
# 2 parse counters into a list of (name, value) tuples (dictionary maybe?)
benchmarks = []
transcript = open(fileName, 'r')
HPMClist = { }
testName = ''
for line in transcript.readlines():
lineToken = line.split()
if(len(lineToken) > 3 and lineToken[1] == 'Read' and lineToken[2] == 'memfile'):
opt = lineToken[3].split('/')[-4]
testName = lineToken[3].split('/')[-1].split('.')[0]
HPMClist = { }
elif(len(lineToken) > 4 and lineToken[1][0:3] == 'Cnt'):
countToken = line.split('=')[1].split()
value = int(countToken[0]) if countToken[0] != 'x' else 0
name = ' '.join(countToken[1:])
HPMClist[name] = value
elif ('is done' in line):
benchmarks.append((testName, opt, HPMClist))
return benchmarks
def ComputeStats(benchmarks):
for benchmark in benchmarks:
(nameString, opt, dataDict) = benchmark
dataDict['CPI'] = 1.0 * int(dataDict['Mcycle']) / int(dataDict['InstRet'])
dataDict['BDMR'] = 100.0 * int(dataDict['BP Dir Wrong']) / int(dataDict['Br Count'])
dataDict['BTMR'] = 100.0 * int(dataDict['BP Target Wrong']) / (int(dataDict['Br Count']) + int(dataDict['Jump Not Return']))
dataDict['RASMPR'] = 100.0 * int(dataDict['RAS Wrong']) / int(dataDict['Return'])
dataDict['ClassMPR'] = 100.0 * int(dataDict['Instr Class Wrong']) / int(dataDict['InstRet'])
dataDict['ICacheMR'] = 100.0 * int(dataDict['I Cache Miss']) / int(dataDict['I Cache Access'])
cycles = int(dataDict['I Cache Miss'])
if(cycles == 0): ICacheMR = 0
else: ICacheMR = 100.0 * int(dataDict['I Cache Cycles']) / cycles
dataDict['ICacheMT'] = ICacheMR
dataDict['DCacheMR'] = 100.0 * int(dataDict['D Cache Miss']) / int(dataDict['D Cache Access'])
(nameString, opt, dataDict) = benchmark
cycles = int(dataDict['D Cache Miss'])
if(cycles == 0): DCacheMR = 0
else: DCacheMR = 100.0 * int(dataDict['D Cache Cycles']) / cycles
dataDict['DCacheMT'] = DCacheMR
def ComputeGeometricAverage(benchmarks):
fields = ['BDMR', 'BTMR', 'RASMPR', 'ClassMPR', 'ICacheMR', 'DCacheMR', 'CPI', 'ICacheMT', 'DCacheMT']
AllAve = {}
for field in fields:
Product = 1
index = 0
for (testName, opt, HPMCList) in benchmarks:
#print(HPMCList)
value = HPMCList[field]
if(value != 0): Product *= value # if that value is 0 exclude from mean because it destories the geo mean
index += 1
AllAve[field] = Product ** (1.0/index)
benchmarks.append(('Mean', '', AllAve))
def GenerateName(predictorType, predictorParams):
if(predictorType == 'gshare' or predictorType == 'twobit' or predictorType == 'btb' or predictorType == 'class' or predictorType == 'ras' or predictorType == 'global'):
return predictorType + predictorParams[0]
elif(predictorType == 'local'):
return predictorType + predictorParams[0] + '_' + predictorParams[1]
else:
print(f'Error unsupported predictor type {predictorType}')
sys.exit(-1)
def GenerateDisplayName(predictorType, predictorParams):
if(predictorType == 'gshare' or predictorType == 'twobit' or predictorType == 'btb' or predictorType == 'class' or predictorType == 'ras' or predictorType == 'global'):
return predictorType
elif(predictorType == 'local'):
return predictorType + predictorParams[0]
else:
print(f'Error unsupported predictor type {predictorType}')
sys.exit(-1)
def ComputePredNumEntries(predictorType, predictorParams):
if(predictorType == 'gshare' or predictorType == 'twobit' or predictorType == 'btb' or predictorType == 'class' or predictorType == 'global'):
return 2**int(predictorParams[0])
elif(predictorType == 'ras'):
return int(predictorParams[0])
elif(predictorType == 'local'):
return 2**int(predictorParams[0]) * int(predictorParams[1]) + 2**int(predictorParams[1])
else:
print(f'Error unsupported predictor type {predictorType}')
sys.exit(-1)
def ComputePredSize(predictorType, predictorParams):
if(predictorType == 'gshare' or predictorType == 'twobit' or predictorType == 'btb' or predictorType == 'class' or predictorType == 'global'):
return 2*2**int(predictorParams[0])
elif(predictorType == 'ras'):
return int(predictorParams[0])
elif(predictorType == 'local'):
return 2**int(predictorParams[0]) * int(predictorParams[1]) + 2*2**int(predictorParams[1])
else:
print(f'Error unsupported predictor type {predictorType}')
sys.exit(-1)
def BuildDataBase(predictorLogs):
# Once done with the following loop, performanceCounterList will contain the predictor type and size along with the
# raw performance counter data and the processed data on a per benchmark basis. It also includes the geometric mean.
# list
# branch predictor configuration 0 (tuple)
# benchmark name
# compiler optimization
# data (dictionary)
# dictionary of performance counters
# branch predictor configuration 1 (tuple)
# benchmark name (dictionary)
# compiler optimization
# data
# dictionary of performance counters
# ...
performanceCounterList = []
for trace in predictorLogs:
predictorLog = trace[0]
predictorType = trace[1]
predictorParams = trace[2]
# Extract the performance counter data
performanceCounters = ProcessFile(predictorLog)
ComputeStats(performanceCounters)
ComputeGeometricAverage(performanceCounters)
#print(performanceCounters)
performanceCounterList.append([GenerateName(predictorType, predictorParams), GenerateDisplayName(predictorType, predictorParams), performanceCounters, ComputePredNumEntries(predictorType, predictorParams), ComputePredSize(predictorType, predictorParams)])
return performanceCounterList
def ReorderDataBase(performanceCounterList):
# Reorder the data so the benchmark name comes first, then the branch predictor configuration
benchmarkFirstList = []
for (predictorName, predictorPrefixName, benchmarks, entries, size) in performanceCounterList:
for benchmark in benchmarks:
(nameString, opt, dataDict) = benchmark
benchmarkFirstList.append((nameString, opt, predictorName, predictorPrefixName, entries, size, dataDict))
return benchmarkFirstList
def ExtractSelectedData(benchmarkFirstList):
# now extract all branch prediction direction miss rates for each
# namestring + opt, config
benchmarkDict = { }
for benchmark in benchmarkFirstList:
(name, opt, config, prefixName, entries, size, dataDict) = benchmark
#print(f'config = {config}, prefixName = {prefixName} entries = {entries}')
# use this code to distinguish speed opt and size opt.
#if opt == 'bd_speedopt_speed': NewName = name+'Sp'
#elif opt == 'bd_sizeopt_speed': NewName = name+'Sz'
#else: NewName = name
NewName = name
#print(NewName)
#NewName = name+'_'+opt
if NewName in benchmarkDict:
benchmarkDict[NewName].append((config, prefixName, entries, size, dataDict[ReportPredictorType]))
else:
benchmarkDict[NewName] = [(config, prefixName, entries, size, dataDict[ReportPredictorType])]
return benchmarkDict
def ReportAsTable(benchmarkDict):
refLine = benchmarkDict['Mean']
FirstLine = []
SecondLine = []
for Elements in refLine:
(name, typ, size, entries, val) = Elements
FirstLine.append(name)
SecondLine.append(entries if not args.size else size)
sys.stdout.write('benchmark\t\t')
for name in FirstLine:
if(len(name) < 8): sys.stdout.write('%s\t\t' % name)
else: sys.stdout.write('%s\t' % name)
sys.stdout.write('\n')
sys.stdout.write('size\t\t\t')
for size in SecondLine:
if(len(str(size)) < 8): sys.stdout.write('%d\t\t' % size)
else: sys.stdout.write('%d\t' % size)
sys.stdout.write('\n')
if(args.summary):
sys.stdout.write('Mean\t\t\t')
for (name, typ, size, entries, val) in refLine:
sys.stdout.write('%0.2f\t\t' % (val if not args.invert else 100 - val))
sys.stdout.write('\n')
if(not args.summary):
for benchmark in benchmarkDict:
length = len(benchmark)
if(length < 8): sys.stdout.write('%s\t\t\t' % benchmark)
elif(length < 16): sys.stdout.write('%s\t\t' % benchmark)
else: sys.stdout.write('%s\t' % benchmark)
for (name, typ, entries, size, val) in benchmarkDict[benchmark]:
sys.stdout.write('%0.2f\t\t' % (val if not args.invert else 100 -val))
sys.stdout.write('\n')
def ReportAsText(benchmarkDict):
if(args.summary):
mean = benchmarkDict['Mean']
print('Mean')
for (name, typ, entries. size, val) in mean:
sys.stdout.write('%s %s %0.2f\n' % (name, entries if not args.size else size, val if not args.invert else 100 - val))
if(not args.summary):
for benchmark in benchmarkDict:
print(benchmark)
for (name, type, entries, size, val) in benchmarkDict[benchmark]:
sys.stdout.write('%s %s %0.2f\n' % (name, entries if not args.size else size, val if not args.invert else 100 - val))
def Inversion(lst):
return [x if not args.invert else 100 - x for x in lst]
def BarGraph(seriesDict, xlabelList, BenchPerRow, FileName, IncludeLegend):
index = 0
NumberInGroup = len(seriesDict)
# Figure out width of bars. NumberInGroup bars + want 2 bar space
# the space between groups is 1
EffectiveNumInGroup = NumberInGroup + 2
barWidth = 1 / EffectiveNumInGroup
fig = plt.subplots(figsize = (EffectiveNumInGroup*BenchPerRow/8, 4))
colors = ['blue', 'blue', 'blue', 'blue', 'blue', 'blue', 'black', 'black', 'black', 'black', 'black', 'black']
for name in seriesDict:
values = seriesDict[name]
xpos = np.arange(len(values))
xpos = [x + index*barWidth for x in xpos]
plt.bar(xpos, Inversion(values), width=barWidth, edgecolor='grey', label=name, color=colors[index%len(colors)])
index += 1
plt.xticks([r + barWidth*(NumberInGroup/2-0.5) for r in range(0, BenchPerRow)], xlabelList)
plt.xlabel('Benchmark')
if(not args.invert): plt.ylabel('Misprediction Rate (%)')
else: plt.ylabel('Prediction Accuracy (%)')
if(IncludeLegend): plt.legend(loc='upper right', ncol=2)
plt.savefig(FileName)
def SelectPartition(xlabelListBig, seriesDictBig, group, BenchPerRow):
seriesDictTrunk = {}
for benchmarkName in seriesDictBig:
lst = seriesDictBig[benchmarkName]
seriesDictTrunk[benchmarkName] = lst[group*BenchPerRow:(group+1)*BenchPerRow]
xlabelListTrunk = xlabelListBig[group*BenchPerRow:(group+1)*BenchPerRow]
return(xlabelListTrunk, seriesDictTrunk)
def ReportAsGraph(benchmarkDict, bar, FileName):
def FormatToPlot(currBenchmark):
names = []
sizes = []
values = []
typs = []
for config in currBenchmark:
names.append(config[0])
sizes.append(config[1])
values.append(config[2])
typs.append(config[3])
return (names, sizes, values, typs)
titlesInvert = {'BDMR' : 'Branch Direction Accuracy',
'BTMR' : 'Branch Target Accuracy',
'RASMPR': 'RAS Accuracy',
'ClassMPR': 'Class Prediction Accuracy'}
titles = {'BDMR' : 'Branch Direction Misprediction',
'BTMR' : 'Branch Target Misprediction',
'RASMPR': 'RAS Misprediction',
'ClassMPR': 'Class Misprediction'}
if(args.summary):
markers = ['x', '.', '+', '*', '^', 'o', ',', 's']
colors = ['blue', 'black', 'gray', 'dodgerblue', 'lightsteelblue', 'turquoise', 'black', 'blue']
temp = benchmarkDict['Mean']
# the benchmarkDict['Mean'] contains sequencies of results for multiple
# branch predictors with various parameterizations
# group the parameterizations by the common typ.
sequencies = {}
for (name, typ, entries, size, value) in benchmarkDict['Mean']:
if not typ in sequencies:
sequencies[typ] = [(entries if not args.size else int(size/8), value)]
else:
sequencies[typ].append((entries if not args.size else int(size/8) ,value))
# then graph the common typ as a single line+scatter plot
# finally repeat for all typs of branch predictors and overlay
fig, axes = plt.subplots()
index = 0
if(args.invert): plt.title(titlesInvert[ReportPredictorType])
else: plt.title(titles[ReportPredictorType])
for branchPredName in sequencies:
data = sequencies[branchPredName]
(xdata, ydata) = zip(*data)
if args.invert: ydata = [100 - x for x in ydata]
axes.plot(xdata, ydata, color=colors[index])
axes.scatter(xdata, ydata, label=branchPredName, color=colors[index], marker=markers[index])
index = (index + 1) % len(markers)
axes.legend(loc='upper left')
axes.set_xscale("log")
axes.set_ylabel('Prediction Accuracy')
Xlabel = 'Entries' if not args.size else 'Size (bytes)'
axes.set_xlabel(Xlabel)
axes.set_xticks(xdata)
axes.set_xticklabels(xdata)
axes.grid(color='b', alpha=0.5, linestyle='dashed', linewidth=0.5)
if(FileName == None): plt.show()
else: plt.savefig(FileName)
# if(not args.summary):
# size = len(benchmarkDict)
# sizeSqrt = math.sqrt(size)
# isSquare = math.isclose(sizeSqrt, round(sizeSqrt))
# numCol = math.floor(sizeSqrt)
# numRow = numCol + (0 if isSquare else 1)
# index = 1
# fig = plt.figure()
# for benchmarkName in benchmarkDict:
# currBenchmark = benchmarkDict[benchmarkName]
# (names, typs, sizes, values) = FormatToPlot(currBenchmark)
# #axes.plot(numRow, numCol, index)
# ax = fig.add_subplot(numRow, numCol, index)
# ax.bar(names, values)
# ax.title.set_text(benchmarkName)
# #plt.ylabel('BR Dir Miss Rate (%)')
# #plt.xlabel('Predictor')
# index += 1
if(not args.summary):
NumBenchmarks = len(benchmarkDict)
NumBenchmarksSqrt = math.sqrt(NumBenchmarks)
isSquare = math.isclose(NumBenchmarksSqrt, round(NumBenchmarksSqrt))
numCol = math.floor(NumBenchmarksSqrt)
numRow = numCol + (0 if isSquare else 1)
index = 1
BenchPerRow = 5
xlabelList = []
seriesDict = {}
for benchmarkName in benchmarkDict:
currBenchmark = benchmarkDict[benchmarkName]
xlabelList.append(benchmarkName)
for (name, typ, entries, size, value) in currBenchmark:
if(name not in seriesDict):
seriesDict[name] = [value]
else:
seriesDict[name].append(value)
if(index >= BenchPerRow): break
index += 1
xlabelListBig = []
seriesDictBig = {}
for benchmarkName in benchmarkDict:
currBenchmark = benchmarkDict[benchmarkName]
xlabelListBig.append(benchmarkName)
for (name, typ, entries, size, value) in currBenchmark:
if(name not in seriesDictBig):
seriesDictBig[name] = [value]
else:
seriesDictBig[name].append(value)
#The next step will be to split the benchmarkDict into length BenchPerRow pieces then repeat the following code
# on each piece.
for row in range(0, math.ceil(NumBenchmarks / BenchPerRow)):
(xlabelListTrunk, seriesDictTrunk) = SelectPartition(xlabelListBig, seriesDictBig, row, BenchPerRow)
FileName = 'barSegment%d.svg' % row
groupLen = len(xlabelListTrunk)
BarGraph(seriesDictTrunk, xlabelListTrunk, groupLen, FileName, (row == 0))
# main
parser = argparse.ArgumentParser(description='Parses performance counters from a Questa Sim trace to produce a graph or graphs.')
# parse program arguments
metric = parser.add_mutually_exclusive_group()
metric.add_argument('-r', '--ras', action='store_const', help='Plot return address stack (RAS) performance.', default=False, const=True)
metric.add_argument('-d', '--direction', action='store_const', help='Plot direction prediction (2-bit, Gshare, local, etc) performance.', default=False, const=True)
metric.add_argument('-t', '--target', action='store_const', help='Plot branch target buffer (BTB) performance.', default=False, const=True)
metric.add_argument('-c', '--iclass', action='store_const', help='Plot instruction classification performance.', default=False, const=True)
parser.add_argument('-s', '--summary', action='store_const', help='Show only the geometric average for all benchmarks.', default=False, const=True)
parser.add_argument('-b', '--bar', action='store_const', help='Plot graphs.', default=False, const=True)
parser.add_argument('-g', '--reference', action='store_const', help='Include the golden reference model from branch-predictor-simulator. Data stored statically at the top of %(prog)s. If you need to regenreate use CModelBranchAcurracy.sh', default=False, const=True)
parser.add_argument('-i', '--invert', action='store_const', help='Invert metric. Example Branch miss prediction becomes prediction accuracy. 100 - miss rate', default=False, const=True)
parser.add_argument('--size', action='store_const', help='Display x-axis as size in bits rather than number of table entries', default=False, const=True)
displayMode = parser.add_mutually_exclusive_group()
displayMode.add_argument('--text', action='store_const', help='Display in text format only.', default=False, const=True)
displayMode.add_argument('--table', action='store_const', help='Display in text format only.', default=False, const=True)
displayMode.add_argument('--gui', action='store_const', help='Display in text format only.', default=False, const=True)
displayMode.add_argument('--debug', action='store_const', help='Display in text format only.', default=False, const=True)
parser.add_argument('sources', nargs=1, help='File lists the input Questa transcripts to process.')
parser.add_argument('FileName', metavar='FileName', type=str, nargs='?', help='output graph to file <name>.png If not included outputs to screen.', default=None)
args = parser.parse_args()
# Figure what we are reporting
ReportPredictorType = 'BDMR' # default
if(args.ras): ReportPredictorType = 'RASMPR'
if(args.target): ReportPredictorType = 'BTMR'
if(args.iclass): ReportPredictorType = 'ClassMPR'
# Figure how we are displaying the data
ReportMode = 'gui' # default
if(args.text): ReportMode = 'text'
if(args.table): ReportMode = 'table'
if(args.debug): ReportMode = 'debug'
# read the questa sim list file.
# row, col format. each row is a questa sim run with performance counters and a particular
# branch predictor type and size. size can be multiple parameters for more complex predictors like
# local history and tage.
# <file> <type> <size>
predictorLogs = ParseBranchListFile(args.sources[0]) # digests the traces
performanceCounterList = BuildDataBase(predictorLogs) # builds a database of performance counters by trace and then by benchmark
benchmarkFirstList = ReorderDataBase(performanceCounterList) # reorder first by benchmark then trace
benchmarkDict = ExtractSelectedData(benchmarkFirstList) # filters to just the desired performance counter metric
if(args.reference and args.direction): benchmarkDict['Mean'].extend(RefDataBP)
if(args.reference and args.target): benchmarkDict['Mean'].extend(RefDataBTB)
#print(benchmarkDict['Mean'])
#print(benchmarkDict['aha-mont64Speed'])
#print(benchmarkDict)
# table format
if(ReportMode == 'table'):
ReportAsTable(benchmarkDict)
if(ReportMode == 'text'):
ReportAsText(benchmarkDict)
if(ReportMode == 'gui'):
ReportAsGraph(benchmarkDict, args.bar, args.FileName)
# *** this is only needed of -b (no -s)
# debug
#config0 = performanceCounterList[0][0]
#data0 = performanceCounterList[0][1]
#bench0 = data0[0]
#bench0name = bench0[0]
#bench0data = bench0[2]
#bench0BrCount = bench0data['Br Count']
#bench1 = data0[1]
#print(data0)
#print(bench0)
#print(bench1)
#print(bench0name)
#print(bench0BrCount)
Reference in New Issue View Git Blame Copy Permalink