mirror of
https://github.com/andre-wojtowicz/blas-benchmarks
synced 2024-08-25 00:01:56 +02:00
229 lines
7.3 KiB
Plaintext
229 lines
7.3 KiB
Plaintext
---
|
|
title: "Results"
|
|
author: "Andrzej Wójtowicz"
|
|
output: html_document
|
|
---
|
|
|
|
|
|
```{r checkpoint, echo=FALSE, warning=FALSE, message=FALSE}
|
|
library(checkpoint)
|
|
checkpoint("2016-04-01", scanForPackages=FALSE, verbose=FALSE)
|
|
library(reshape2)
|
|
library(dplyr)
|
|
library(ggplot2)
|
|
```
|
|
|
|
```{r config, echo=FALSE}
|
|
|
|
HOSTNAMES = suppressWarnings(readLines("hosts-list.txt"))
|
|
|
|
BENCHMARKS = c("urbanek",
|
|
"revolution",
|
|
"gcbd")
|
|
|
|
LIBRARIES = c("netlib",
|
|
"atlas_st",
|
|
"openblas",
|
|
"atlas_mt",
|
|
"gotoblas2",
|
|
"mkl",
|
|
"blis",
|
|
"cublas")
|
|
|
|
LIBRARY.REF = "netlib"
|
|
|
|
HOST.INFO.PATTERN = "host-info-<HOST>.log"
|
|
BENCHMKAR.PATTERN = "test-<BENCHMARK>-<HOST>-<LIBRARY>.rds"
|
|
|
|
RESULTS.DIR = "results"
|
|
GEN.DIR = "gen"
|
|
DATA.DIR = file.path(GEN.DIR, "data")
|
|
IMAGES.DIR = file.path(GEN.DIR, "img")
|
|
|
|
```
|
|
|
|
```{r read-write-data, echo=FALSE}
|
|
|
|
for (d in c(GEN.DIR, DATA.DIR, IMAGES.DIR))
|
|
if (!dir.exists(d))
|
|
dir.create(d)
|
|
|
|
hosts.info = data.frame(Host=character(), CPU=character(), GPU=character())
|
|
|
|
for (hostname in HOSTNAMES)
|
|
{
|
|
fp = file.path(RESULTS.DIR, gsub("<HOST>", hostname, HOST.INFO.PATTERN))
|
|
|
|
if (file.exists(fp))
|
|
{
|
|
dr = readLines(fp)
|
|
dr = unname(sapply(dr, trimws))
|
|
|
|
hosts.info = rbind(hosts.info,
|
|
data.frame(Host= hostname,
|
|
CPU = gsub("\\(.*?\\)", "", dr[1]),
|
|
GPU = ifelse(length(dr)==3,
|
|
gsub("(NVIDIA).*\\[(.*)\\]", "\\1 \\2", dr[3]), NA)))
|
|
} else {
|
|
warning(paste("File", fp, "does not exist"))
|
|
}
|
|
}
|
|
|
|
saveRDS(hosts.info, file.path(DATA.DIR, "hosts.info.rds"))
|
|
|
|
|
|
benchmark.results = data.frame(Host=character(), Benchmark=character(),
|
|
Library=character(), Test=character(),
|
|
Runs=integer(), Size=integer(),
|
|
Time=numeric())
|
|
|
|
for (host in hosts.info$Host)
|
|
{
|
|
for (benchmark in BENCHMARKS)
|
|
{
|
|
for (lib in LIBRARIES)
|
|
{
|
|
if (lib=="cublas" && is.na(hosts.info[which(hosts.info$Host==host),
|
|
"GPU"]))
|
|
next
|
|
|
|
fp = file.path(RESULTS.DIR, gsub("<HOST>", host, (
|
|
gsub("<LIBRARY>", lib, (
|
|
gsub("<BENCHMARK>", benchmark, BENCHMKAR.PATTERN)))))
|
|
)
|
|
|
|
if (file.exists(fp))
|
|
{
|
|
db = readRDS(fp)
|
|
|
|
runs = NA
|
|
if (length(attr(db, "runs"))==1) {
|
|
runs = attr(db, "runs")
|
|
} else {
|
|
# gcbd
|
|
db = melt(cbind(db, runs=attr(db, "runs"),
|
|
size=attr(db, "size")),
|
|
id.vars=c("runs", "size"),
|
|
variable.name="name",
|
|
value.name="time")
|
|
runs = db$runs
|
|
}
|
|
|
|
size = NA
|
|
if ("size" %in% colnames(db)) # gcbd
|
|
size = db$size
|
|
|
|
benchmark.results = rbind(benchmark.results,
|
|
data.frame(Host=host, Benchmark=benchmark,
|
|
Library=lib, Test=db$name,
|
|
Runs=runs, Size=size,
|
|
Time=db$time))
|
|
} else {
|
|
warning(paste("File", fp, "does not exist"))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
saveRDS(benchmark.results, file.path(DATA.DIR, "benchmark.results.rds"))
|
|
|
|
```
|
|
|
|
```{r per-host, echo=FALSE, results='asis'}
|
|
|
|
cat("# Per host\n\n")
|
|
|
|
for (host in hosts.info$Host)
|
|
{
|
|
host.cpu = hosts.info[which(hosts.info$Host==host), "CPU"]
|
|
host.gpu = hosts.info[which(hosts.info$Host==host), "GPU"]
|
|
|
|
cat(paste("##", host.cpu))
|
|
|
|
if (!is.na(host.gpu))
|
|
cat(paste(" +", host.gpu))
|
|
|
|
cat("\n\n\n\n")
|
|
|
|
for (benchmark in BENCHMARKS)
|
|
{
|
|
if (benchmark == "gcbd")
|
|
next
|
|
|
|
cat(paste0("## ", paste(toupper(substring(benchmark, 1,1)), substring(benchmark, 2),sep="", collapse=" "), " benchmark\n\n"))
|
|
|
|
testnames = benchmark.results %>%
|
|
filter(Benchmark == benchmark) %>%
|
|
select(Test) %>%
|
|
unique
|
|
|
|
#browser()
|
|
|
|
if (benchmark == "urbanek")
|
|
# remove tests with almost the same results
|
|
testnames = testnames %>% filter(row_number() %in% c(4, 5, 7, 8, 9, 10, 15))
|
|
|
|
|
|
for (i in 1:nrow(testnames))
|
|
{
|
|
test = as.character(testnames[i, "Test"])
|
|
|
|
data.to.plot = benchmark.results %>%
|
|
filter(Host == host & Benchmark == benchmark &
|
|
Test == test)
|
|
|
|
data.to.plot = data.to.plot %>%
|
|
mutate(PerfGain=data.to.plot[which(data.to.plot$Library==LIBRARY.REF), "Time"]/Time)
|
|
|
|
levels(data.to.plot$Library) = list(
|
|
"cuBLAS"="cublas",
|
|
"BLIS"="blis",
|
|
"MKL"="mkl",
|
|
"ATLAS (mt)"="atlas_mt",
|
|
"OpenBLAS"="openblas",
|
|
"GotoBLAS2"="gotoblas2",
|
|
"ATLAS (st)"="atlas_st",
|
|
"Netlib"="netlib")
|
|
|
|
image.path = file.path(IMAGES.DIR,
|
|
paste0("img_ph",
|
|
"_h", which(host == hosts.info$Host),
|
|
"_b", which(benchmark == BENCHMARKS),
|
|
"_t", i,
|
|
".png"))
|
|
|
|
cat(paste0("#### ", gsub(" \\(.*\\)", "", test)), "\n\n")
|
|
cat("Time in seconds - lower is better\n\n")
|
|
|
|
png(image.path, width=600, height=50*nrow(data.to.plot))
|
|
print(ggplot(data.to.plot, aes(x=Library, y=Time)) +
|
|
theme_classic() +
|
|
theme(
|
|
panel.border = element_blank(),
|
|
axis.title.x=element_blank(),
|
|
axis.title.y=element_blank(),
|
|
axis.text.x = element_text(colour="grey60"),
|
|
axis.ticks.x=element_line(color="grey60"),
|
|
axis.ticks.y=element_line(color="grey60")
|
|
) +
|
|
scale_y_continuous(expand = c(0, 0),
|
|
limits = c(0, 1.20*max(data.to.plot$Time))) +
|
|
geom_bar(stat="identity", width=.75) +
|
|
coord_flip() +
|
|
geom_text(aes(y=Time, x=Library, label=as.character(round(Time, 2))), hjust=-0.25, size=3, colour="grey45") +
|
|
geom_text(aes(y=1.15*max(data.to.plot$Time), x=Library, label=paste0("x", round(PerfGain, 1))), hjust=1, size=4) +
|
|
geom_hline(yintercept = 0, color="grey") +
|
|
geom_vline(xintercept = 0.4, color="grey") +
|
|
annotate("text", x=nrow(data.to.plot)/2, y=1.19*max(data.to.plot$Time), label="Performance gain", angle=-90, hjust=0.6)
|
|
)
|
|
dev.off()
|
|
|
|
cat(paste0("![](", image.path, ")\n\n\n\n"))
|
|
}
|
|
|
|
|
|
}
|
|
}
|
|
|
|
```
|