andre/blas-benchmarks
1
0
Fork 0
mirror of https://github.com/andre-wojtowicz/blas-benchmarks synced 2024-11-03 19:45:27 +01:00
Code Releases Activity
5ec7e7f9fa
blas-benchmarks/results.Rmd

323 lines
12 KiB
Plaintext
Raw Blame History

---
title: "Results"
author: "Andrzej Wójtowicz"
output: html_document
---
```{r checkpoint, echo=FALSE, warning=FALSE, message=FALSE}
library(checkpoint)
checkpoint("2016-04-01", scanForPackages=FALSE, verbose=FALSE)
library(reshape2)
library(dplyr)
library(ggplot2)
library(RColorBrewer)
```
```{r config, echo=FALSE}
HOSTNAMES = suppressWarnings(readLines("hosts-list.txt"))
BENCHMARKS = c("urbanek",
"revolution",
"gcbd")
LIBRARIES = c("netlib",
"atlas_st",
"openblas",
"atlas_mt",
"gotoblas2",
"mkl",
"blis",
"cublas")
LIBRARY.REF = "netlib"
HOST.INFO.PATTERN = "host-info-<HOST>.log"
BENCHMKAR.PATTERN = "test-<BENCHMARK>-<HOST>-<LIBRARY>.rds"
RESULTS.DIR = "results"
GEN.DIR = "gen"
DATA.DIR = file.path(GEN.DIR, "data")
IMAGES.DIR = file.path(GEN.DIR, "img")
```
```{r read-write-data, echo=FALSE}
for (d in c(GEN.DIR, DATA.DIR, IMAGES.DIR))
if (!dir.exists(d))
dir.create(d)
hosts.info = data.frame(Host=character(), CPU=character(), GPU=character())
for (hostname in HOSTNAMES)
{
fp = file.path(RESULTS.DIR, gsub("<HOST>", hostname, HOST.INFO.PATTERN))
if (file.exists(fp))
{
dr = readLines(fp)
dr = unname(sapply(dr, trimws))
hosts.info = rbind(hosts.info,
data.frame(Host= hostname,
CPU = gsub("\\(.*?\\)", "", dr[1]),
GPU = ifelse(length(dr)==3,
gsub("(NVIDIA).*\\[(.*)\\]", "\\1 \\2", dr[3]), NA)))
} else {
warning(paste("File", fp, "does not exist"))
}
}
saveRDS(hosts.info, file.path(DATA.DIR, "hosts.info.rds"))
benchmark.results = data.frame(Host=character(), Benchmark=character(),
Library=character(), Test=character(),
Runs=integer(), Size=integer(),
Time=numeric())
for (host in hosts.info$Host)
{
for (benchmark in BENCHMARKS)
{
for (lib in LIBRARIES)
{
if (lib=="cublas" && is.na(hosts.info[which(hosts.info$Host==host),
"GPU"]))
next
fp = file.path(RESULTS.DIR, gsub("<HOST>", host, (
gsub("<LIBRARY>", lib, (
gsub("<BENCHMARK>", benchmark, BENCHMKAR.PATTERN)))))
)
if (file.exists(fp))
{
db = readRDS(fp)
runs = NA
if (length(attr(db, "runs"))==1) {
runs = attr(db, "runs")
} else {
# gcbd
db = melt(cbind(db, runs=attr(db, "runs"),
size=attr(db, "size")),
id.vars=c("runs", "size"),
variable.name="name",
value.name="time")
runs = db$runs
}
size = NA
if ("size" %in% colnames(db)) # gcbd
size = db$size
benchmark.results = rbind(benchmark.results,
data.frame(Host=host, Benchmark=benchmark,
Library=lib, Test=db$name,
Runs=runs, Size=size,
Time=db$time))
} else {
warning(paste("File", fp, "does not exist"))
}
}
}
}
saveRDS(benchmark.results, file.path(DATA.DIR, "benchmark.results.rds"))
```
```{r per-host, echo=FALSE, results='asis'}
cat("# Per host\n\n")
for (host in hosts.info$Host)
{
host.cpu = hosts.info[which(hosts.info$Host==host), "CPU"]
host.gpu = hosts.info[which(hosts.info$Host==host), "GPU"]
cat(paste("##", host.cpu))
if (!is.na(host.gpu))
cat(paste(" +", host.gpu))
cat("\n\n\n\n")
for (benchmark in BENCHMARKS)
{
cat(paste0("## ", paste(toupper(substring(benchmark, 1,1)), substring(benchmark, 2),sep="", collapse=" "), " benchmark\n\n"))
testnames = benchmark.results %>%
filter(Benchmark == benchmark) %>%
select(Test) %>%
unique
if (benchmark == "urbanek")
# remove tests with almost the same results
testnames = testnames %>% filter(row_number() %in% c(4, 5, 7, 8, 9, 10, 15))
for (i in 1:nrow(testnames))
{
test = as.character(testnames[i, "Test"])
data.to.plot = benchmark.results %>%
filter(Host == host & Benchmark == benchmark &
Test == test)
if (benchmark != "gcbd")
data.to.plot = data.to.plot %>%
mutate(PerfGain=data.to.plot[which(data.to.plot$Library==LIBRARY.REF), "Time"]/Time)
renamedLibs = list(
"cuBLAS"="cublas",
"BLIS"="blis",
"MKL"="mkl",
"ATLAS (mt)"="atlas_mt",
"OpenBLAS"="openblas",
"GotoBLAS2"="gotoblas2",
"ATLAS (st)"="atlas_st",
"Netlib"="netlib")
if (benchmark != "gcbd")
levels(data.to.plot$Library) = renamedLibs
else
levels(data.to.plot$Library) = rev(renamedLibs)
data.to.plot = data.to.plot %>%
mutate(Color=sapply(Library, function(x){
if(x=="Netlib") "A"
else if(x=="ATLAS (st)") "B"
else if(x=="cuBLAS") "D"
else "C"}))
image.path = file.path(IMAGES.DIR,
paste0("img_ph",
"_h", which(host == hosts.info$Host),
"_b", which(benchmark == BENCHMARKS),
"_t", i,
".png"))
cat(paste0("#### ", gsub(" \\(.*\\)", "", test)), "\n\n")
cat(paste0("Time in seconds ", ifelse(benchmark=="gcbd", "regarding matrix size - right panel on log scale", ""), " - ", ifelse(benchmark!="gcbd", data.to.plot$Runs[1], paste(" from ", max(data.to.plot$Runs), " to ", min(data.to.plot$Runs))),
" runs - lower is better\n\n"))
myColors = c("#E6191A", "#F25F1E", "#636363", "#5AAC45")
names(myColors) = levels(data.to.plot$Color)
colScale = scale_fill_manual(name = "Color", values = myColors)
#if (benchmark=="gcbd")
# browser()
if (benchmark != "gcbd")
{
png(image.path, width=600, height=35*nrow(data.to.plot))
print(ggplot(data.to.plot, aes(x=reorder(Library, -Time), y=Time, fill=Color)) +
theme_classic() +
theme(
panel.border = element_blank(),
axis.title.x=element_blank(),
axis.title.y=element_blank(),
axis.text.x = element_text(colour="grey60"),
axis.ticks.x=element_line(color="grey60"),
axis.ticks.y=element_line(color="grey60"),
legend.position="none"
) +
scale_y_continuous(expand = c(0, 0),
limits = c(0, 1.20*max(data.to.plot$Time))) +
geom_bar(stat="identity", width=.7) +
coord_flip() +
geom_text(aes(y=Time, x=Library, label=format(Time, digits=2, nsmall=2)), hjust=-0.25, size=3, colour="grey45") +
geom_text(aes(y=1.2*max(data.to.plot$Time), x=Library, label=paste0("x", trimws(format(PerfGain, digits=1, nsmall=1)))), hjust=1, size=4) +
geom_hline(yintercept = 0, color="grey") +
geom_vline(xintercept = 0.4, color="grey") +
annotate("text", x=nrow(data.to.plot)+0.55, y=1.2*max(data.to.plot$Time), label="Performance gain", hjust=1) +
colScale
)
dev.off()
cat(paste0("![](", image.path, ")\n\n\n\n"))
} else {
rbColors = c("#F40000","#FF8000","#0094FF","#E900FF","#B35807","#7F00FF","#FFBC70","#00DD0E")
png(image.path, width=400, height=350, type="cairo")
print(ggplot(data.to.plot,
aes(x=Size, y=Time, group=Library, color=Library,
shape=Library, fill=Library)) +
theme_classic() +
theme(
panel.border = element_blank(),
axis.title.x=element_blank(),
axis.title.y=element_blank(),
axis.text.x = element_text(colour="grey60"),
axis.text.y = element_text(colour="grey60"),
axis.ticks.x=element_line(color="grey60"),
axis.ticks.y=element_line(color="grey60"),
legend.margin=unit(c(-13,8), "mm"),
legend.title=element_blank()
) +
geom_line(size=1) +
geom_point(size=2, color="grey30") +
scale_x_continuous(expand = c(0, 0),
limits = c(0, 5000)) +
scale_y_continuous(expand = c(0, 0)) +
coord_cartesian(ylim=c(0, max(20, max(data.to.plot$Time)/3))) +
geom_hline(yintercept = 0, color="grey") +
geom_vline(xintercept = 0.4, color="grey") +
scale_shape_manual(values=c(21,24,22,25,23,21,22,23)) +
scale_color_manual(values=rbColors) +
scale_fill_manual(values=rbColors)
)
dev.off()
cat(paste0("![](", image.path, ")"))
png(gsub(".png", "b.png", image.path), width=350, height=350, type="cairo")
print(ggplot(data.to.plot,
aes(x=Size, y=Time, group=Library, color=Library,
shape=Library, fill=Library)) +
theme_classic() +
theme(
panel.border = element_blank(),
axis.title.x=element_blank(),
axis.title.y=element_blank(),
axis.text.x = element_text(colour="grey60"),
axis.text.y = element_text(colour="grey60"),
axis.ticks.x=element_line(color="grey60"),
axis.ticks.y=element_line(color="grey60"),
legend.position="none"
) +
geom_line(size=1) +
geom_point(size=2, color="grey30") +
scale_x_log10(expand=c(0,0),
breaks=c(100, 200, 400, 800, 1600, 3200, 5000),
limits=c(100, 5500)) +
scale_y_log10() +
scale_shape_manual(values=c(21,24,22,25,23,21,22,23)) +
geom_hline(yintercept = 0, color="grey") +
geom_vline(xintercept = min(data.to.plot$Size), color="grey") +
scale_color_manual(values=rbColors) +
scale_fill_manual(values=rbColors)
)
dev.off()
cat(paste0("![](", gsub(".png", "b.png", image.path), ")\n\n\n\n"))
}
}
}
}
```
View Git Blame Copy Permalink