--- title: "Results" author: "Andrzej Wójtowicz" output: html_document --- ```{r checkpoint, echo=FALSE, warning=FALSE, message=FALSE} library(checkpoint) checkpoint("2016-04-01", scanForPackages=FALSE, verbose=FALSE) library(reshape2) library(dplyr) library(ggplot2) library(RColorBrewer) ``` ```{r config, echo=FALSE} HOSTNAMES = suppressWarnings(readLines("hosts-list.txt")) BENCHMARKS = c("urbanek", "revolution", "gcbd") LIBRARIES = c("netlib", "atlas_st", "openblas", "atlas_mt", "gotoblas2", "mkl", "blis", "cublas") LIBRARY.REF = "netlib" HOST.INFO.PATTERN = "host-info-.log" BENCHMKAR.PATTERN = "test---.rds" RESULTS.DIR = "results" GEN.DIR = "gen" DATA.DIR = file.path(GEN.DIR, "data") IMAGES.DIR = file.path(GEN.DIR, "img") ``` ```{r read-write-data, echo=FALSE} for (d in c(GEN.DIR, DATA.DIR, IMAGES.DIR)) if (!dir.exists(d)) dir.create(d) hosts.info = data.frame(Host=character(), CPU=character(), GPU=character()) for (hostname in HOSTNAMES) { fp = file.path(RESULTS.DIR, gsub("", hostname, HOST.INFO.PATTERN)) if (file.exists(fp)) { dr = readLines(fp) dr = unname(sapply(dr, trimws)) hosts.info = rbind(hosts.info, data.frame(Host= hostname, CPU = gsub("\\(.*?\\)", "", dr[1]), GPU = ifelse(length(dr)==3, gsub("(NVIDIA).*\\[(.*)\\]", "\\1 \\2", dr[3]), NA))) } else { warning(paste("File", fp, "does not exist")) } } saveRDS(hosts.info, file.path(DATA.DIR, "hosts.info.rds")) benchmark.results = data.frame(Host=character(), Benchmark=character(), Library=character(), Test=character(), Runs=integer(), Size=integer(), Time=numeric()) for (host in hosts.info$Host) { for (benchmark in BENCHMARKS) { for (lib in LIBRARIES) { if (lib=="cublas" && is.na(hosts.info[which(hosts.info$Host==host), "GPU"])) next fp = file.path(RESULTS.DIR, gsub("", host, ( gsub("", lib, ( gsub("", benchmark, BENCHMKAR.PATTERN))))) ) if (file.exists(fp)) { db = readRDS(fp) runs = NA if (length(attr(db, "runs"))==1) { runs = attr(db, "runs") } else { # gcbd db = melt(cbind(db, runs=attr(db, "runs"), size=attr(db, "size")), id.vars=c("runs", "size"), variable.name="name", value.name="time") runs = db$runs } size = NA if ("size" %in% colnames(db)) # gcbd size = db$size benchmark.results = rbind(benchmark.results, data.frame(Host=host, Benchmark=benchmark, Library=lib, Test=db$name, Runs=runs, Size=size, Time=db$time)) } else { warning(paste("File", fp, "does not exist")) } } } } saveRDS(benchmark.results, file.path(DATA.DIR, "benchmark.results.rds")) ``` ```{r per-host, echo=FALSE, results='asis'} cat("# Per host\n\n") for (host in hosts.info$Host) { host.cpu = hosts.info[which(hosts.info$Host==host), "CPU"] host.gpu = hosts.info[which(hosts.info$Host==host), "GPU"] cat(paste("##", host.cpu)) if (!is.na(host.gpu)) cat(paste(" +", host.gpu)) cat("\n\n\n\n") for (benchmark in BENCHMARKS) { cat(paste0("## ", paste(toupper(substring(benchmark, 1,1)), substring(benchmark, 2),sep="", collapse=" "), " benchmark\n\n")) testnames = benchmark.results %>% filter(Benchmark == benchmark) %>% select(Test) %>% unique if (benchmark == "urbanek") # remove tests with almost the same results testnames = testnames %>% filter(row_number() %in% c(4, 5, 7, 8, 9, 10, 15)) for (i in 1:nrow(testnames)) { test = as.character(testnames[i, "Test"]) data.to.plot = benchmark.results %>% filter(Host == host & Benchmark == benchmark & Test == test) if (benchmark != "gcbd") data.to.plot = data.to.plot %>% mutate(PerfGain=data.to.plot[which(data.to.plot$Library==LIBRARY.REF), "Time"]/Time) renamedLibs = list( "cuBLAS"="cublas", "BLIS"="blis", "MKL"="mkl", "ATLAS (mt)"="atlas_mt", "OpenBLAS"="openblas", "GotoBLAS2"="gotoblas2", "ATLAS (st)"="atlas_st", "Netlib"="netlib") if (benchmark != "gcbd") levels(data.to.plot$Library) = renamedLibs else levels(data.to.plot$Library) = rev(renamedLibs) data.to.plot = data.to.plot %>% mutate(Color=sapply(Library, function(x){ if(x=="Netlib") "A" else if(x=="ATLAS (st)") "B" else if(x=="cuBLAS") "D" else "C"})) image.path = file.path(IMAGES.DIR, paste0("img_ph", "_h", which(host == hosts.info$Host), "_b", which(benchmark == BENCHMARKS), "_t", i, ".png")) cat(paste0("#### ", gsub(" \\(.*\\)", "", test)), "\n\n") cat(paste0("Time in seconds ", ifelse(benchmark=="gcbd", "regarding matrix size - right panel on log scale", ""), " - ", ifelse(benchmark!="gcbd", data.to.plot$Runs[1], paste(" from ", max(data.to.plot$Runs), " to ", min(data.to.plot$Runs))), " runs - lower is better\n\n")) myColors = c("#E6191A", "#F25F1E", "#636363", "#5AAC45") names(myColors) = levels(data.to.plot$Color) colScale = scale_fill_manual(name = "Color", values = myColors) #if (benchmark=="gcbd") # browser() if (benchmark != "gcbd") { png(image.path, width=600, height=35*nrow(data.to.plot)) print(ggplot(data.to.plot, aes(x=reorder(Library, -Time), y=Time, fill=Color)) + theme_classic() + theme( panel.border = element_blank(), axis.title.x=element_blank(), axis.title.y=element_blank(), axis.text.x = element_text(colour="grey60"), axis.ticks.x=element_line(color="grey60"), axis.ticks.y=element_line(color="grey60"), legend.position="none" ) + scale_y_continuous(expand = c(0, 0), limits = c(0, 1.20*max(data.to.plot$Time))) + geom_bar(stat="identity", width=.7) + coord_flip() + geom_text(aes(y=Time, x=Library, label=format(Time, digits=2, nsmall=2)), hjust=-0.25, size=3, colour="grey45") + geom_text(aes(y=1.2*max(data.to.plot$Time), x=Library, label=paste0("x", trimws(format(PerfGain, digits=1, nsmall=1)))), hjust=1, size=4) + geom_hline(yintercept = 0, color="grey") + geom_vline(xintercept = 0.4, color="grey") + annotate("text", x=nrow(data.to.plot)+0.55, y=1.2*max(data.to.plot$Time), label="Performance gain", hjust=1) + colScale ) dev.off() cat(paste0("![](", image.path, ")\n\n\n\n")) } else { rbColors = c("#F40000","#FF8000","#0094FF","#E900FF","#B35807","#7F00FF","#FFBC70","#00DD0E") png(image.path, width=400, height=350, type="cairo") print(ggplot(data.to.plot, aes(x=Size, y=Time, group=Library, color=Library, shape=Library, fill=Library)) + theme_classic() + theme( panel.border = element_blank(), axis.title.x=element_blank(), axis.title.y=element_blank(), axis.text.x = element_text(colour="grey60"), axis.text.y = element_text(colour="grey60"), axis.ticks.x=element_line(color="grey60"), axis.ticks.y=element_line(color="grey60"), legend.margin=unit(c(-13,8), "mm"), legend.title=element_blank() ) + geom_line(size=1) + geom_point(size=2, color="grey30") + scale_x_continuous(expand = c(0, 0), limits = c(0, 5000)) + scale_y_continuous(expand = c(0, 0)) + coord_cartesian(ylim=c(0, max(20, max(data.to.plot$Time)/3))) + geom_hline(yintercept = 0, color="grey") + geom_vline(xintercept = 0.4, color="grey") + scale_shape_manual(values=c(21,24,22,25,23,21,22,23)) + scale_color_manual(values=rbColors) + scale_fill_manual(values=rbColors) ) dev.off() cat(paste0("![](", image.path, ")")) png(gsub(".png", "b.png", image.path), width=350, height=350, type="cairo") print(ggplot(data.to.plot, aes(x=Size, y=Time, group=Library, color=Library, shape=Library, fill=Library)) + theme_classic() + theme( panel.border = element_blank(), axis.title.x=element_blank(), axis.title.y=element_blank(), axis.text.x = element_text(colour="grey60"), axis.text.y = element_text(colour="grey60"), axis.ticks.x=element_line(color="grey60"), axis.ticks.y=element_line(color="grey60"), legend.position="none" ) + geom_line(size=1) + geom_point(size=2, color="grey30") + scale_x_log10(expand=c(0,0), breaks=c(100, 200, 400, 800, 1600, 3200, 5000), limits=c(100, 5500)) + scale_y_log10() + scale_shape_manual(values=c(21,24,22,25,23,21,22,23)) + geom_hline(yintercept = 0, color="grey") + geom_vline(xintercept = min(data.to.plot$Size), color="grey") + scale_color_manual(values=rbColors) + scale_fill_manual(values=rbColors) ) dev.off() cat(paste0("![](", gsub(".png", "b.png", image.path), ")\n\n\n\n")) } } } } ```