blas-benchmarks/results.Rmd

---
title: "Results"
author: "Andrzej Wójtowicz"
output: html_document
---


```{r checkpoint, echo=FALSE, warning=FALSE, message=FALSE}
library(checkpoint)
checkpoint("2016-04-01", scanForPackages=FALSE, verbose=FALSE)
library(reshape2)
library(dplyr)
library(ggplot2)
```

```{r config, echo=FALSE}

HOSTNAMES = suppressWarnings(readLines("hosts-list.txt"))

BENCHMARKS = c("urbanek",
               "revolution",
               "gcbd")

LIBRARIES = c("netlib",
              "atlas_st",
              "openblas",
              "atlas_mt",
              "gotoblas2",
              "mkl",
              "blis",
              "cublas")

LIBRARY.REF = "netlib"

HOST.INFO.PATTERN = "host-info-<HOST>.log"
BENCHMKAR.PATTERN = "test-<BENCHMARK>-<HOST>-<LIBRARY>.rds"

RESULTS.DIR = "results"
GEN.DIR     = "gen"
DATA.DIR    = file.path(GEN.DIR, "data")
IMAGES.DIR  = file.path(GEN.DIR, "img")

```

```{r read-write-data, echo=FALSE}

for (d in c(GEN.DIR, DATA.DIR, IMAGES.DIR))
    if (!dir.exists(d))
        dir.create(d)

hosts.info = data.frame(Host=character(), CPU=character(), GPU=character())

for (hostname in HOSTNAMES)
{
    fp = file.path(RESULTS.DIR, gsub("<HOST>", hostname, HOST.INFO.PATTERN))
    
    if (file.exists(fp))
    {
        dr = readLines(fp)
        dr = unname(sapply(dr, trimws))
        
        hosts.info = rbind(hosts.info,
            data.frame(Host= hostname, 
                       CPU = gsub("\\(.*?\\)", "", dr[1]), 
                       GPU = ifelse(length(dr)==3,
                        gsub("(NVIDIA).*\\[(.*)\\]", "\\1 \\2", dr[3]), NA)))
    } else {
        warning(paste("File", fp, "does not exist"))
    }
}

saveRDS(hosts.info, file.path(DATA.DIR, "hosts.info.rds"))


benchmark.results = data.frame(Host=character(), Benchmark=character(), 
                               Library=character(), Test=character(),
                               Runs=integer(), Size=integer(),
                               Time=numeric())

for (host in hosts.info$Host)
{
    for (benchmark in BENCHMARKS)
    {
        for (lib in LIBRARIES)
        {
            if (lib=="cublas" && is.na(hosts.info[which(hosts.info$Host==host),
                                                  "GPU"]))
                next
            
            fp = file.path(RESULTS.DIR, gsub("<HOST>", host, (
                                        gsub("<LIBRARY>", lib, ( 
                                        gsub("<BENCHMARK>", benchmark, BENCHMKAR.PATTERN)))))
                           )
            
            if (file.exists(fp))
            {
                db = readRDS(fp)
                
                runs = NA
                if (length(attr(db, "runs"))==1) {
                    runs = attr(db, "runs")
                } else {
                    # gcbd
                    db = melt(cbind(db, runs=attr(db, "runs"), 
                                    size=attr(db, "size")), 
                              id.vars=c("runs", "size"), 
                              variable.name="name", 
                              value.name="time")
                    runs = db$runs
                }
                
                size = NA
                if ("size" %in% colnames(db)) # gcbd
                    size = db$size
                
                benchmark.results = rbind(benchmark.results,
                    data.frame(Host=host, Benchmark=benchmark, 
                               Library=lib, Test=db$name, 
                               Runs=runs, Size=size,
                               Time=db$time))
            } else {
                warning(paste("File", fp, "does not exist"))
            }
        }
    }
}

saveRDS(benchmark.results, file.path(DATA.DIR, "benchmark.results.rds"))

```

```{r per-host, echo=FALSE, results='asis'}

cat("# Per host\n\n")

for (host in hosts.info$Host)
{
    host.cpu = hosts.info[which(hosts.info$Host==host), "CPU"]
    host.gpu = hosts.info[which(hosts.info$Host==host), "GPU"]
    
    cat(paste("##", host.cpu))
    
    if (!is.na(host.gpu))
        cat(paste(" +", host.gpu))
    
    cat("\n\n")
    
    for (benchmark in BENCHMARKS)
    {
        if (benchmark != "urbanek")
            next
        
        testnames = benchmark.results %>% 
            filter(Benchmark == benchmark) %>% 
            select(Test) %>% 
            unique
        
        #browser()
        
        if (benchmark == "urbanek")
            # remove tests with almost the same results
            testnames = testnames %>% filter(row_number() %in% c(4, 5, 7, 8, 9, 10, 15))
        
        
        for (i in 1:nrow(testnames)) 
        {
            test = as.character(testnames[i, "Test"])

            data.to.plot = benchmark.results %>% 
                filter(Host == host & Benchmark == benchmark &
                       Test == test)
            
            data.to.plot$Library = factor(data.to.plot$Library,
                levels=rev(levels(data.to.plot$Library)))
            
            image.path = file.path(IMAGES.DIR,
               paste0("img_ph", 
                      "_h", which(host == hosts.info$Host),
                      "_b", which(benchmark == BENCHMARKS),
                      "_t", i,
                      ".png"))
            
            png(image.path, width=500, height=50*nrow(data.to.plot))
                print(ggplot(data.to.plot, aes(x=Library, y=Time)) +
                    theme_classic() + 
                    theme(
                        panel.border = element_blank()
                    ) +
                    scale_y_continuous(expand = c(0, 0), 
                        limits = c(0, 1.12*max(data.to.plot$Time))) +
                    ggtitle(test) + 
                    ylab("Seconds (performance gain)") +
                    geom_bar(stat="identity", width=.75) + 
                    geom_text(aes(label=round(Time, 2)), hjust=-0.25) +
                    coord_flip() + 
                    geom_hline(yintercept = 0) + 
                    geom_vline(xintercept = 0.39)
                ) 
            dev.off()
            
            cat(paste0("![](", image.path, ")\n\n"))
        }
        
        
    }
}

```