#!/usr/bin/env bash

export HADOOP_CONF_DIR=$HOME/hadoop/conf
export HADOOP_HOME=$HOME/hadoop
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME/lib/native
export PYTHONIOENCODING=utf8

/home/faculty/hrucinska/spark/bin/spark-submit\
 --master yarn\
 --deploy-mode client\
 --queue default\
 --name "$1"\
 --num-executors 10\
 --conf "spark.driver.memory=1g"\
 --conf "spark.driver.memoryOverhead=600m"\
 --conf "spark.executor.memory=600m"\
 --conf "spark.executor.memoryOverhead=300m"\
 --conf "spark.sql.autoBroadcastJoinThreshold=-1"\
 --conf "spark.sql.shuffle.partitions=100"\
 --packages mysql:mysql-connector-java:5.1.38 $1