2018-05-29 12:52:27 +02:00
|
|
|
#!/usr/bin/env bash
|
|
|
|
|
|
|
|
|
|
export HADOOP_CONF_DIR=$HOME/hadoop/conf
|
|
|
|
|
export HADOOP_HOME=$HOME/hadoop
|
|
|
|
|
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME/lib/native
|
2018-05-27 12:25:47 +02:00
|
|
|
export PYTHONIOENCODING=utf8
|
|
|
|
|
|
|
|
|
|
/home/faculty/hrucinska/spark/bin/spark-submit\
|
|
|
|
|
--master yarn\
|
|
|
|
|
--deploy-mode client\
|
|
|
|
|
--queue default\
|
|
|
|
|
--name "$1"\
|
|
|
|
|
--num-executors 10\
|
|
|
|
|
--conf "spark.driver.memory=1g"\
|
|
|
|
|
--conf "spark.driver.memoryOverhead=600m"\
|
|
|
|
|
--conf "spark.executor.memory=600m"\
|
|
|
|
|
--conf "spark.executor.memoryOverhead=300m"\
|
|
|
|
|
--conf "spark.sql.autoBroadcastJoinThreshold=-1"\
|
|
|
|
|
--conf "spark.sql.shuffle.partitions=100"\
|
|
|
|
|
--packages mysql:mysql-connector-java:5.1.38 $1
|
