cwiczenia/mysql-submit.sh
Hanna Rucinska d149f7d56e cwiczenia
2018-05-27 12:25:47 +02:00

18 lines
489 B
Bash
Executable File

#!/bin/bash
export PYTHONIOENCODING=utf8
/home/faculty/hrucinska/spark/bin/spark-submit\
--master yarn\
--deploy-mode client\
--queue default\
--name "$1"\
--num-executors 10\
--conf "spark.driver.memory=1g"\
--conf "spark.driver.memoryOverhead=600m"\
--conf "spark.executor.memory=600m"\
--conf "spark.executor.memoryOverhead=300m"\
--conf "spark.sql.autoBroadcastJoinThreshold=-1"\
--conf "spark.sql.shuffle.partitions=100"\
--packages mysql:mysql-connector-java:5.1.38 $1