14 lines
349 B
Bash
14 lines
349 B
Bash
#!/usr/bin/env bash
|
|
|
|
export HADOOP_CONF_DIR=$HOME/hadoop/conf
|
|
export HADOOP_HOME=$HOME/hadoop
|
|
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME/lib/native
|
|
export PYTHONIOENCODING=utf8
|
|
|
|
|
|
$HOME/spark/bin/spark-submit\
|
|
--master local[10]\
|
|
--name "$1"\
|
|
--conf "spark.sql.autoBroadcastJoinThreshold=-1"\
|
|
--conf "spark.sql.shuffle.partitions=2" $1
|