#!/bin/bash

. /etc/profile

export JAVA_HOME="/usr/lib/jvm/java-8-oracle"

/home/hc_client_bddm_prod/spark-2.3.0-bin-2.6.0-cdh5.6.0/bin/pyspark\
 --master yarn\
 --deploy-mode client\
 --queue bddm_prod_users\
 --name "OffersDailyCleanData"\
 --num-executors 100\
 --conf "spark.driver.memory=1g"\
 --conf "spark.yarn.driver.memoryOverhead=600m"\
 --conf "spark.executor.memory=2000m"\
 --conf "spark.yarn.executor.memoryOverhead=1200m"\
 --conf "spark.driver.extraClassPath=/opt/hive_extras/*:/opt/spark_extras/*"\
 --conf "spark.yarn.appMasterEnv.JAVA_HOME=/opt/jre1.8.0"\
 --conf "spark.executorEnv.JAVA_HOME=/opt/jre1.8.0"\
 --packages mysql:mysql-connector-java:5.1.38