hrucinska/cwiczenia
1
0
Fork 3
Code Issues Pull Requests Releases Wiki Activity

Delete 'pyspark.sh'

Browse Source
This commit is contained in:
hrucinska 2018-05-29 10:53:20 +00:00
parent 3868f4d01b
commit ee8224b1c3
1 changed files with 0 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
pyspark.sh
View File

@ -1,20 +0,0 @@
#!/bin/bash
. /etc/profile
export JAVA_HOME="/usr/lib/jvm/java-8-oracle"
/home/hc_client_bddm_prod/spark-2.3.0-bin-2.6.0-cdh5.6.0/bin/pyspark\
--master yarn\
--deploy-mode client\
--queue bddm_prod_users\
--name "OffersDailyCleanData"\
--num-executors 100\
--conf "spark.driver.memory=1g"\
--conf "spark.yarn.driver.memoryOverhead=600m"\
--conf "spark.executor.memory=2000m"\
--conf "spark.yarn.executor.memoryOverhead=1200m"\
--conf "spark.driver.extraClassPath=/opt/hive_extras/*:/opt/spark_extras/*"\
--conf "spark.yarn.appMasterEnv.JAVA_HOME=/opt/jre1.8.0"\
--conf "spark.executorEnv.JAVA_HOME=/opt/jre1.8.0"\
--packages mysql:mysql-connector-java:5.1.38