hrucinska/cwiczenia
1
0
Fork 3
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: hrucinska:master
Branches Tags
hrucinska:master
szwedek:master
..
compare: szwedek:master
Branches Tags
szwedek:master
hrucinska:master

No commits in common. "master" and "master" have entirely different histories.
master ... master

7 changed files with 42 additions and 18 deletions
Show Stats Expand all files Collapse all files

5
mysql-submit.sh
View File

@ -1,8 +1,5 @@
#!/usr/bin/env bash #!/bin/bash
export HADOOP_CONF_DIR=$HOME/hadoop/conf
export HADOOP_HOME=$HOME/hadoop
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME/lib/native
export PYTHONIOENCODING=utf8 export PYTHONIOENCODING=utf8
/home/faculty/hrucinska/spark/bin/spark-submit\ /home/faculty/hrucinska/spark/bin/spark-submit\

20
pyspark.sh Normal file
View File

@ -0,0 +1,20 @@
#!/bin/bash
. /etc/profile
export JAVA_HOME="/usr/lib/jvm/java-8-oracle"
/home/hc_client_bddm_prod/spark-2.3.0-bin-2.6.0-cdh5.6.0/bin/pyspark\
--master yarn\
--deploy-mode client\
--queue bddm_prod_users\
--name "OffersDailyCleanData"\
--num-executors 100\
--conf "spark.driver.memory=1g"\
--conf "spark.yarn.driver.memoryOverhead=600m"\
--conf "spark.executor.memory=2000m"\
--conf "spark.yarn.executor.memoryOverhead=1200m"\
--conf "spark.driver.extraClassPath=/opt/hive_extras/*:/opt/spark_extras/*"\
--conf "spark.yarn.appMasterEnv.JAVA_HOME=/opt/jre1.8.0"\
--conf "spark.executorEnv.JAVA_HOME=/opt/jre1.8.0"\
--packages mysql:mysql-connector-java:5.1.38

7
spark-submit.sh
View File

@ -1,9 +1,4 @@
#!/usr/bin/env bash #!/bin/bash
export HADOOP_CONF_DIR=$HOME/hadoop/conf
export HADOOP_HOME=$HOME/hadoop
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME/lib/native
export PYTHONIOENCODING=utf8
$HOME/spark/bin/spark-submit\ $HOME/spark/bin/spark-submit\
--master local[*]\ --master local[*]\

19
spark_shell.sh Normal file
View File

@ -0,0 +1,19 @@
#!/bin/bash
. /etc/profile
export JAVA_HOME="/usr/lib/jvm/java-8-oracle"
/home/hc_client_bddm_prod/spark-2.2.0-bin-2.6.0-cdh5.6.0/bin/spark-shell\
--master yarn\
--deploy-mode client\
--queue bddm_prod_users\
--name "OffersDailyCleanData"\
--num-executors 100\
--conf "spark.driver.memory=1g"\
--conf "spark.yarn.driver.memoryOverhead=600m"\
--conf "spark.executor.memory=2000m"\
--conf "spark.yarn.executor.memoryOverhead=1200m"\
--conf "spark.driver.extraClassPath=/opt/hive_extras/*:/opt/spark_extras/*"\
--conf "spark.yarn.appMasterEnv.JAVA_HOME=/opt/jre1.8.0"\
--conf "spark.executorEnv.JAVA_HOME=/opt/jre1.8.0"

1
sparksubmit.sh
View File

@ -4,7 +4,6 @@ export HADOOP_CONF_DIR=$HOME/hadoop/conf
export HADOOP_HOME=$HOME/hadoop export HADOOP_HOME=$HOME/hadoop
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME/lib/native export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME/lib/native
export SPARK_JAVA_OPTS="-Dhdp.version=2.6.3.0-235" export SPARK_JAVA_OPTS="-Dhdp.version=2.6.3.0-235"
export PYTHONIOENCODING=utf8
$HOME/spark/bin/spark-submit\ $HOME/spark/bin/spark-submit\
--master yarn\ --master yarn\

1
sparksubmitcache.sh
View File

@ -4,7 +4,6 @@ export HADOOP_CONF_DIR=$HOME/hadoop/conf
export HADOOP_HOME=$HOME/hadoop export HADOOP_HOME=$HOME/hadoop
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME/lib/native export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME/lib/native
export SPARK_JAVA_OPTS="-Dhdp.version=2.6.3.0-235" export SPARK_JAVA_OPTS="-Dhdp.version=2.6.3.0-235"
export PYTHONIOENCODING=utf8
$HOME/spark/bin/spark-submit\ $HOME/spark/bin/spark-submit\
--master yarn\ --master yarn\

7
too_few_partitions-submit_bad.sh
View File

@ -1,9 +1,4 @@
#!/usr/bin/env bash #!/bin/bash
export HADOOP_CONF_DIR=$HOME/hadoop/conf
export HADOOP_HOME=$HOME/hadoop
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME/lib/native
export PYTHONIOENCODING=utf8
$HOME/spark/bin/spark-submit\ $HOME/spark/bin/spark-submit\