23 lines
766 B
Python
23 lines
766 B
Python
|
from pyspark.sql import SparkSession
|
||
|
|
import logging
|
||
|
|
import datetime
|
||
|
|
import time
|
||
|
|
|
||
|
|
spark = SparkSession.builder.appName("mySqlApp").enableHiveSupport().getOrCreate()
|
||
|
|
|
||
|
|
log4jLogger = spark._jvm.org.apache.log4j
|
||
|
|
logger = log4jLogger.LogManager.getLogger(__name__)
|
||
|
|
|
||
|
|
try:
|
||
|
|
logger.info("SPARKAPP START")
|
||
|
|
|
||
|
|
cat = spark.sql("select * from hrucinska.uam_categories")
|
||
|
|
cat.write.mode("overwrite").format("jdbc").options(url ="jdbc:mysql://mysql.wmi.amu.edu.pl/hrucinska_spark_test",driver="com.mysql.jdbc.Driver",user="hrucinska",password="hrucinska", dbtable="uam_categories_mysql").saveAsTable("uam_categories_mysql")
|
||
|
|
|
||
|
|
time.sleep(180)
|
||
|
|
|
||
|
|
except Exception as inst:
|
||
|
|
logger.info("SPARKAPP ERROR {0}".format(inst))
|
||
|
|
finally:
|
||
|
|
logger.info("SPARKAPP STOP")
|