petite-difference-challenge2/main.py

35 lines
968 B
Python
Raw Normal View History

2022-04-25 01:15:46 +02:00
from pyspark.sql import SparkSession
from pyspark.sql.functions import concat, lit, col
from pyspark.sql.types import StructType,StructField, StringType, IntegerType
if __name__ == '__main__':
spark = SparkSession\
.builder\
.appName('He Said She Said')\
.getOrCreate()
X_schema = StructType([StructField('X', StringType(), True)])
X_train = spark.read.csv('train/in.tsv', schema=X_schema)
Y_train = spark.read.csv('train/expected.tsv', schema=StructType([StructField('Y', IntegerType(), True)]))
X_train.show()
train = X_train.join(Y_train, how='full_outer')
train.show()
# # $example on$
# # Load training data
# data = spark.read.format("libsvm") \
# .load("sample_libsvm_data.txt")
# print('data = ', data)
# # Split the data into train and test
# splits = data.randomSplit([0.6, 0.4], 1234)
# train = splits[0]
# test = splits[1]
# print('train = ', train)