35 lines
968 B
Python
35 lines
968 B
Python
|
from pyspark.sql import SparkSession
|
||
|
|
from pyspark.sql.functions import concat, lit, col
|
||
|
|
from pyspark.sql.types import StructType,StructField, StringType, IntegerType
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
spark = SparkSession\
|
||
|
|
.builder\
|
||
|
|
.appName('He Said She Said')\
|
||
|
|
.getOrCreate()
|
||
|
|
|
||
|
|
X_schema = StructType([StructField('X', StringType(), True)])
|
||
|
|
|
||
|
|
X_train = spark.read.csv('train/in.tsv', schema=X_schema)
|
||
|
|
Y_train = spark.read.csv('train/expected.tsv', schema=StructType([StructField('Y', IntegerType(), True)]))
|
||
|
|
|
||
|
|
X_train.show()
|
||
|
|
|
||
|
|
train = X_train.join(Y_train, how='full_outer')
|
||
|
|
train.show()
|
||
|
|
|
||
|
|
# # $example on$
|
||
|
|
# # Load training data
|
||
|
|
# data = spark.read.format("libsvm") \
|
||
|
|
# .load("sample_libsvm_data.txt")
|
||
|
|
|
||
|
|
# print('data = ', data)
|
||
|
|
|
||
|
|
# # Split the data into train and test
|
||
|
|
# splits = data.randomSplit([0.6, 0.4], 1234)
|
||
|
|
# train = splits[0]
|
||
|
|
# test = splits[1]
|
||
|
|
|
||
|
|
# print('train = ', train)
|