hadoop_sorted/mr/python/wordcount.sh

13 lines
421 B
Bash
Raw Permalink Normal View History

2024-04-05 17:14:07 +02:00
hdfs dfs -mkdir -p tmp/python
hdfs dfs -rm -r tmp/python/output
yarn jar /usr/lib/hadoop/hadoop-streaming.jar \
-D mapred.reduce.tasks=2 \
-input tmp/books \
-output tmp/python/output \
2024-04-05 17:25:06 +02:00
-mapper ~/hadoop_sorted/mr/python/mapper.py \
-reducer ~/hadoop_sorted/mr/python/reducer.py \
-file ~/hadoop_sorted/mr/python/mapper.py \
-file ~/hadoop_sorted/mr/python/reducer.py
2024-04-05 17:14:07 +02:00
hdfs dfs -cat tmp/python/output/part-*