当前位置 : 主页 > 编程语言 > c++ >

spark版的WordCount(Java,jdk1.8)

来源:互联网 收集:自由互联 发布时间:2021-07-03
WordCount import java.util.Arrays;import org.apache.spark.api.java.JavaPairRDD;import org.apache.spark.api.java.JavaRDD;import org.apache.spark.sql.SparkSession;import scala.Tuple2;public class WordCount3 {public static void main(String[] a
WordCount
import java.util.Arrays;

import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.sql.SparkSession;

import scala.Tuple2;

public class WordCount3 {

	public static void main(String[] args) {
		
		SparkSession spark = SparkSession.builder().master("local").appName("WordCount3").getOrCreate();
		
		JavaRDD
 
   input = spark.read().textFile("words.txt").javaRDD();
		
		JavaRDD
  
    words = input.flatMap(line -> Arrays.asList(line.split(",")).iterator()); JavaPairRDD
   
     pair = words.mapToPair(word -> new Tuple2
    
     (word, 1)); JavaPairRDD
     
       output = pair.reduceByKey((v1, v2) -> v1+v2); output.foreach(res -> System.out.println(res)); } }
     
    
   
  
 
网友评论