Upload New File

JavaWordCount.java

+package cdist;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import scala.Tuple2;
+
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.function.FlatMapFunction;
+import org.apache.spark.api.java.function.Function2;
+import org.apache.spark.api.java.function.PairFunction;
+
+
+/*  This example has been taken from Oreally examples */
+
+public class JavaWordCount {
+
+	
+	public static void main(String[] args) throws Exception {
+		
+	
+		// Set the input and output default files
+		// it can be a file from hdfs -> hdfs://server:port/path
+		// or a local file file:///path
+		// or a relative local file "fileName" under the working directory (ie. out)
+		String inputFile = "file:///var/home/lab/asig/labgcd/workspace-cdist-spark-and-streaming/spark-aptel/in.txt";
+		String outputFile = "out";
+		
+		// let the user add params optionally to define input and output file
+		if(args.length > 2)
+		{
+			inputFile = args[0];
+			outputFile = args[1];
+		}
+		
+			
+		
+		// Create a Java Spark Context, for the application with name "wordCount", use a local cluster 
+		// (for using a existing cluster just substitute "local" with the name of the machine
+		JavaSparkContext sc = new JavaSparkContext(
+			      "local", "wordcount", System.getenv("SPARK_HOME"), System.getenv("JARS"));
+		
+		// Load our input data. 
+		// will create an inmutable (RDD) set of strings (one per line)
+		JavaRDD<String> input = sc.textFile(inputFile);
+		
+		
+		// Split up into words.
+		// make a map (line -> words in that line) and make it flat (so a sequence of words irrespectively of their line)
+		JavaRDD<String> words = input.flatMap(x -> Arrays.asList(x.split(" ")).iterator());
+		
+		// Transform into word and count.
+		// associate 1 per word
+		// and then reduce by adding all the numbers per word (key)
+		JavaPairRDD<String, Integer> counts = words.mapToPair(s -> new Tuple2<String, Integer>(s, 1))
+				.reduceByKey((x,y) -> x+y);
+		
+		// Save the word count back out to a text file, causing evaluation.
+		counts.saveAsTextFile(outputFile);
+	}
+}