1$ install xcode-select
2$ xcode-select -v
3xcode-select version 2373.

1$ brew install scala
2Updating Homebrew...
3==> Auto-updated Homebrew!
4...
5==> Downloading https://downloads.lightbend.com/scala/2.13.2/scala-2.13.2.tgz
6######################################################################## 100.0%
7==> Caveats
8To use with IntelliJ, set the Scala home to:
9  /usr/local/opt/scala/idea
10==> Summary
11🍺  /usr/local/Cellar/scala/2.13.2: 41 files, 22.5MB, built in 24 seconds
12
13$ scala -version
14Scala code runner version 2.13.2 -- Copyright 2002-2020, LAMP/EPFL and Lightbend, Inc.

1$ brew install apache-spark
2Updating Homebrew...
3==> Downloading https://www.apache.org/dyn/closer.lua?path=spark/spark-2.4.5/spark-2.4.5-bin-hadoop2.7.tgz
4==> Downloading from https://downloads.apache.org/spark/spark-2.4.5/spark-2.4.5-bin-hadoop2.7.tgz
5######################################################################## 100.0%
6🍺  /usr/local/Cellar/apache-spark/2.4.5: 1,059 files, 250.9MB, built in 9 minutes 10 seconds
7
8$ spark-shell
920/06/04 10:50:21 WARN Utils: Your hostname, Wayans-MacBook-Pro.local resolves to a loopback address: 127.0.0.1; using 192.168.1.4 instead (on interface en0)
1020/06/04 10:50:21 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address
1120/06/04 10:50:21 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
12Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
13Setting default log level to "WARN".
14To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
15Spark context Web UI available at http://192.168.1.4:4040
16Spark context available as 'sc' (master = local[*], app id = local-1591239030237).
17Spark session available as 'spark'.
18Welcome to
19      ____              __
20     / __/__  ___ _____/ /__
21    _\ \/ _ \/ _ `/ __/  '_/
22   /___/ .__/\_,_/_/ /_/\_\   version 2.4.5
23      /_/
24         
25Using Scala version 2.11.12 (OpenJDK 64-Bit Server VM, Java 1.8.0_242)
26Type in expressions to have them evaluated.
27Type :help for more information.
28
29scala>

1<?xml version="1.0" encoding="UTF-8"?>
2<project xmlns="http://maven.apache.org/POM/4.0.0"
3         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5    <modelVersion>4.0.0</modelVersion>
6
7    <groupId>org.example</groupId>
8    <artifactId>spark-local-mapred</artifactId>
9    <version>1.0-SNAPSHOT</version>
10
11    <properties>
12        <maven.compiler.target>1.8</maven.compiler.target>
13        <maven.compiler.source>1.8</maven.compiler.source>
14    </properties>
15
16    <dependencies>
17        <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core -->
18        <dependency>
19            <groupId>org.apache.spark</groupId>
20            <artifactId>spark-core_2.11</artifactId>
21            <version>2.4.5</version>
22        </dependency>
23        <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql -->
24        <dependency>
25            <groupId>org.apache.spark</groupId>
26            <artifactId>spark-sql_2.11</artifactId>
27            <version>2.4.5</version>
28        </dependency>
29    </dependencies>
30    
31</project>

1package com.teknologibigdata.spark;
2//WordCount.java
3
4import org.apache.spark.sql.*;
5import scala.Tuple2;
6import java.io.BufferedReader;
7import java.io.IOException;
8import java.io.InputStreamReader;
9import java.util.ArrayList;
10import java.util.Arrays;
11import java.util.List;
12import java.util.regex.Pattern;
13
14public class WordCount {
15
16    private static final Pattern SPACE = Pattern.compile(" ");
17    public List<String> enStopwords = new ArrayList<>();
18    public final SparkSession spark;
19
20    public WordCount() throws IOException {
21        spark = SparkSession
22                .builder()
23                .appName("WordCount")
24                //.master("local[1]")//ローカルで実行する場合、この行は必要です（uncommentして下さい)。
25                .getOrCreate();
26        readStopwords();
27    }
28
29    private void readStopwords() throws IOException {
30        BufferedReader bfr = new BufferedReader(
31                new InputStreamReader(
32                        WordCount.class.getResourceAsStream("/en_stopwords.txt")
33                )
34        );
35
36        String line = null;
37        while ((line = bfr.readLine()) != null) {
38            enStopwords.add(line);
39        }
40    }
41
42    public static void main(String[] args) throws IOException {
43
44        if (args.length < 2) {
45            System.err.println("Usage: JavaWordCount <inputFile> <outputFile>");
46            System.exit(1);
47        }
48
49        WordCount wc = new WordCount();
50        List<String> stopwords = wc.enStopwords;
51        Dataset<String> textDf = wc.spark.read().textFile(args[0]);
52        textDf.show(10);
53
54        Dataset<Row> wordCount = textDf
55                .flatMap(line -> Arrays.asList(SPACE.split(line.replaceAll("\\W", " "))).iterator(), Encoders.STRING())
56                .filter(str -> !str.isEmpty())
57                .filter(str->!stopwords.contains(str)) //stopwordsを入力テキストから排除
58                .map(word -> new Tuple2<>(word.toLowerCase(), 1L), Encoders.tuple(Encoders.STRING(), Encoders.LONG()))
59                .toDF("word", "one") //テキストからDataframeを作り、wordフィルドにテキストから抽出した単語、oneフィルドに数字の1を格納
60                .groupBy("word")
61                .sum("one").orderBy(new Column("sum(one)").desc())
62                .withColumnRenamed("sum(one)", "count");
63
64        wordCount.show(10);
65        wordCount.write().format("csv").save(args[1]);
66    }
67}