create spark environment to test feature
- Docker
- docker-compose
# start spark master and workers
docker-compose -f docker-compose.yml up -d
# exec spark master shell in local
docker exec -it spark bash
# run Java example in spark master shell
./bin/spark-submit \
--class org.apache.spark.examples.SparkPi \
--master spark://spark:7077 \
--deploy-mode cluster \
./examples/jars/spark-examples_2.12-3.0.0.jar \
100
# run Python example in spark master shell
./bin/spark-submit \
--master spark://spark:7077 \
./examples/src/main/python/pi.py \
100
You can open interface to see what happened in spark master and workers
-
open spark master UI
-
open spark worker UI
-
remove all container and volume
docker-compose down -v
Submit your spark application
docker-compose -f docker-compose-notebook.yml up -d
# or simple one-liner if spark setup not required
docker run -it -p 8888:8888 jupyter/pyspark-notebook