apache-spark-on-k8s / userdocs Goto Github PK

</tr> </tr> </table> ## Current Limitations Running Spark on Kubernetes is currently an experimental feature. Some restrictions on the current implementation that should be lifted in the future include: * Applications can only run in cluster mode. * Only Scala, Java, and Python applications can be run.

$ minikube status
minikube: Running
cluster: Running
kubectl: Correctly Configured: pointing to minikube-vm at 192.168.99.100

$ pwd
/Users/arnold/run_spark_on_k8s
$ ls
run.sh		spark

$ kubectl cluster-info
Kubernetes master is running at https://192.168.99.100:8443

To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.

k8s_apiserver_host="192.168.99.100"
k8s_apiserver_port="8443"
cwd=`pwd`

echo $k8s_apiserver_host
echo $k8s_apiserver_port
echo $cwd

spark/bin/spark-submit \
  --deploy-mode cluster \
  --class org.apache.spark.examples.SparkPi \
  --master k8s://https://$k8s_apiserver_host:$k8s_apiserver_port \
  --kubernetes-namespace default \
  --conf spark.executor.instances=5 \
  --conf spark.app.name=spark-pi \
  --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.2.0-kubernetes-0.5.0 \
  --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.2.0-kubernetes-0.5.0 \
  local:///$cwd/spark/examples/target/scala-2.11/jars/spark-examples_2.11-2.2.0-k8s-0.5.0.jar

$ ./run.sh 
192.168.99.100
8443
/Users/arnold/run_spark_on_k8s
java.lang.ClassNotFoundException: org.apache.spark.deploy.k8s.submit.Client
	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at org.apache.spark.util.Utils$.classForName(Utils.scala:230)
	at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:743)
	at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)
	at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)
	at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120)
	at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

$ ls -la /Users/arnold/run_spark_on_k8s/spark/examples/target/scala-2.11/jars/spark-examples_2.11-2.2.0-k8s-0.5.0.jar
-rw-r--r--  1 arnold  1113146471  1991195 Dec 22 14:21 /Users/arnold/run_spark_on_k8s/spark/examples/target/scala-2.11/jars/spark-examples_2.11-2.2.0-k8s-0.5.0.jar