1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586 |
- #!/bin/bash
- #
- # Licensed to the Apache Software Foundation (ASF) under one or more
- # contributor license agreements. See the NOTICE file distributed with
- # this work for additional information regarding copyright ownership.
- # The ASF licenses this file to You under the Apache License, Version 2.0
- # (the "License"); you may not use this file except in compliance with
- # the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #
- # This is a fake version of "spark-submit" to be used during Livy tests that run Spark as
- # a child process. It does some basic parsing of Spark options to figure out the classpath
- # to be used, and then just runs the SparkSubmit class directly.
- #
- PROP_FILE=
- DRIVER_CP=
- DRIVER_OPTS=
- CP_KEY="spark.driver.extraClassPath"
- OPTS_KEY="spark.driver.extraJavaOptions"
- INDEX=1
- ARGS=($@)
- if [ -n "$SPARK_CONF_DIR" ]; then
- PROP_FILE="$SPARK_CONF_DIR/spark-defaults.conf"
- fi
- for IDX in $(seq 0 ${#ARGS[@]}); do
- ARG=${ARGS[$IDX]}
- NEXT=$((IDX + 1))
- case $ARG in
- --conf)
- CONF="${ARGS[$NEXT]}"
- IFS='=' read KEY VALUE <<< "$CONF"
- if [ "$KEY" = "$CP_KEY" ]; then
- DRIVER_CP="$VALUE"
- elif [ "$KEY" = "$OPTS_KEY" ]; then
- DRIVER_OPTS="$VALUE"
- fi
- ;;
- --driver-class-path)
- DRIVER_CP="${ARGS[$NEXT]}"
- ;;
- --properties-file)
- PROP_FILE="${ARGS[$NEXT]}"
- ;;
- esac
- done
- read_opt() {
- local FILE="$1"
- local KEY="$2"
- CONF=$(grep -s "^$KEY=" "$PROP_FILE" | tail -n 1)
- if [ -n "$CONF" ]; then
- IFS='=' read KEY VALUE <<< "$CONF"
- echo "$VALUE"
- fi
- }
- if [ -n "$PROP_FILE" ]; then
- if [ -z "$DRIVER_CP" ]; then
- DRIVER_CP=$(read_opt "$PROP_FILE" "$CP_KEY")
- fi
- if [ -z "$DRIVER_OPTS" ]; then
- DRIVER_OPTS=$(read_opt "$PROP_FILE" "$OPTS_KEY")
- fi
- fi
- if [ -n "$LIVY_TEST_CLASSPATH" ]; then
- DRIVER_CP="$DRIVER_CP:$LIVY_TEST_CLASSPATH"
- fi
- if [ -n "$HADOOP_CONF_DIR" ]; then
- DRIVER_CP="$HADOOP_CONF_DIR:$DRIVER_CP"
- fi
- echo "Running Spark: " $JAVA_HOME/bin/java $DRIVER_OPTS org.apache.spark.deploy.SparkSubmit "$@" >&2
- exec $JAVA_HOME/bin/java $DRIVER_OPTS -cp "$DRIVER_CP" org.apache.spark.deploy.SparkSubmit "$@"
|