yl-jinghui/submit.sh

sourceDIR=/workspace
who=root

YARN_QUEUE=
DEPLOY_MODE=cluster

${SPARK_HOME}/bin/spark-submit \
  --master yarn \
  --name "spark_${ModelType}_${who}" \
  --deploy-mode ${DEPLOY_MODE} \
  --driver-memory 1g \
  --driver-cores 1 \
  --executor-memory 1g \
  --executor-cores 2 \
  --num-executors 1 \
  --archives ${sourceDIR}/py37.zip#python3env \
  --conf spark.default.parallelism=2 \
  --conf spark.executor.memoryOverhead=1g \
  --conf spark.driver.memoryOverhead=1g \
  --conf spark.yarn.maxAppAttempts=3 \
  --conf spark.yarn.submit.waitAppCompletion=true \
  --conf spark.pyspark.driver.python=python3env/py37/bin/python \
  --conf spark.yarn.appMasterEnv.PYSPARK_PYTHON=python3env/py37/bin/python \
  --conf spark.pyspark.python=python3env/py37/bin/python \
  --files "${sourceDIR}/yl-jinghui/demo.yml#demo.yml,${sourceDIR}/yl-jinghui/t_order_demo.csv#t_order_demo.csv" \
  ./main.py
#  --conf spark.yarn.appMasterEnv.CLASSPATH="$CLASSPATH:/opt/cloudera/parcels/CDH/lib/hadoop-mapreduce/*:$HADOOP_COMMON_HOME/lib/*" \