datax-admin/auo_tests/spark_submit/spark-submit_1009.sh

inputs='{"train_df": "train", "test_df": "test"}'
outputs='["results_tmp_10_10_0"]' # results_tmp_{task_id}_{sub_node_id}_{output_pin_id}

sourceDIR=/home/sxkj/bigdata

${SPARK_HOME}/bin/spark-submit \
  --master yarn \
  --name "spark_demo_1009" \
  --deploy-mode cluster \
  --driver-memory 1g \
  --driver-cores 1 \
  --executor-memory 1g \
  --executor-cores 1 \
  --num-executors 1 \
  --archives ${sourceDIR}/py37.zip#python3env \
  --conf spark.default.parallelism=1 \
  --conf spark.executor.memoryOverhead=1g \
  --conf spark.driver.memoryOverhead=1g \
  --conf spark.yarn.maxAppAttempts=3 \
  --conf spark.yarn.submit.waitAppCompletion=true \
  --conf spark.pyspark.driver.python=${sourceDIR}/py37/bin/python \
  --conf spark.yarn.appMasterEnv.PYSPARK_PYTHON=python3env/py37/bin/python \
  --conf spark.pyspark.python=python3env/py37/bin/python \
  ./spark_script_demo_1009.py "$inputs" "$outputs"