- ${SPARK_HOME}/bin/spark-submit --name spark_25_hrgKx21DwpKfXAetpGoW8 --master yarn --deploy-mode cluster --queue root.users.aiuser --driver-memory 1g --driver-cores 1 --executor-memory 20g --executor-cores 3 --num-executors 2 --archives /workspace/py37.zip#python3env --conf spark.default.parallelism=1 --conf spark.executor.memoryOverhead=1g --conf spark.driver.memoryOverhead=1g --conf spark.yarn.maxAppAttempts=1 --conf spark.yarn.submit.waitAppCompletion=true --conf spark.pyspark.driver.python=python3env/py37/bin/python --conf spark.yarn.appMasterEnv.PYSPARK_PYTHON=python3env/py37/bin/python --conf spark.pyspark.python=python3env/py37/bin/python run.py
|
