|
@@ -112,8 +112,12 @@ class SparksTaskCompiler(TaskCompiler):
|
|
|
"executor-memory": "1g",
|
|
|
"executor-cores": 1,
|
|
|
"num-executors": 1,
|
|
|
- "archives": f"{self.task.envs.get('requirement_package_path','/workspace/py37.zip')}#python3env"
|
|
|
+ "archives": "/workspace/py37.zip'#python3env"
|
|
|
}
|
|
|
+ if item:
|
|
|
+ parameters.update({
|
|
|
+ "archives": f"{self.task.envs.get('requirement_package_path', '/workspace/py37.zip')}#python3env"
|
|
|
+ })
|
|
|
spark_config = {'spark.default.parallelism': 1,
|
|
|
"spark.executor.memoryOverhead": "1g",
|
|
|
"spark.driver.memoryOverhead": "1g",
|