|
@@ -112,7 +112,7 @@ class SparksTaskCompiler(TaskCompiler):
|
|
"executor-memory": "1g",
|
|
"executor-memory": "1g",
|
|
"executor-cores": 1,
|
|
"executor-cores": 1,
|
|
"num-executors": 1,
|
|
"num-executors": 1,
|
|
- "archives": "/workspace/py37.zip#python3env"
|
|
|
|
|
|
+ "archives": f"{self.task.envs.get('requirement_package_path','/workspace/py37.zip')}#python3env"
|
|
}
|
|
}
|
|
spark_config = {'spark.default.parallelism': 1,
|
|
spark_config = {'spark.default.parallelism': 1,
|
|
"spark.executor.memoryOverhead": "1g",
|
|
"spark.executor.memoryOverhead": "1g",
|