Prechádzať zdrojové kódy

update spark requirement_package_path

Zhang Li 2 rokov pred
rodič
commit
653af8bf38
1 zmenil súbory, kde vykonal 1 pridanie a 1 odobranie
  1. 1 1
      app/core/airflow/task.py

+ 1 - 1
app/core/airflow/task.py

@@ -112,7 +112,7 @@ class SparksTaskCompiler(TaskCompiler):
                       "executor-memory": "1g",
                       "executor-cores": 1,
                       "num-executors": 1,
-                      "archives": "/workspace/py37.zip#python3env"
+                      "archives": f"{self.task.envs.get('requirement_package_path','/workspace/py37.zip')}#python3env"
                       }
         spark_config = {'spark.default.parallelism': 1,
                         "spark.executor.memoryOverhead": "1g",