Merge branch 'master' of http://gogsb.soaringnova.com/sxwl_DL/datax-admin

app/core/airflow/templates/dag_template.py.jinja2

@@ -66,6 +66,7 @@ with DAG(start_date=datetime(2022,6,1),catchup=False,schedule_interval=None if '
                                                                     get_logs=True,
                                                                     {% if image_pull_key != None or image_pull_key != ""%}image_pull_secrets='{{ image_pull_key }}',{% endif %}
                                                                     log_events_on_failure=True,
+                                                                    image_pull_policy='Always',
                                                                     cmds={{ spark_sub_node['cmds'] }},
                                                                     env_vars={{ spark_sub_node['env'] }},
                                                                     on_success_callback=task_finish_alert,
@@ -91,6 +92,7 @@ with DAG(start_date=datetime(2022,6,1),catchup=False,schedule_interval=None if '
         reattach_on_restart=True,
         is_delete_operator_pod=False,
         get_logs=True,
+        image_pull_policy='Always',
         log_events_on_failure=True,
         {% if image_pull_key not in ['None',"",None]%}image_pull_secrets='{{ image_pull_key }}',{% endif %}
         cmds={{ node['cmds'] }},

app/core/airflow/templates/sql_script_template.py.jinja2

@@ -8,7 +8,7 @@ def run(inputs: dict, outputs: list):
     for k, tb_name in inputs.items():
         script=script.replace(k,tb_name)
     ss = SparkSession.builder.config('hive.metastore.uris',
-                                     'thrift://192.168.199.27:9083').enableHiveSupport().getOrCreate()
+                                     '{{ hive_metastore_uris }}').enableHiveSupport().getOrCreate()
     ret_df = ss.sql(script)
     ret_df.write.mode("overwrite").saveAsTable(outputs[0])
 

production.ini

@@ -43,7 +43,8 @@ principal = ailab@EMR-5XJSY31F
 
 
 [HIVE_METASTORE]
-uris=thrift://10.254.20.18:7004
+uris=thrift://10.254.20.18:7004,thrift://10.254.20.22:7004
+;uris=thrift://10.254.20.22:7004
 
 [TASK_IMAGES]
 datax=yldc-docker.pkg.coding.yili.com/aiplatform/docker/aihub-datax-yili:latest