Przeglądaj źródła

update hadoopconfig for idctest

Zhang Li 2 lat temu
rodzic
commit
33144550f1
2 zmienionych plików z 8 dodań i 7 usunięć
  1. 1 0
      app/core/airflow/templates/dag_template.py.jinja2
  2. 7 7
      idctest.ini

+ 1 - 0
app/core/airflow/templates/dag_template.py.jinja2

@@ -88,6 +88,7 @@ with DAG(start_date=datetime(2022,6,1),catchup=False,schedule_interval=None if '
         get_logs=True,
         image_pull_policy='Always',
         log_events_on_failure=True,
+        pod_template_file='/opt/airflow/pod_templates/pod_template_file.yaml',
         {% if image_pull_key not in ['None',"",None]%}image_pull_secrets='{{ image_pull_key }}',{% endif %}
         cmds={{ node['cmds'] }},
         env_vars= { **{{ node['env'] }},

+ 7 - 7
idctest.ini

@@ -57,16 +57,16 @@ sparks=registry.cn-hangzhou.aliyuncs.com/sxtest/idctest:spark
 
 [HADOOP_INNER]
 datasource = -100
-default_fs = hdfs://HDFS8000912
+default_fs = hdfs://nameservice1
 hadoop_config={
-            "dfs.nameservices": "HDFS8000912",
-            "dfs.ha.namenodes.HDFS8000912": "nn1,nn2",
-            "dfs.namenode.rpc-address.HDFS8000912.nn1": "10.254.20.18:4007",
-            "dfs.namenode.rpc-address.HDFS8000912.nn2": "10.254.20.22:4007",
-            "dfs.client.failover.proxy.provider.HDFS8000912": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+            "dfs.nameservices": "nameservice1",
+            "dfs.ha.namenodes.nameservice1": "nn1,nn2",
+            "dfs.namenode.rpc-address.nameservice1.nn1": "10.119.14.67:8020",
+            "dfs.namenode.rpc-address.nameservice1.nn2": "10.119.14.68:8020",
+            "dfs.client.failover.proxy.provider.nameservice1": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
              }
 kerberos_config = {
-                    "haveKerberos": "true",
+                    "haveKerberos": "false",
                     "kerberosKeytabFilePath": "/workspace/confs/test/user.keytab",
                     "kerberosPrincipal": "ailab@EMR-5XJSY31F"
                   }