delete old dockerfile

README.md

@@ -26,15 +26,9 @@ mvn clean package \
 项目中提供了 Dockerfile, 可以将源码打包为镜像.
 
 ```bash
-# 构建基础镜像
-cd docker_build
-docker build -t ylproj-livy-build:1.0.0 .
-cd ../docker_livy_base
-docker build -t ylproj-livy-base:1.0.0 .
-cd ..
 
 # 开始构建 Livy 镜像
-docker build -t ylproj-livy .
+docker build -t ylproj-livy -f docker/Dockerfile .
 ```
 
 通过构建好的 Livy 镜像运行服务
@@ -89,7 +83,7 @@ docker run -d \
 - log4j.properties 是日志配置
 - spark-defaults.conf 是 spark 的配置文件
 - user.keytab 是 Livy 服务的 keytab
-  
+
 ### 关于 Kerberos
 
 如果需要 Kerberos 认证的话, 就要把 KDC 配置放到 /etc/krb5.conf 这里

build.sh

@@ -1,7 +0,0 @@
-mvn clean package \
-    -B -V -e \
-    -Pspark-3.0 \
-    -Pthriftserver \
-    -DskipTests \
-    -DskipITs \
-    -Dmaven.javadoc.skip=true

docker_build/Dockerfile

@@ -1,8 +0,0 @@
-FROM maven:3.8.6-openjdk-8-slim
-COPY settings.xml /root/.m2/
-RUN echo "deb http://mirror.nju.edu.cn/debian/ bullseye main contrib non-free" > /etc/apt/sources.list && \
-    echo "deb http://mirror.nju.edu.cn/debian/ bullseye-updates main contrib non-free" >> /etc/apt/sources.list && \
-    echo "deb http://mirror.nju.edu.cn/debian/ bullseye-backports main contrib non-free" >> /etc/apt/sources.list && \
-    echo "deb http://mirror.nju.edu.cn/debian-security bullseye-security main contrib non-free" >> /etc/apt/sources.list && \
-    apt update && apt install -y python3 python3-pip python-is-python3 \
-    && pip config set global.index-url https://mirror.nju.edu.cn/pypi/web/simple \

docker_build/settings.xml

@@ -1,269 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied.  See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-<!--
- | This is the configuration file for Maven. It can be specified at two levels:
- |
- |  1. User Level. This settings.xml file provides configuration for a single user,
- |                 and is normally provided in ${user.home}/.m2/settings.xml.
- |
- |                 NOTE: This location can be overridden with the CLI option:
- |
- |                 -s /path/to/user/settings.xml
- |
- |  2. Global Level. This settings.xml file provides configuration for all Maven
- |                 users on a machine (assuming they're all using the same Maven
- |                 installation). It's normally provided in
- |                 ${maven.conf}/settings.xml.
- |
- |                 NOTE: This location can be overridden with the CLI option:
- |
- |                 -gs /path/to/global/settings.xml
- |
- | The sections in this sample file are intended to give you a running start at
- | getting the most out of your Maven installation. Where appropriate, the default
- | values (values used when the setting is not specified) are provided.
- |
- |-->
-<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
-          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-          xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
-  <!-- localRepository
-   | The path to the local repository maven will use to store artifacts.
-   |
-   | Default: ${user.home}/.m2/repository
-  <localRepository>/path/to/local/repo</localRepository>
-  -->
-
-  <!-- interactiveMode
-   | This will determine whether maven prompts you when it needs input. If set to false,
-   | maven will use a sensible default value, perhaps based on some other setting, for
-   | the parameter in question.
-   |
-   | Default: true
-  <interactiveMode>true</interactiveMode>
-  -->
-
-  <!-- offline
-   | Determines whether maven should attempt to connect to the network when executing a build.
-   | This will have an effect on artifact downloads, artifact deployment, and others.
-   |
-   | Default: false
-  <offline>false</offline>
-  -->
-
-  <!-- pluginGroups
-   | This is a list of additional group identifiers that will be searched when resolving plugins by their prefix, i.e.
-   | when invoking a command line like "mvn prefix:goal". Maven will automatically add the group identifiers
-   | "org.apache.maven.plugins" and "org.codehaus.mojo" if these are not already contained in the list.
-   |-->
-  <pluginGroups>
-    <!-- pluginGroup
-     | Specifies a further group identifier to use for plugin lookup.
-    <pluginGroup>com.your.plugins</pluginGroup>
-    -->
-  </pluginGroups>
-
-  <!-- proxies
-   | This is a list of proxies which can be used on this machine to connect to the network.
-   | Unless otherwise specified (by system property or command-line switch), the first proxy
-   | specification in this list marked as active will be used.
-   |-->
-  <proxies>
-    <!-- proxy
-     | Specification for one proxy, to be used in connecting to the network.
-     |
-    <proxy>
-      <id>optional</id>
-      <active>true</active>
-      <protocol>http</protocol>
-      <username>proxyuser</username>
-      <password>proxypass</password>
-      <host>proxy.host.net</host>
-      <port>80</port>
-      <nonProxyHosts>local.net|some.host.com</nonProxyHosts>
-    </proxy>
-    -->
-    <!-- <proxy>
-      <active>true</active>
-      <protocol>http</protocol>
-      <host>localhost</host>
-      <port>7890</port>
-    </proxy> -->
-  </proxies>
-
-  <!-- servers
-   | This is a list of authentication profiles, keyed by the server-id used within the system.
-   | Authentication profiles can be used whenever maven must make a connection to a remote server.
-   |-->
-  <servers>
-    <!-- server
-     | Specifies the authentication information to use when connecting to a particular server, identified by
-     | a unique name within the system (referred to by the 'id' attribute below).
-     |
-     | NOTE: You should either specify username/password OR privateKey/passphrase, since these pairings are
-     |       used together.
-     |
-    <server>
-      <id>deploymentRepo</id>
-      <username>repouser</username>
-      <password>repopwd</password>
-    </server>
-    -->
-
-    <!-- Another sample, using keys to authenticate.
-    <server>
-      <id>siteServer</id>
-      <privateKey>/path/to/private/key</privateKey>
-      <passphrase>optional; leave empty if not used.</passphrase>
-    </server>
-    -->
-  </servers>
-
-  <!-- mirrors
-   | This is a list of mirrors to be used in downloading artifacts from remote repositories.
-   |
-   | It works like this: a POM may declare a repository to use in resolving certain artifacts.
-   | However, this repository may have problems with heavy traffic at times, so people have mirrored
-   | it to several places.
-   |
-   | That repository definition will have a unique id, so we can create a mirror reference for that
-   | repository, to be used as an alternate download site. The mirror site will be the preferred
-   | server for that repository.
-   |-->
-  <mirrors>
-    <!-- mirror
-     | Specifies a repository mirror site to use instead of a given repository. The repository that
-     | this mirror serves has an ID that matches the mirrorOf element of this mirror. IDs are used
-     | for inheritance and direct lookup purposes, and must be unique across the set of mirrors.
-     |
-    <mirror>
-      <id>mirrorId</id>
-      <mirrorOf>repositoryId</mirrorOf>
-      <name>Human Readable Name for this Mirror.</name>
-      <url>http://my.repository.com/repo/path</url>
-    </mirror>
-     -->
-    <mirror>
-      <id>aliyunmaven</id>
-      <mirrorOf>*</mirrorOf>
-      <name>阿里云公共仓库</name>
-      <url>https://maven.aliyun.com/repository/public</url>
-    </mirror>
-  </mirrors>
-
-  <!-- profiles
-   | This is a list of profiles which can be activated in a variety of ways, and which can modify
-   | the build process. Profiles provided in the settings.xml are intended to provide local machine-
-   | specific paths and repository locations which allow the build to work in the local environment.
-   |
-   | For example, if you have an integration testing plugin - like cactus - that needs to know where
-   | your Tomcat instance is installed, you can provide a variable here such that the variable is
-   | dereferenced during the build process to configure the cactus plugin.
-   |
-   | As noted above, profiles can be activated in a variety of ways. One way - the activeProfiles
-   | section of this document (settings.xml) - will be discussed later. Another way essentially
-   | relies on the detection of a system property, either matching a particular value for the property,
-   | or merely testing its existence. Profiles can also be activated by JDK version prefix, where a
-   | value of '1.4' might activate a profile when the build is executed on a JDK version of '1.4.2_07'.
-   | Finally, the list of active profiles can be specified directly from the command line.
-   |
-   | NOTE: For profiles defined in the settings.xml, you are restricted to specifying only artifact
-   |       repositories, plugin repositories, and free-form properties to be used as configuration
-   |       variables for plugins in the POM.
-   |
-   |-->
-  <profiles>
-    <!-- profile
-     | Specifies a set of introductions to the build process, to be activated using one or more of the
-     | mechanisms described above. For inheritance purposes, and to activate profiles via <activatedProfiles/>
-     | or the command line, profiles have to have an ID that is unique.
-     |
-     | An encouraged best practice for profile identification is to use a consistent naming convention
-     | for profiles, such as 'env-dev', 'env-test', 'env-production', 'user-jdcasey', 'user-brett', etc.
-     | This will make it more intuitive to understand what the set of introduced profiles is attempting
-     | to accomplish, particularly when you only have a list of profile id's for debug.
-     |
-     | This profile example uses the JDK version to trigger activation, and provides a JDK-specific repo.
-    <profile>
-      <id>jdk-1.4</id>
-
-      <activation>
-        <jdk>1.4</jdk>
-      </activation>
-
-      <repositories>
-        <repository>
-          <id>jdk14</id>
-          <name>Repository for JDK 1.4 builds</name>
-          <url>http://www.myhost.com/maven/jdk14</url>
-          <layout>default</layout>
-          <snapshotPolicy>always</snapshotPolicy>
-        </repository>
-      </repositories>
-    </profile>
-    -->
-
-    <!--
-     | Here is another profile, activated by the system property 'target-env' with a value of 'dev',
-     | which provides a specific path to the Tomcat instance. To use this, your plugin configuration
-     | might hypothetically look like:
-     |
-     | ...
-     | <plugin>
-     |   <groupId>org.myco.myplugins</groupId>
-     |   <artifactId>myplugin</artifactId>
-     |
-     |   <configuration>
-     |     <tomcatLocation>${tomcatPath}</tomcatLocation>
-     |   </configuration>
-     | </plugin>
-     | ...
-     |
-     | NOTE: If you just wanted to inject this configuration whenever someone set 'target-env' to
-     |       anything, you could just leave off the <value/> inside the activation-property.
-     |
-    <profile>
-      <id>env-dev</id>
-
-      <activation>
-        <property>
-          <name>target-env</name>
-          <value>dev</value>
-        </property>
-      </activation>
-
-      <properties>
-        <tomcatPath>/path/to/tomcat/instance</tomcatPath>
-      </properties>
-    </profile>
-    -->
-  </profiles>
-
-  <!-- activeProfiles
-   | List of profiles that are active for all builds.
-   |
-  <activeProfiles>
-    <activeProfile>alwaysActiveProfile</activeProfile>
-    <activeProfile>anotherAlwaysActiveProfile</activeProfile>
-  </activeProfiles>
-  -->
-</settings>

docker_livy_base/Dockerfile

@@ -1,15 +0,0 @@
-FROM openjdk:8-jre
-WORKDIR /opt/cluster
-RUN echo "deb http://mirror.nju.edu.cn/debian/ bullseye main contrib non-free" > /etc/apt/sources.list && \
-    echo "deb http://mirror.nju.edu.cn/debian/ bullseye-updates main contrib non-free" >> /etc/apt/sources.list && \
-    echo "deb http://mirror.nju.edu.cn/debian/ bullseye-backports main contrib non-free" >> /etc/apt/sources.list && \
-    echo "deb http://mirror.nju.edu.cn/debian-security bullseye-security main contrib non-free" >> /etc/apt/sources.list && \
-    apt update && apt install -y krb5-user && mkdir conf && \
-    curl -fL http://mirror.nju.edu.cn/apache/hadoop/common/hadoop-3.2.3/hadoop-3.2.3.tar.gz -o hadoop.tar.gz && \
-    tar -xf hadoop.tar.gz && mv hadoop-3.2.3 hadoop && rm hadoop.tar.gz && \
-    curl -fL http://mirror.nju.edu.cn/apache/spark/spark-3.0.3/spark-3.0.3-bin-hadoop3.2.tgz -o spark.tgz && \
-    tar -xf spark.tgz && mv spark-3.0.3-bin-hadoop3.2 spark && rm spark.tgz
-ENV HADOOP_HOME=/opt/cluster/hadoop
-ENV HADOOP_CONF_DIR=/opt/cluster/conf
-ENV SPARK_HOME=/opt/cluster/spark
-ENV SPARK_CONF_DIR=/opt/cluster/conf

yili-conf/cluster/conf/core-site.xml

@@ -1,256 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<configuration>
-
-    <property>
-        <name>emr.cfs.group.id.map</name>
-        <value>root:0;hadoop:500</value>
-    </property>
-
-    <property>
-        <name>emr.cfs.io.blocksize</name>
-        <value>1048576</value>
-    </property>
-
-    <property>
-        <name>emr.cfs.user.id.map</name>
-        <value>root:0;hadoop:500</value>
-    </property>
-
-    <property>
-        <name>emr.cfs.write.level</name>
-        <value>2</value>
-    </property>
-
-    <property>
-        <name>emr.temrfs.download.md5</name>
-        <value>3d695d5306f3df7cbc1eff5676acbc11</value>
-    </property>
-
-    <property>
-        <name>emr.temrfs.download.region</name>
-        <value>bj</value>
-    </property>
-
-    <property>
-        <name>emr.temrfs.download.version</name>
-        <value>3.1.0-8.1.4-1.0.6</value>
-    </property>
-
-    <property>
-        <name>emr.temrfs.tmp.cache.dir</name>
-        <value>/data/emr/hdfs/tmp/temrfs</value>
-    </property>
-
-    <property>
-        <name>fs.AbstractFileSystem.ofs.impl</name>
-        <value>com.qcloud.chdfs.fs.CHDFSDelegateFSAdapter</value>
-    </property>
-
-    <property>
-        <name>fs.cfs.impl</name>
-        <value>com.tencent.cloud.emr.CFSFileSystem</value>
-    </property>
-
-    <property>
-        <name>fs.cos.buffer.dir</name>
-        <value>/data/emr/hdfs/tmp</value>
-    </property>
-
-    <property>
-        <name>fs.cos.local_block_size</name>
-        <value>2097152</value>
-    </property>
-
-    <property>
-        <name>fs.cos.userinfo.appid</name>
-        <value>1302259445</value>
-    </property>
-
-    <property>
-        <name>fs.cos.userinfo.region</name>
-        <value>bj</value>
-    </property>
-
-    <property>
-        <name>fs.cos.userinfo.useCDN</name>
-        <value>false</value>
-    </property>
-
-    <property>
-        <name>fs.cosn.block.size</name>
-        <value>67108864</value>
-    </property>
-
-    <property>
-        <name>fs.cosn.credentials.provider</name>
-        <value>org.apache.hadoop.fs.auth.EMRInstanceCredentialsProvider</value>
-    </property>
-
-    <property>
-        <name>fs.cosn.emrv2.instance.provider.enabled</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>fs.cosn.impl</name>
-        <value>com.qcloud.emr.fs.TemrfsHadoopFileSystemAdapter</value>
-    </property>
-
-    <property>
-        <name>fs.cosn.local_block_size</name>
-        <value>2097152</value>
-    </property>
-
-    <property>
-        <name>fs.cosn.posix_bucket.fs.impl</name>
-        <value>com.qcloud.chdfs.fs.CHDFSHadoopFileSystemAdapter</value>
-    </property>
-
-    <property>
-        <name>fs.cosn.tmp.dir</name>
-        <value>/data/emr/hdfs/tmp/hadoop_cos</value>
-    </property>
-
-    <property>
-        <name>fs.cosn.trsf.fs.AbstractFileSystem.ofs.impl</name>
-        <value>com.qcloud.chdfs.fs.CHDFSDelegateFSAdapter</value>
-    </property>
-
-    <property>
-        <name>fs.cosn.trsf.fs.ofs.bucket.region</name>
-        <value>ap-beijing</value>
-    </property>
-
-    <property>
-        <name>fs.cosn.trsf.fs.ofs.impl</name>
-        <value>com.qcloud.chdfs.fs.CHDFSHadoopFileSystemAdapter</value>
-    </property>
-
-    <property>
-        <name>fs.cosn.trsf.fs.ofs.tmp.cache.dir</name>
-        <value>/data/emr/hdfs/tmp/posix</value>
-    </property>
-
-    <property>
-        <name>fs.cosn.trsf.fs.ofs.user.appid</name>
-        <value>1302259445</value>
-    </property>
-
-    <property>
-        <name>fs.cosn.upload.buffer</name>
-        <value>mapped_disk</value>
-    </property>
-
-    <property>
-        <name>fs.cosn.upload.buffer.size</name>
-        <value>-1</value>
-    </property>
-
-    <property>
-        <name>fs.cosn.userinfo.region</name>
-        <value>ap-beijing</value>
-    </property>
-
-    <property>
-        <name>fs.defaultFS</name>
-        <value>hdfs://HDFS8000912</value>
-    </property>
-
-    <property>
-        <name>fs.emr.version</name>
-        <value>9c06b7b</value>
-    </property>
-
-    <property>
-        <name>fs.ofs.bucket.region</name>
-        <value>ap-beijing</value>
-    </property>
-
-    <property>
-        <name>fs.ofs.impl</name>
-        <value>com.qcloud.chdfs.fs.CHDFSHadoopFileSystemAdapter</value>
-    </property>
-
-    <property>
-        <name>fs.ofs.tmp.cache.dir</name>
-        <value>/data/emr/hdfs/tmp/chdfs</value>
-    </property>
-
-    <property>
-        <name>fs.ofs.user.appid</name>
-        <value>1302259445</value>
-    </property>
-
-    <property>
-        <name>fs.trash.interval</name>
-        <value>1440</value>
-    </property>
-
-    <property>
-        <name>ha.health-monitor.rpc-timeout.ms</name>
-        <value>180000</value>
-    </property>
-
-    <property>
-        <name>ha.zookeeper.session-timeout.ms</name>
-        <value>180000</value>
-    </property>
-
-    <property>
-        <name>hadoop.http.staticuser.user</name>
-        <value>hadoop</value>
-    </property>
-
-    <property>
-        <name>hadoop.logfile.count</name>
-        <value>20</value>
-    </property>
-
-    <property>
-        <name>hadoop.logfile.size</name>
-        <value>1000000000</value>
-    </property>
-
-    <property>
-        <name>hadoop.proxyuser.hadoop.groups</name>
-        <value>*</value>
-    </property>
-
-    <property>
-        <name>hadoop.proxyuser.hadoop.hosts</name>
-        <value>*</value>
-    </property>
-
-    <property>
-        <name>hadoop.security.authentication</name>
-        <value>kerberos</value>
-    </property>
-
-    <property>
-        <name>hadoop.security.authorization</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>hadoop.tmp.dir</name>
-        <value>/data/emr/hdfs/tmp</value>
-    </property>
-
-    <property>
-        <name>hadoop.zk.timeout-ms</name>
-        <value>60000</value>
-    </property>
-
-    <property>
-        <name>io.compression.codec.lzo.class</name>
-        <value>com.hadoop.compression.lzo.LzoCodec</value>
-    </property>
-
-    <property>
-        <name>io.compression.codecs</name>
-        <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
-    </property>
-
-</configuration>

yili-conf/cluster/conf/hdfs-site.xml

@@ -1,325 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-	
-	<property>
-		<name>dfs.balance.bandwidthPerSec</name>
-		<value>10485760</value>
-	</property>
-	
-	<property>
-		<name>dfs.block.access.token.enable</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>dfs.blockdfs.webhdfs.enabled.access.token.enable</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>dfs.blocksize</name>
-		<value>134217728</value>
-	</property>
-	
-	<property>
-		<name>dfs.client.block.write.replace-datanode-on-failure.enable</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>dfs.client.failover.proxy.provider.HDFS8000912</name>
-		<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
-	</property>
-	
-	<property>
-		<name>dfs.client.slow.io.warning.threshold.ms</name>
-		<value>900000</value>
-	</property>
-	
-	<property>
-		<name>dfs.client.socket-timeout</name>
-		<value>60000</value>
-	</property>
-	
-	<property>
-		<name>dfs.client.use.datanode.hostname</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>dfs.datanode.handler.count</name>
-		<value>128</value>
-	</property>
-	
-	<property>
-		<name>dfs.datanode.kerberos.principal</name>
-		<value>hadoop/_HOST@EMR-5XJSY31F</value>
-	</property>
-	
-	<property>
-		<name>dfs.datanode.keytab.file</name>
-		<value>/var/krb5kdc/emr.keytab</value>
-	</property>
-	
-	<property>
-		<name>dfs.datanode.max.transfer.threads</name>
-		<value>16480</value>
-	</property>
-	
-	<property>
-		<name>dfs.datanode.max.xcievers</name>
-		<value>8000</value>
-	</property>
-	
-	<property>
-		<name>dfs.datanode.slow.io.warning.threshold.ms</name>
-		<value>90000</value>
-	</property>
-	
-	<property>
-		<name>dfs.datanode.socket.write.timeout</name>
-		<value>480000</value>
-	</property>
-	
-	<property>
-		<name>dfs.datanode.use.datanode.hostname</name>
-		<value>false</value>
-	</property>
-	
-	<property>
-		<name>dfs.encryption.key.provider.uri</name>
-		<value></value>
-	</property>
-	
-	<property>
-		<name>dfs.ha.automatic-failover.enabled</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>dfs.ha.fencing.methods</name>
-		<value>shell(/bin/true)</value>
-	</property>
-	
-	<property>
-		<name>dfs.ha.namenodes.HDFS8000912</name>
-		<value>nn1,nn2</value>
-	</property>
-	
-	<property>
-		<name>dfs.heartbeat.interval</name>
-		<value>2</value>
-	</property>
-	
-	<property>
-		<name>dfs.hosts</name>
-		<value>/usr/local/service/hadoop/etc/hadoop/hdfshosts</value>
-	</property>
-	
-	<property>
-		<name>dfs.hosts.exclude</name>
-		<value>/usr/local/service/hadoop/etc/hadoop/hdfsexcludedhosts</value>
-	</property>
-	
-	<property>
-		<name>dfs.journalnode.edits.dir</name>
-		<value>/data/emr/hdfs/journalnode</value>
-	</property>
-	
-	<property>
-		<name>dfs.journalnode.kerberos.internal.spnego.principal</name>
-		<value>HTTP/_HOST@EMR-5XJSY31F</value>
-	</property>
-	
-	<property>
-		<name>dfs.journalnode.kerberos.principal</name>
-		<value>hadoop/_HOST@EMR-5XJSY31F</value>
-	</property>
-	
-	<property>
-		<name>dfs.journalnode.keytab.file</name>
-		<value>/var/krb5kdc/emr.keytab</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.avoid.read.stale.datanode</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.avoid.write.stale.datanode</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.datanode.registration.ip-hostname-check</name>
-		<value>false</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.handler.count</name>
-		<value>64</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.heartbeat.recheck-interval</name>
-		<value>90000</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.http-address.HDFS8000912.nn1</name>
-		<value>10.254.20.18:4008</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.http-address.HDFS8000912.nn2</name>
-		<value>10.254.20.22:4008</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.https-address.HDFS8000912.nn1</name>
-		<value>10.254.20.18:4009</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.https-address.HDFS8000912.nn2</name>
-		<value>10.254.20.22:4009</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.inode.attributes.provider.class</name>
-		<value>org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.kerberos.internal.spnego.principal</name>
-		<value>HTTP/_HOST@EMR-5XJSY31F</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.kerberos.principal</name>
-		<value>hadoop/_HOST@EMR-5XJSY31F</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.keytab.file</name>
-		<value>/var/krb5kdc/emr.keytab</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.name.dir</name>
-		<value>/data/emr/hdfs/namenode</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.rpc-address.HDFS8000912.nn1</name>
-		<value>10.254.20.18:4007</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.rpc-address.HDFS8000912.nn2</name>
-		<value>10.254.20.22:4007</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.service.handler.count</name>
-		<value>64</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.servicerpc-address.HDFS8000912.nn1</name>
-		<value>10.254.20.18:4010</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.servicerpc-address.HDFS8000912.nn2</name>
-		<value>10.254.20.22:4010</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.shared.edits.dir</name>
-		<value>qjournal://10.254.20.23:4005;10.254.20.29:4005;10.254.20.26:4005/hadoop</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.stale.datanode.interval</name>
-		<value>30000</value>
-	</property>
-	
-	<property>
-		<name>dfs.namenode.write.stale.datanode.ratio</name>
-		<value>0.5f</value>
-	</property>
-	
-	<property>
-		<name>dfs.nameservices</name>
-		<value>HDFS8000912</value>
-	</property>
-	
-	<property>
-		<name>dfs.permissions</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>dfs.permissions.umask-mode</name>
-		<value>077</value>
-	</property>
-	
-	<property>
-		<name>dfs.replication</name>
-		<value>3</value>
-	</property>
-	
-	<property>
-		<name>dfs.secondary.namenode.kerberos.internal.spnego.principal</name>
-		<value>HTTP/_HOST@EMR-5XJSY31F</value>
-	</property>
-	
-	<property>
-		<name>dfs.secondary.namenode.kerberos.principal</name>
-		<value>hadoop/_HOST@EMR-5XJSY31F</value>
-	</property>
-	
-	<property>
-		<name>dfs.secondary.namenode.keytab.file</name>
-		<value>/var/krb5kdc/emr.keytab</value>
-	</property>
-	
-	<property>
-		<name>dfs.socket.timeout</name>
-		<value>900000</value>
-	</property>
-	
-	<property>
-		<name>dfs.web.authentication.kerberos.keytab</name>
-		<value>/var/krb5kdc/emr.keytab</value>
-	</property>
-	
-	<property>
-		<name>dfs.web.authentication.kerberos.principal</name>
-		<value>HTTP/_HOST@EMR-5XJSY31F</value>
-	</property>
-	
-	<property>
-		<name>dfs.webhdfs.enabled</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>ha.zookeeper.quorum</name>
-		<value>10.254.20.23:2181,10.254.20.29:2181,10.254.20.26:2181</value>
-	</property>
-	
-	<property>
-		<name>ignore.secure.ports.for.testing</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>output.replace-datanode-on-failure</name>
-		<value>false</value>
-	</property>
-	
-</configuration>

yili-conf/cluster/conf/hive-site.xml

@@ -1,405 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration>
-
-    <property>
-        <name>datanucleus.schema.autoCreateTables</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>hbase.zookeeper.quorum</name>
-        <value>10.254.20.23:2181,10.254.20.29:2181,10.254.20.26:2181</value>
-    </property>
-
-    <property>
-        <name>hive.downloaded.resources.dir</name>
-        <value>/data/emr/hive/tmp/${hive.session.id}_resources</value>
-    </property>
-
-    <property>
-        <name>hive.exec.dynamic.partition.mode</name>
-        <value>nonstrict</value>
-    </property>
-
-    <property>
-        <name>hive.exec.failure.hooks</name>
-        <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
-    </property>
-
-    <property>
-        <name>hive.exec.local.scratchdir</name>
-        <value>/data/emr/hive/tmp</value>
-    </property>
-
-    <property>
-        <name>hive.exec.parallel = true</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>hive.exec.parallel.thread.number</name>
-        <value>16</value>
-    </property>
-
-    <property>
-        <name>hive.exec.post.hooks</name>
-        <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
-    </property>
-
-    <property>
-        <name>hive.exec.pre.hooks</name>
-        <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
-    </property>
-
-    <property>
-        <name>hive.execution.engine</name>
-        <value>tez</value>
-    </property>
-
-    <property>
-        <name>hive.fetch.task.aggr</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>hive.hwi.listen.host</name>
-        <value>0.0.0.0</value>
-    </property>
-
-    <property>
-        <name>hive.hwi.listen.port</name>
-        <value>7002</value>
-    </property>
-
-    <property>
-        <name>hive.llap.daemon.output.service.port</name>
-        <value>7009</value>
-    </property>
-
-    <property>
-        <name>hive.llap.daemon.rpc.port</name>
-        <value>7007</value>
-    </property>
-
-    <property>
-        <name>hive.llap.daemon.web.port</name>
-        <value>7008</value>
-    </property>
-
-    <property>
-        <name>hive.llap.daemon.yarn.shuffle.port</name>
-        <value>7006</value>
-    </property>
-
-    <property>
-        <name>hive.llap.management.rpc.port</name>
-        <value>7005</value>
-    </property>
-
-    <property>
-        <name>hive.merge.mapfiles</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>hive.merge.mapredfiles</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>hive.merge.size.per.task</name>
-        <value>134217728</value>
-    </property>
-
-    <property>
-        <name>hive.merge.smallfiles.avgsize</name>
-        <value>134217728</value>
-    </property>
-
-    <property>
-        <name>hive.merge.sparkfiles</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>hive.merge.tezfiles</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>hive.metastore.db.encoding</name>
-        <value>UTF-8</value>
-    </property>
-
-    <property>
-        <name>hive.metastore.event.db.notification.api.auth</name>
-        <value>false</value>
-    </property>
-
-    <property>
-        <name>hive.metastore.kerberos.keytab.file</name>
-        <value>/var/krb5kdc/emr.keytab</value>
-    </property>
-
-    <property>
-        <name>hive.metastore.kerberos.principal</name>
-        <value>hadoop/_HOST@EMR-5XJSY31F</value>
-    </property>
-
-    <property>
-        <name>hive.metastore.metrics.enabled</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>hive.metastore.port</name>
-        <value>7004</value>
-    </property>
-
-    <property>
-        <name>hive.metastore.sasl.enabled</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>hive.metastore.schema.verification</name>
-        <value>false</value>
-    </property>
-
-    <property>
-        <name>hive.metastore.schema.verification.record.version</name>
-        <value>false</value>
-    </property>
-
-    <property>
-        <name>hive.metastore.uris</name>
-        <value>thrift://10.254.20.18:7004,thrift://10.254.20.22:7004</value>
-    </property>
-
-    <property>
-        <name>hive.metastore.warehouse.dir</name>
-        <value>/usr/hive/warehouse</value>
-    </property>
-
-    <property>
-        <name>hive.querylog.location</name>
-        <value>/data/emr/hive/tmp</value>
-    </property>
-
-    <property>
-        <name>hive.server2.authentication</name>
-        <value>kerberos</value>
-    </property>
-
-    <property>
-        <name>hive.server2.authentication.kerberos.keytab</name>
-        <value>/var/krb5kdc/emr.keytab</value>
-    </property>
-
-    <property>
-        <name>hive.server2.authentication.kerberos.principal</name>
-        <value>hadoop/_HOST@EMR-5XJSY31F</value>
-    </property>
-
-    <property>
-        <name>hive.server2.logging.operation.log.location</name>
-        <value>/data/emr/hive/tmp/operation_logs</value>
-    </property>
-
-    <property>
-        <name>hive.server2.metrics.enabled</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>hive.server2.support.dynamic.service.discovery</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>hive.server2.thrift.bind.host</name>
-        <value>10.254.20.28</value>
-    </property>
-
-    <property>
-        <name>hive.server2.thrift.http.port</name>
-        <value>7000</value>
-    </property>
-
-    <property>
-        <name>hive.server2.thrift.port</name>
-        <value>7001</value>
-    </property>
-
-    <property>
-        <name>hive.server2.webui.host</name>
-        <value>0.0.0.0</value>
-    </property>
-
-    <property>
-        <name>hive.server2.webui.max.threads</name>
-        <value>50</value>
-    </property>
-
-    <property>
-        <name>hive.server2.webui.port</name>
-        <value>7003</value>
-    </property>
-
-    <property>
-        <name>hive.server2.zookeeper.namespace</name>
-        <value>hiveserver2</value>
-    </property>
-
-    <property>
-        <name>hive.service.metrics.reporter</name>
-        <value>JMX</value>
-    </property>
-
-    <property>
-        <name>hive.stats.autogather</name>
-        <value>false</value>
-    </property>
-
-    <property>
-        <name>hive.tez.am.max.app.attempts</name>
-        <value>5</value>
-    </property>
-
-    <property>
-        <name>hive.tez.am.task.max.failed.attempts</name>
-        <value>10</value>
-    </property>
-
-    <property>
-        <name>hive.tez.auto.reducer.parallelism</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>hive.tez.container.size</name>
-        <value>1024</value>
-    </property>
-
-    <property>
-        <name>hive.vectorized.execution.enabled</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>hive.vectorized.execution.mapjoin.minmax.enabled</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>hive.vectorized.execution.mapjoin.native.fast.hashtable.enabled</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>hive.vectorized.execution.mapjoin.native.multikey.only.enabled</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>hive.zookeeper.client.port</name>
-        <value>2181</value>
-    </property>
-
-    <property>
-        <name>hive.zookeeper.quorum</name>
-        <value>10.254.20.23:2181,10.254.20.29:2181,10.254.20.26:2181</value>
-    </property>
-
-    <property>
-        <name>io.compression.codec.lzo.class</name>
-        <value>com.hadoop.compression.lzo.LzoCodec</value>
-    </property>
-
-    <property>
-        <name>io.compression.codecs</name>
-        <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
-    </property>
-
-    <property>
-        <name>javax.jdo.option.ConnectionDriverName</name>
-        <value>com.mysql.jdbc.Driver</value>
-    </property>
-
-    <property>
-        <name>javax.jdo.option.ConnectionPassword</name>
-        <value>UYdfjsdDigf8d3U</value>
-    </property>
-
-    <property>
-        <name>javax.jdo.option.ConnectionURL</name>
-        <value>jdbc:mysql://10.254.20.16:3306/hivemetastore?useSSL=false&amp;createDatabaseIfNotExist=true&amp;characterEncoding=UTF-8</value>
-    </property>
-
-    <property>
-        <name>javax.jdo.option.ConnectionUserName</name>
-        <value>root</value>
-    </property>
-
-    <property>
-        <name>mapreduce.cluster.local.dir</name>
-        <value>/data/emr/hive/tmp/tmp</value>
-    </property>
-
-    <property>
-        <name>spark.driver.extraLibraryPath</name>
-        <value>/usr/local/service/hadoop/lib/native:/usr/local/service/hadoop/lib/native/Linux-amd64-64/lib</value>
-    </property>
-
-    <property>
-        <name>spark.eventLog.dir</name>
-        <value>hdfs://HDFS8000912/spark-history</value>
-    </property>
-
-    <property>
-        <name>spark.eventLog.enabled</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>spark.executor.extraLibraryPath</name>
-        <value>/usr/local/service/hadoop/lib/native:/usr/local/service/hadoop/lib/native/Linux-amd64-64/lib</value>
-    </property>
-
-    <property>
-        <name>spark.history.fs.cleaner.enabled</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>spark.home</name>
-        <value>/usr/local/service/spark</value>
-    </property>
-
-    <property>
-        <name>spark.yarn.jars</name>
-        <value>hdfs:///spark/jars/*</value>
-    </property>
-
-</configuration>

yili-conf/cluster/conf/livy-client.conf

@@ -1,104 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#
-# Configurations for a Livy Client, any configurations set here will override any
-# livy or spark-default configurations.
-#
-# Before a Livy Client is able to load these configurations the folder containing
-# this file must be added to the application classpath
-#
-
-#
-# Configurations for Livy HTTPClient
-#
-
-# HTTP Request configurations
-# How long before a request times out
-# livy.client.http.connection.timeout = 10s
-# How long between data packets before a request times out
-# livy.client.http.connection.socket.timeout = 5m
-# Whether content is compressed
-# livy.client.http.content.compress.enable = true
-
-# How long before idle connections are closed
-# livy.client.http.connection.idle.timeout = 10m
-
-# Initial interval before polling for Job results
-# livy.client.http.job.initial-poll-interval = 100ms
-# Maximum interval between successive polls
-# livy.client.http.job.max-poll-interval = 5s
-
-#
-# Configurations for Livy RSCClient
-#
-
-# Configurations for registering a client with the rpc server
-# Unique client id for connections to the rpc server
-# livy.rsc.client.auth.id =
-# Secret value for authenticating client connections with server
-# livy.rsc.client.auth.secret =
-
-# Timeout when stopping a rsc client
-# livy.rsc.client.shutdown-timeout = 10s
-
-# Class of the rsc driver to use
-# livy.rsc.driver-class =
-# The kind of rsc session. Examples: pyspark or sparkr
-# livy.rsc.session.kind =
-
-# Comma-separated list of Livy RSC jars. By default Livy will upload jars from its installation
-# directory every time a session is started. By caching these files in HDFS, for example, startup
-# time of sessions on YARN can be reduced.
-#livy.rsc.jars = hdfs:/user/aidevuser/rsc-jars/*
-# Location of the SparkR package for running sparkr
-# livy.rsc.sparkr.package =
-# Location of the PySpark package for running pyspark
-# livy.rsc.pyspark.archives =
-
-# Address for the RSC driver to connect back with it's connection info.
-livy.rsc.launcher.address = please_set_correct_addresss
-# Port Range on which RPC will launch . Port range in inclusive of start and end port .
-livy.rsc.launcher.port.range = 31100~31103
-
-# How long will the RSC wait for a connection for a Livy server before shutting itself down.
-livy.rsc.server.idle-timeout = 30m
-
-# The user that should be impersonated when requesting a Livy session
-#livy.rsc.proxy-user = aidevuser
-
-# Host or IP adress of the rpc server
-livy.rsc.rpc.server.address = 0.0.0.0
-# How long the rsc client will wait when attempting to connect to the Livy server
-livy.rsc.server.connect.timeout = 5m
-
-# The logging level for the rpc channel. Possible values: TRACE, DEBUG, INFO, WARN, or ERROR
-# livy.rsc.channel.log.level =
-
-# SASL configurations for authentication
-# SASL mechanism used for authentication
-# livy.rsc.rpc.sasl.mechanisms = DIGEST-MD5
-# SASL qop used for authentication
-# livy.rsc.rpc.sasl.qop =
-
-# Time between status checks for cancelled a Job
-# livy.rsc.job-cancel.trigger-interval = 100ms
-# Time before a cancelled a Job is forced into a Cancelled state
-# livy.rsc.job-cancel.timeout = 30s
-
-# Number of statements kept in driver's memory
-# livy.rsc.retained-statements = 100

yili-conf/cluster/conf/livy.conf

@@ -1,189 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Use this keystore for the SSL certificate and key.
-# livy.keystore =
-
-# Specify the keystore password.
-# livy.keystore.password =
-#
-# Specify the key password.
-# livy.key-password =
-
-# Hadoop Credential Provider Path to get "livy.keystore.password" and "livy.key-password".
-# Credential Provider can be created using command as follow:
-# hadoop credential create "livy.keystore.password" -value "secret" -provider jceks://hdfs/path/to/livy.jceks
-# livy.hadoop.security.credential.provider.path =
-
-# What host address to start the server on. By default, Livy will bind to all network interfaces.
-# livy.server.host = 0.0.0.0
-
-# What port to start the server on.
-livy.server.port = 8998
-
-# What base path ui should work on. By default UI is mounted on "/".
-# E.g.: livy.ui.basePath = /my_livy - result in mounting UI on /my_livy/
-# livy.ui.basePath = ""
-
-# What spark master Livy sessions should use.
-livy.spark.master = yarn
-
-# What spark deploy mode Livy sessions should use.
-livy.spark.deploy-mode = cluster
-
-# Configure Livy server http request and response header size.
-# livy.server.request-header.size = 131072
-# livy.server.response-header.size = 131072
-
-# Enabled to check whether timeout Livy sessions should be stopped.
-# livy.server.session.timeout-check = true
-#
-# Whether or not to skip timeout check for a busy session
-# livy.server.session.timeout-check.skip-busy = false
-
-# Time in milliseconds on how long Livy will wait before timing out an inactive session.
-# Note that the inactive session could be busy running jobs.
-# livy.server.session.timeout = 1h
-#
-# How long a finished session state should be kept in LivyServer for query.
-# livy.server.session.state-retain.sec = 600s
-
-# If livy should impersonate the requesting users when creating a new session.
-# livy.impersonation.enabled = false
-
-# Logs size livy can cache for each session/batch. 0 means don't cache the logs.
-# livy.cache-log.size = 200
-
-# Comma-separated list of Livy RSC jars. By default Livy will upload jars from its installation
-# directory every time a session is started. By caching these files in HDFS, for example, startup
-# time of sessions on YARN can be reduced.
-#livy.rsc.jars = hdfs:/user/aidevuser/rsc-jars/*
-
-# Comma-separated list of Livy REPL jars. By default Livy will upload jars from its installation
-# directory every time a session is started. By caching these files in HDFS, for example, startup
-# time of sessions on YARN can be reduced. Please list all the repl dependencies including
-# Scala version-specific livy-repl jars, Livy will automatically pick the right dependencies
-# during session creation.
-#livy.repl.jars = hdfs:/user/aidevuser/repl_2.12-jars/*
-
-# Location of PySpark archives. By default Livy will upload the file from SPARK_HOME, but
-# by caching the file in HDFS, startup time of PySpark sessions on YARN can be reduced.
-# livy.pyspark.archives =
-
-# Location of the SparkR package. By default Livy will upload the file from SPARK_HOME, but
-# by caching the file in HDFS, startup time of R sessions on YARN can be reduced.
-# livy.sparkr.package =
-
-# List of local directories from where files are allowed to be added to user sessions. By
-# default it's empty, meaning users can only reference remote URIs when starting their
-# sessions.
-# livy.file.local-dir-whitelist =
-
-# Whether to enable csrf protection, by default it is false. If it is enabled, client should add
-# http-header "X-Requested-By" in request if the http method is POST/DELETE/PUT/PATCH.
-# livy.server.csrf-protection.enabled =
-
-# Whether to enable HiveContext in livy interpreter, if it is true hive-site.xml will be detected
-# on user request and then livy server classpath automatically.
-livy.repl.enable-hive-context = true
-
-# Recovery mode of Livy. Possible values:
-# off: Default. Turn off recovery. Every time Livy shuts down, it stops and forgets all sessions.
-# recovery: Livy persists session info to the state store. When Livy restarts, it recovers
-#           previous sessions from the state store.
-# Must set livy.server.recovery.state-store and livy.server.recovery.state-store.url to
-# configure the state store.
-# livy.server.recovery.mode = off
-# Zookeeper address used for HA and state store. e.g. host1:port1, host2:port2
-# livy.server.zookeeper.url =
-
-# Where Livy should store state to for recovery. Possible values:
-# <empty>: Default. State store disabled.
-# filesystem: Store state on a file system.
-# zookeeper: Store state in a Zookeeper instance.
-# livy.server.recovery.state-store =
-
-# For filesystem state store, the path of the state store directory. Please don't use a filesystem
-# that doesn't support atomic rename (e.g. S3). e.g. file:///tmp/livy or hdfs:///.
-# For zookeeper, the address to the Zookeeper servers. e.g. host1:port1,host2:port2
-# If livy.server.recovery.state-store is zookeeper, this config is for back-compatibility,
-# so if both this config and livy.server.zookeeper.url exist,
-# livy uses livy.server.zookeeper.url first.
-# livy.server.recovery.state-store.url =
-
-# The policy of curator connecting to zookeeper.
-# For example, m, n means retry m times and the interval of retry is n milliseconds.
-# Please use the new config: livy.server.zk.retry-policy.
-# Keep this config for back-compatibility.
-# If both this config and livy.server.zk.retry-policy exist,
-# livy uses livy.server.zk.retry-policy first.
-# livy.server.recovery.zk-state-store.retry-policy = 5,100
-
-# The policy of curator connecting to zookeeper.
-# For example, m, n means retry m times and the interval of retry is n milliseconds
-# livy.server.zk.retry-policy =
-
-# The dir in zk to store the data about session.
-# livy.server.recovery.zk-state-store.key-prefix = livy
-
-# If Livy can't find the yarn app within this time, consider it lost.
-livy.server.yarn.app-lookup-timeout = 5m
-# When the cluster is busy, we may fail to launch yarn app in app-lookup-timeout, then it would
-# cause session leakage, so we need to check session leakage.
-# How long to check livy session leakage
-# livy.server.yarn.app-leakage.check-timeout = 600s
-# how often to check livy session leakage
-# livy.server.yarn.app-leakage.check-interval = 60s
-
-# How often Livy polls YARN to refresh YARN app state.
-# livy.server.yarn.poll-interval = 5s
-#
-# Days to keep Livy server request logs.
-# livy.server.request-log-retain.days = 5
-
-# If the Livy Web UI should be included in the Livy Server. Enabled by default.
-# livy.ui.enabled = true
-
-# Whether to enable Livy server access control, if it is true then all the income requests will
-# be checked if the requested user has permission.
-# livy.server.access-control.enabled = false
-
-# Allowed users to access Livy, by default any user is allowed to access Livy. If user want to
-# limit who could access Livy, user should list all the permitted users with comma separated.
-# livy.server.access-control.allowed-users = *
-
-# A list of users with comma separated has the permission to change other user's submitted
-# session, like submitting statements, deleting session.
-# livy.server.access-control.modify-users =
-
-# A list of users with comma separated has the permission to view other user's infomation, like
-# submitted session state, statement results.
-# livy.server.access-control.view-users =
-#
-# Authentication support for Livy server
-# Livy has a built-in SPnego authentication support for HTTP requests  with below configurations.
-# livy.server.auth.type = kerberos
-# livy.server.auth.kerberos.principal = <spnego principal>
-# livy.server.auth.kerberos.keytab = <spnego keytab>
-# livy.server.auth.kerberos.name-rules = DEFAULT
-livy.server.launch.kerberos.principal = ailab
-livy.server.launch.kerberos.keytab = /opt/cluster/conf/user.keytab
-#
-# If user wants to use custom authentication filter, configurations are:
-# livy.server.auth.type = <custom>
-# livy.server.auth.<custom>.class = <class of custom auth filter>
-# livy.server.auth.<custom>.param.<foo1> = <bar1>
-# livy.server.auth.<custom>.param.<foo2> = <bar2>

yili-conf/cluster/conf/log4j.properties

@@ -1,341 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Define some default values that can be overridden by system properties
-hadoop.root.logger=INFO,console
-hadoop.log.dir=.
-hadoop.log.file=hadoop.log
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hadoop.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshold=ALL
-
-# Null Appender
-log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
-
-#
-# Rolling File Appender - cap space usage at 5gb.
-#
-hadoop.log.maxfilesize=256MB
-hadoop.log.maxbackupindex=20
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
-
-log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize}
-log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex}
-
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# Daily Rolling File Appender
-#
-
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
-
-# Rollover at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-
-#
-# TaskLog Appender
-#
-log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
-
-log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
-log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-
-#
-# HDFS block state change log from block manager
-#
-# Uncomment the following to log normal block state change
-# messages from BlockManager in NameNode.
-#log4j.logger.BlockStateChange=DEBUG
-
-#
-#Security appender
-#
-hadoop.security.logger=INFO,NullAppender
-hadoop.security.log.maxfilesize=256MB
-hadoop.security.log.maxbackupindex=20
-log4j.category.SecurityLogger=${hadoop.security.logger}
-hadoop.security.log.file=SecurityAuth-${user.name}.audit
-log4j.appender.RFAS=org.apache.log4j.RollingFileAppender
-log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
-log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}
-log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}
-
-#
-# Daily Rolling Security appender
-#
-log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
-log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
-log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd
-
-#
-# hadoop configuration logging
-#
-
-# Uncomment the following line to turn off configuration deprecation warnings.
-# log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN
-
-#
-# hdfs audit logging
-#
-hdfs.audit.logger=INFO,NullAppender
-hdfs.audit.log.maxfilesize=256MB
-hdfs.audit.log.maxbackupindex=20
-log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}
-log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false
-log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender
-log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log
-log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize}
-log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex}
-
-#
-# NameNode metrics logging.
-# The default is to retain two namenode-metrics.log files up to 64MB each.
-#
-namenode.metrics.logger=INFO,NullAppender
-log4j.logger.NameNodeMetricsLog=${namenode.metrics.logger}
-log4j.additivity.NameNodeMetricsLog=false
-log4j.appender.NNMETRICSRFA=org.apache.log4j.RollingFileAppender
-log4j.appender.NNMETRICSRFA.File=${hadoop.log.dir}/namenode-metrics.log
-log4j.appender.NNMETRICSRFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.NNMETRICSRFA.layout.ConversionPattern=%d{ISO8601} %m%n
-log4j.appender.NNMETRICSRFA.MaxBackupIndex=1
-log4j.appender.NNMETRICSRFA.MaxFileSize=64MB
-
-#
-# DataNode metrics logging.
-# The default is to retain two datanode-metrics.log files up to 64MB each.
-#
-datanode.metrics.logger=INFO,NullAppender
-log4j.logger.DataNodeMetricsLog=${datanode.metrics.logger}
-log4j.additivity.DataNodeMetricsLog=false
-log4j.appender.DNMETRICSRFA=org.apache.log4j.RollingFileAppender
-log4j.appender.DNMETRICSRFA.File=${hadoop.log.dir}/datanode-metrics.log
-log4j.appender.DNMETRICSRFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.DNMETRICSRFA.layout.ConversionPattern=%d{ISO8601} %m%n
-log4j.appender.DNMETRICSRFA.MaxBackupIndex=1
-log4j.appender.DNMETRICSRFA.MaxFileSize=64MB
-
-# Custom Logging levels
-
-#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
-#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
-#log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=DEBUG
-
-
-# AWS SDK & S3A FileSystem
-#log4j.logger.com.amazonaws=ERROR
-log4j.logger.com.amazonaws.http.AmazonHttpClient=ERROR
-#log4j.logger.org.apache.hadoop.fs.s3a.S3AFileSystem=WARN
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
-
-#
-# Job Summary Appender
-#
-# Use following logger to send summary to separate file defined by
-# hadoop.mapreduce.jobsummary.log.file :
-# hadoop.mapreduce.jobsummary.logger=INFO,JSA
-# 
-hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
-hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
-hadoop.mapreduce.jobsummary.log.maxfilesize=256MB
-hadoop.mapreduce.jobsummary.log.maxbackupindex=20
-log4j.appender.JSA=org.apache.log4j.RollingFileAppender
-log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file}
-log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize}
-log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex}
-log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
-log4j.appender.JSA.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger}
-log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false
-
-#
-# shuffle connection log from shuffleHandler
-# Uncomment the following line to enable logging of shuffle connections
-# log4j.logger.org.apache.hadoop.mapred.ShuffleHandler.audit=DEBUG
-
-#
-# Yarn ResourceManager Application Summary Log
-#
-# Set the ResourceManager summary log filename
-yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log
-# Set the ResourceManager summary log level and appender
-yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger}
-#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY
-
-# To enable AppSummaryLogging for the RM,
-# set yarn.server.resourcemanager.appsummary.logger to
-# <LEVEL>,RMSUMMARY in hadoop-env.sh
-
-# Appender for ResourceManager Application Summary Log
-# Requires the following properties to be set
-#    - hadoop.log.dir (Hadoop Log directory)
-#    - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename)
-#    - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender)
-
-log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger}
-log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false
-log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender
-log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file}
-log4j.appender.RMSUMMARY.MaxFileSize=256MB
-log4j.appender.RMSUMMARY.MaxBackupIndex=20
-log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout
-log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-
-#
-# YARN ResourceManager audit logging
-#
-rm.audit.logger=INFO,NullAppender
-rm.audit.log.maxfilesize=256MB
-rm.audit.log.maxbackupindex=20
-log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger=${rm.audit.logger}
-log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger=false
-log4j.appender.RMAUDIT=org.apache.log4j.RollingFileAppender
-log4j.appender.RMAUDIT.File=${hadoop.log.dir}/rm-audit.log
-log4j.appender.RMAUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.RMAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.RMAUDIT.MaxFileSize=${rm.audit.log.maxfilesize}
-log4j.appender.RMAUDIT.MaxBackupIndex=${rm.audit.log.maxbackupindex}
-
-#
-# YARN NodeManager audit logging
-#
-nm.audit.logger=INFO,NullAppender
-nm.audit.log.maxfilesize=256MB
-nm.audit.log.maxbackupindex=20
-log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger=${nm.audit.logger}
-log4j.additivity.org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger=false
-log4j.appender.NMAUDIT=org.apache.log4j.RollingFileAppender
-log4j.appender.NMAUDIT.File=${hadoop.log.dir}/nm-audit.log
-log4j.appender.NMAUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.NMAUDIT.layout.ConversionPattern=%d{ISO8601}%p %c{2}: %m%n
-log4j.appender.NMAUDIT.MaxFileSize=${nm.audit.log.maxfilesize}
-log4j.appender.NMAUDIT.MaxBackupIndex=${nm.audit.log.maxbackupindex}
-
-# HS audit log configs
-#mapreduce.hs.audit.logger=INFO,HSAUDIT
-#log4j.logger.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=${mapreduce.hs.audit.logger}
-#log4j.additivity.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=false
-#log4j.appender.HSAUDIT=org.apache.log4j.DailyRollingFileAppender
-#log4j.appender.HSAUDIT.File=${hadoop.log.dir}/hs-audit.log
-#log4j.appender.HSAUDIT.layout=org.apache.log4j.PatternLayout
-#log4j.appender.HSAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-#log4j.appender.HSAUDIT.DatePattern=.yyyy-MM-dd
-
-# Http Server Request Logs
-#log4j.logger.http.requests.namenode=INFO,namenoderequestlog
-#log4j.appender.namenoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender
-#log4j.appender.namenoderequestlog.Filename=${hadoop.log.dir}/jetty-namenode-yyyy_mm_dd.log
-#log4j.appender.namenoderequestlog.RetainDays=3
-
-#log4j.logger.http.requests.datanode=INFO,datanoderequestlog
-#log4j.appender.datanoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender
-#log4j.appender.datanoderequestlog.Filename=${hadoop.log.dir}/jetty-datanode-yyyy_mm_dd.log
-#log4j.appender.datanoderequestlog.RetainDays=3
-
-#log4j.logger.http.requests.resourcemanager=INFO,resourcemanagerrequestlog
-#log4j.appender.resourcemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
-#log4j.appender.resourcemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-resourcemanager-yyyy_mm_dd.log
-#log4j.appender.resourcemanagerrequestlog.RetainDays=3
-
-#log4j.logger.http.requests.jobhistory=INFO,jobhistoryrequestlog
-#log4j.appender.jobhistoryrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
-#log4j.appender.jobhistoryrequestlog.Filename=${hadoop.log.dir}/jetty-jobhistory-yyyy_mm_dd.log
-#log4j.appender.jobhistoryrequestlog.RetainDays=3
-
-#log4j.logger.http.requests.nodemanager=INFO,nodemanagerrequestlog
-#log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
-#log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log
-#log4j.appender.nodemanagerrequestlog.RetainDays=3
-
-
-# WebHdfs request log on datanodes
-# Specify -Ddatanode.webhdfs.logger=INFO,HTTPDRFA on datanode startup to
-# direct the log to a separate file.
-#datanode.webhdfs.logger=INFO,console
-#log4j.logger.datanode.webhdfs=${datanode.webhdfs.logger}
-#log4j.appender.HTTPDRFA=org.apache.log4j.DailyRollingFileAppender
-#log4j.appender.HTTPDRFA.File=${hadoop.log.dir}/hadoop-datanode-webhdfs.log
-#log4j.appender.HTTPDRFA.layout=org.apache.log4j.PatternLayout
-#log4j.appender.HTTPDRFA.layout.ConversionPattern=%d{ISO8601} %m%n
-#log4j.appender.HTTPDRFA.DatePattern=.yyyy-MM-dd
-
-
-# Appender for viewing information for errors and warnings
-yarn.ewma.cleanupInterval=300
-yarn.ewma.messageAgeLimitSeconds=86400
-yarn.ewma.maxUniqueMessages=250
-log4j.appender.EWMA=org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender
-log4j.appender.EWMA.cleanupInterval=${yarn.ewma.cleanupInterval}
-log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds}
-log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages}
-
-#
-# Fair scheduler state dump
-#
-# Use following logger to dump the state to a separate file
-
-#log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=DEBUG,FSSTATEDUMP
-#log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=false
-#log4j.appender.FSSTATEDUMP=org.apache.log4j.RollingFileAppender
-#log4j.appender.FSSTATEDUMP.File=${hadoop.log.dir}/fairscheduler-statedump.log
-#log4j.appender.FSSTATEDUMP.layout=org.apache.log4j.PatternLayout
-#log4j.appender.FSSTATEDUMP.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-#log4j.appender.FSSTATEDUMP.MaxFileSize=${hadoop.log.maxfilesize}
-#log4j.appender.FSSTATEDUMP.MaxBackupIndex=${hadoop.log.maxbackupindex}
-
-# Log levels of third-party libraries
-log4j.logger.org.apache.commons.beanutils=WARN

yili-conf/cluster/conf/mapred-site.xml

@@ -1,28 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-  <property>
-     <name>mapreduce.framework.name</name>
-     <value>yarn</value>
-  </property>
-  <property>
-    <name>mapreduce.application.classpath</name>
-    <value>$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*</value>
-  </property>
-</configuration>

yili-conf/cluster/conf/spark-defaults.conf

@@ -1,33 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Default system properties included when running spark-submit.
-# This is useful for setting default environmental settings.
-
-# Example:
-# spark.master                     spark://master:7077
-# spark.eventLog.enabled           true
-# spark.eventLog.dir               hdfs://namenode:8021/directory
-# spark.serializer                 org.apache.spark.serializer.KryoSerializer
-# spark.driver.memory              5g
-# spark.executor.extraJavaOptions  -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three"
-spark.master yarn
-spark.submit.deployMode cluster
-spark.sql.hive.metastore.version 2.3.7
-spark.yarn.dist.archives hdfs:/user/ailab/python_env.tar.gz#python_env
-spark.pyspark.python ./python_env/bin/python
-spark.yarn.archive hdfs:/user/ailab/spark_libs.zip

yili-conf/cluster/conf/yarn-site.xml

@@ -1,431 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<configuration>
-	
-	<property>
-		<name>yarn.acl.enable</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>yarn.app.mapreduce.am.scheduler.connection.wait.interval-ms</name>
-		<value>5000</value>
-	</property>
-	
-	<property>
-		<name>yarn.app.mapreduce.am.staging-dir</name>
-		<value>/emr/hadoop-yarn/staging</value>
-	</property>
-	
-	<property>
-		<name>yarn.authorization-provider</name>
-		<value>org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer</value>
-	</property>
-	
-	<property>
-		<name>yarn.log-aggregation-enable</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>yarn.log-aggregation.retain-check-interval-seconds</name>
-		<value>604800</value>
-	</property>
-	
-	<property>
-		<name>yarn.log-aggregation.retain-seconds</name>
-		<value>604800</value>
-	</property>
-	
-	<property>
-		<name>yarn.log.server.url</name>
-		<value>http://10.254.20.18:5024/jobhistory/logs</value>
-	</property>
-	
-	<property>
-		<name>yarn.nm.liveness-monitor.expiry-interval-ms</name>
-		<value>100000</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.address</name>
-		<value>10.254.20.28:5006</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.aux-services</name>
-		<value>mapreduce_shuffle</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
-		<value>org.apache.hadoop.mapred.ShuffleHandler</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.container-executor.class</name>
-		<value>org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.container-manager.thread-count</name>
-		<value>100</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.container-metrics.period-ms</name>
-		<value>10000</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.delete.debug-delay-sec</name>
-		<value>180</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.delete.thread-count</name>
-		<value>16</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.hostname</name>
-		<value>10.254.20.28</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.keytab</name>
-		<value>/var/krb5kdc/emr.keytab</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
-		<value>/hadoop-yarn</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.linux-container-executor.cgroups.mount-path</name>
-		<value>/sys/fs/cgroup</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name>
-		<value>false</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.linux-container-executor.group</name>
-		<value>hadoop</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user</name>
-		<value>hadoop</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
-		<value>org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.local-dirs</name>
-		<value>/data/emr/yarn/local</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.localizer.address</name>
-		<value>10.254.20.28:5007</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.log-dirs</name>
-		<value>/data/emr/yarn/logs</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.log.retain-seconds</name>
-		<value>604800</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.pmem-check-enabled</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.principal</name>
-		<value>hadoop/_HOST@EMR-5XJSY31F</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.recovery.dir</name>
-		<value>/data/emr/yarn/nodemanger-recovery</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.recovery.enabled</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.remote-app-log-dir</name>
-		<value>/emr/logs</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.resource.cpu-vcores</name>
-		<value>8</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.resource.memory-mb</name>
-		<value>29491</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
-		<value>100</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.vmem-check-enabled</name>
-		<value>false</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.vmem-pmem-ratio</name>
-		<value>8</value>
-	</property>
-	
-	<property>
-		<name>yarn.nodemanager.webapp.address</name>
-		<value>10.254.20.28:5008</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.address.rm1</name>
-		<value>10.254.20.18:5000</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.address.rm2</name>
-		<value>10.254.20.22:5000</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.admin.address.rm1</name>
-		<value>10.254.20.18:5003</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.admin.address.rm2</name>
-		<value>10.254.20.22:5003</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.cluster-id</name>
-		<value>emr-5xjsy31f</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.connect.retry-interval.ms</name>
-		<value>2000</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.ha.automatic-failover.embedded</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.ha.enabled</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.ha.rm-ids</name>
-		<value>rm1,rm2</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.keytab</name>
-		<value>/var/krb5kdc/emr.keytab</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.max-completed-applications</name>
-		<value>150</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.nodemanagers.heartbeat-interval-ms</name>
-		<value>1000</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.nodes.exclude-path</name>
-		<value>/usr/local/service/hadoop/etc/hadoop/yarnexcludedhosts</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.nodes.include-path</name>
-		<value>/usr/local/service/hadoop/etc/hadoop/yarnhosts</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.principal</name>
-		<value>hadoop/_HOST@EMR-5XJSY31F</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.recovery.enabled</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.resource-tracker.address.rm1</name>
-		<value>10.254.20.18:5002</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.resource-tracker.address.rm2</name>
-		<value>10.254.20.22:5002</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.scheduler.address.rm1</name>
-		<value>10.254.20.18:5001</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.scheduler.address.rm2</name>
-		<value>10.254.20.22:5001</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.scheduler.class</name>
-		<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.store.class</name>
-		<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.webapp.address.rm1</name>
-		<value>10.254.20.18:5004</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.webapp.address.rm2</name>
-		<value>10.254.20.22:5004</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.webapp.https.address.rm1</name>
-		<value>10.254.20.18:5005</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.webapp.https.address.rm2</name>
-		<value>10.254.20.22:5005</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.zk-address</name>
-		<value>10.254.20.23:2181,10.254.20.29:2181,10.254.20.26:2181</value>
-	</property>
-	
-	<property>
-		<name>yarn.resourcemanager.zk.state-store.address</name>
-		<value>10.254.20.23:2181,10.254.20.29:2181,10.254.20.26:2181</value>
-	</property>
-	
-	<property>
-		<name>yarn.scheduler.fair.allow-undeclared-pools</name>
-		<value>false</value>
-	</property>
-	
-	<property>
-		<name>yarn.scheduler.fair.user-as-default-queue</name>
-		<value>false</value>
-	</property>
-	
-	<property>
-		<name>yarn.scheduler.maximum-allocation-mb</name>
-		<value>262144</value>
-	</property>
-	
-	<property>
-		<name>yarn.scheduler.maximum-allocation-vcores</name>
-		<value>128</value>
-	</property>
-	
-	<property>
-		<name>yarn.scheduler.minimum-allocation-mb</name>
-		<value>16</value>
-	</property>
-	
-	<property>
-		<name>yarn.scheduler.minimum-allocation-vcores</name>
-		<value>1</value>
-	</property>
-	
-	<property>
-		<name>yarn.timeline-service.bind-host</name>
-		<value>0.0.0.0</value>
-	</property>
-	
-	<property>
-		<name>yarn.timeline-service.enabled</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>yarn.timeline-service.hostname</name>
-		<value>10.254.20.18</value>
-	</property>
-	
-	<property>
-		<name>yarn.timeline-service.http-cross-origin.enabled</name>
-		<value>true</value>
-	</property>
-	
-	<property>
-		<name>yarn.timeline-service.keytab</name>
-		<value>/var/krb5kdc/emr.keytab</value>
-	</property>
-	
-	<property>
-		<name>yarn.timeline-service.leveldb-timeline-store.path</name>
-		<value>/data/emr/yarn/timeline-db</value>
-	</property>
-	
-	<property>
-		<name>yarn.timeline-service.principal</name>
-		<value>hadoop/_HOST@EMR-5XJSY31F</value>
-	</property>
-	
-</configuration>

yili-conf/krb5.conf

@@ -1,25 +0,0 @@
-[libdefaults]
-    dns_lookup_realm = false
-    dns_lookup_kdc = false
-    ticket_lifetime = 24h
-    renew_lifetime = 7d
-    forwardable = true
-    rdns = false
-    default_realm = EMR-5XJSY31F
-    default_tgs_enctypes = des3-cbc-sha1
-    default_tkt_enctypes = des3-cbc-sha1
-    permitted_enctypes = des3-cbc-sha1
-    kdc_timeout = 3000
-    max_retries = 3
-[realms]
-    EMR-5XJSY31F = {
-        
-        kdc = 10.254.20.18:88
-        admin_server = 10.254.20.18
-        kdc = 10.254.20.22:88
-		admin_server = 10.254.20.22
-        
-    }
-
-[domain_realm]
-# .example.com = EXAMPLE.COM